From 53d48c0b08b494a7343af6ff56e24acf0c152b14 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 11:48:16 +0800 Subject: [PATCH 01/44] fix(session): converge durable activity progress --- .../app/src/context/directory-sync.test.ts | 82 +++++++++- packages/app/src/context/directory-sync.ts | 28 +++- .../context/global-sync/event-reducer.test.ts | 145 +++++++++++++++++- .../src/context/global-sync/event-reducer.ts | 54 ++++++- packages/app/src/context/server-sync.tsx | 20 ++- .../session/message-timeline.data.test.ts | 53 +++++++ .../pages/session/message-timeline.data.ts | 57 +++++-- packages/core/src/session/projector.ts | 4 + packages/core/src/v1/session.ts | 16 ++ packages/core/test/session-projector.test.ts | 56 ++++++- .../deepagent-code/src/session/compaction.ts | 4 +- .../deepagent-code/src/session/message-v2.ts | 118 +++++++++++--- .../src/session/prompt-intent.ts | 139 ++++++++++++----- packages/deepagent-code/src/session/prompt.ts | 32 +++- .../deepagent-code/src/session/session.ts | 47 +++++- .../test/session/message-v2.test.ts | 31 ++++ .../test/session/prompt-intent.test.ts | 113 +++++++++++++- packages/sdk/js/src/gen/types.gen.ts | 8 + packages/sdk/js/src/v2/gen/types.gen.ts | 8 + 19 files changed, 917 insertions(+), 98 deletions(-) diff --git a/packages/app/src/context/directory-sync.test.ts b/packages/app/src/context/directory-sync.test.ts index ac3ed9c35..0c1ef71d2 100644 --- a/packages/app/src/context/directory-sync.test.ts +++ b/packages/app/src/context/directory-sync.test.ts @@ -3,7 +3,7 @@ import { createRoot } from "solid-js" import { createStore } from "solid-js/store" import type { Message, Part } from "@deepagent-code/sdk/v2/client" import { ServerScope } from "@/utils/server-scope" -import { createDirSyncContext } from "./directory-sync" +import { createDirSyncContext, runInflight } from "./directory-sync" const state = () => createStore({ @@ -219,3 +219,83 @@ describe("directory optimistic targeting", () => { expect(current[0].message[sessionID]?.map((message) => message.id)).toEqual([canonical.id]) }) }) + +describe("directory forced session synchronization", () => { + test("runs one trailing forced refresh after an ordinary refresh completes", async () => { + const inflight = new Map>() + const trailing = new Set() + let release: (() => void) | undefined + let calls = 0 + const task = async () => { + calls += 1 + if (calls !== 1) return + await new Promise((resolve) => { + release = resolve + }) + } + + const first = runInflight(inflight, trailing, "session", false, task) + const forced = runInflight(inflight, trailing, "session", true, task) + await Promise.resolve() + expect(calls).toBe(1) + release?.() + await Promise.all([first, forced]) + + expect(calls).toBe(2) + expect(trailing.size).toBe(0) + }) + + test("coalesces concurrent forced refreshes into one trailing refresh", async () => { + const inflight = new Map>() + const trailing = new Set() + let release: (() => void) | undefined + let calls = 0 + const task = async () => { + calls += 1 + if (calls !== 1) return + await new Promise((resolve) => { + release = resolve + }) + } + + const first = runInflight(inflight, trailing, "session", false, task) + const forced = [ + runInflight(inflight, trailing, "session", true, task), + runInflight(inflight, trailing, "session", true, task), + ] + await Promise.resolve() + release?.() + await Promise.all([first, ...forced]) + + expect(calls).toBe(2) + }) + + test("still runs the trailing forced refresh after the ordinary refresh fails", async () => { + const inflight = new Map>() + const trailing = new Set() + let release: (() => void) | undefined + let calls = 0 + const task = async () => { + calls += 1 + if (calls !== 1) return + await new Promise((resolve) => { + release = resolve + }) + throw new Error("ordinary sync failed") + } + + const first = runInflight(inflight, trailing, "session", false, task) + const forced = runInflight(inflight, trailing, "session", true, task) + const outcomes = [first, forced].map((promise) => + promise.then( + () => "resolved", + (error: unknown) => (error instanceof Error ? error.message : String(error)), + ), + ) + await Promise.resolve() + release?.() + + expect(await Promise.all(outcomes)).toEqual(["ordinary sync failed", "resolved"]) + expect(calls).toBe(2) + }) +}) diff --git a/packages/app/src/context/directory-sync.ts b/packages/app/src/context/directory-sync.ts index 6c7ba80c5..ccda49edc 100644 --- a/packages/app/src/context/directory-sync.ts +++ b/packages/app/src/context/directory-sync.ts @@ -21,9 +21,21 @@ function sortParts(parts: Part[]) { return parts.filter((part) => !!part?.id).sort((a, b) => cmp(a.id, b.id)) } -function runInflight(map: Map>, key: string, task: () => Promise) { +export async function runInflight( + map: Map>, + trailing: Set, + key: string, + force: boolean, + task: () => Promise, +): Promise { const pending = map.get(key) - if (pending) return pending + if (pending) { + if (!force) return pending + trailing.add(key) + await pending.catch(() => {}) + if (!trailing.delete(key)) return + return runInflight(map, trailing, key, true, task) + } const promise = task().finally(() => { map.delete(key) }) @@ -210,7 +222,9 @@ export const createDirSyncContext = ( const initialMessagePageSize = 80 const historyMessagePageSize = 200 const inflight = new Map>() + const trailing = new Set() const inflightDiff = new Map>() + const trailingDiff = new Set() const optimistic = new Map>() const maxDirs = 30 const seen = new Map>() @@ -407,7 +421,7 @@ export const createDirSyncContext = ( const loadPlan = (sessionID: string) => serverSync.plan.sync(directory, sessionID) - return { + const result = { get data() { return current()[0] }, @@ -499,7 +513,7 @@ export const createDirSyncContext = ( }) } - return runInflight(inflight, key, async () => { + return runInflight(inflight, trailing, key, opts?.force === true, async () => { const pending = getSessionPrefetchPromise(serverSDK.scope, directory, sessionID) if (pending) { await pending @@ -568,7 +582,7 @@ export const createDirSyncContext = ( if (store.session_diff[sessionID] !== undefined && !opts?.force) return const key = keyFor(directory, sessionID) - return runInflight(inflightDiff, key, () => + return runInflight(inflightDiff, trailingDiff, key, opts?.force === true, () => retry(() => client.session.diff({ sessionID })).then((diff) => { if (!tracked(directory, sessionID)) return setStore("session_diff", sessionID, reconcile(list(diff.data), { key: "file" })) @@ -651,4 +665,8 @@ export const createDirSyncContext = ( return current()[0].path.directory }, } + serverSync.registerSessionReloader?.(directory, (sessionID) => + result.session.sync(sessionID, { force: true }), + ) + return result } diff --git a/packages/app/src/context/global-sync/event-reducer.test.ts b/packages/app/src/context/global-sync/event-reducer.test.ts index a69add1a4..70397a69c 100644 --- a/packages/app/src/context/global-sync/event-reducer.test.ts +++ b/packages/app/src/context/global-sync/event-reducer.test.ts @@ -27,6 +27,27 @@ const userMessage = (id: string, sessionID: string) => model: { providerID: "openai", modelID: "gpt" }, }) as Message +const assistantMessage = ( + id: string, + sessionID: string, + activityProgress?: Extract["activityProgress"], +) => + ({ + id, + sessionID, + role: "assistant", + parentID: "msg_user", + time: { created: 1 }, + modelID: "gpt", + providerID: "openai", + mode: "build", + agent: "assistant", + path: { cwd: "/tmp", root: "/tmp" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + activityProgress, + }) as Extract + const textPart = (id: string, sessionID: string, messageID: string) => ({ id, @@ -405,7 +426,7 @@ describe("applyDirectoryEvent", () => { info: { ...userMessage("msg_2", sessionID), role: "assistant", - } as Message, + } as unknown as Message, }, }, store, @@ -467,6 +488,128 @@ describe("applyDirectoryEvent", () => { expect(store.part_text_accum_delta[clientPart.id]).toBeUndefined() }) + test("preserves server-owned activity progress when a stale message update omits it", () => { + const sessionID = "ses_1" + const current = assistantMessage("msg_assistant", sessionID, { + activityID: "activity-1", + revision: 2, + state: "final", + }) + const [store, setStore] = createStore(baseState({ message: { [sessionID]: [current] } })) + + applyDirectoryEvent({ + event: { type: "message.updated", properties: { info: { ...current, activityProgress: undefined } } }, + store, + setStore, + push() {}, + directory: "/tmp", + loadLsp() {}, + }) + + expect((store.message[sessionID]?.[0] as Extract).activityProgress).toEqual( + current.activityProgress, + ) + }) + + test("rejects conflicting progress identity and requests a canonical session refresh", () => { + const sessionID = "ses_1" + const current = assistantMessage("msg_assistant", sessionID, { + activityID: "activity-1", + revision: 2, + state: "progress", + }) + const [store, setStore] = createStore(baseState({ message: { [sessionID]: [current] } })) + const refreshed: string[] = [] + + applyDirectoryEvent({ + event: { + type: "message.updated", + properties: { + info: assistantMessage(current.id, sessionID, { + activityID: "activity-other", + revision: 0, + state: "final", + }), + }, + }, + store, + setStore, + push() {}, + directory: "/tmp", + loadLsp() {}, + refetchSession: (id) => refreshed.push(id), + }) + + expect((store.message[sessionID]?.[0] as Extract).activityProgress).toEqual( + current.activityProgress, + ) + expect(refreshed).toEqual([sessionID]) + }) + + test("allows a same-revision activity marker to advance but never regress", () => { + const sessionID = "ses_1" + const provisional = assistantMessage("msg_assistant", sessionID, { + activityID: "activity-1", + revision: 0, + state: "provisional", + }) + const [store, setStore] = createStore(baseState({ message: { [sessionID]: [provisional] } })) + const apply = (state: "progress" | "final" | "provisional") => + applyDirectoryEvent({ + event: { + type: "message.updated", + properties: { + info: assistantMessage(provisional.id, sessionID, { + activityID: "activity-1", + revision: 0, + state, + }), + }, + }, + store, + setStore, + push() {}, + directory: "/tmp", + loadLsp() {}, + }) + + apply("progress") + apply("final") + apply("provisional") + + expect((store.message[sessionID]?.[0] as Extract).activityProgress?.state).toBe( + "final", + ) + }) + + test("rejects an invalid activity marker and requests a canonical session refresh", () => { + const sessionID = "ses_1" + const current = assistantMessage("msg_assistant", sessionID) + const [store, setStore] = createStore(baseState({ message: { [sessionID]: [current] } })) + const refreshed: string[] = [] + + applyDirectoryEvent({ + event: { + type: "message.updated", + properties: { + info: { + ...current, + activityProgress: { activityID: "", revision: -1, state: "terminal" }, + } as unknown as Message, + }, + }, + store, + setStore, + push() {}, + directory: "/tmp", + loadLsp() {}, + refetchSession: (id) => refreshed.push(id), + }) + + expect((store.message[sessionID]?.[0] as Extract).activityProgress).toBeUndefined() + expect(refreshed).toEqual([sessionID]) + }) + test("upserts and prunes message parts", () => { const sessionID = "ses_1" const messageID = "msg_1" diff --git a/packages/app/src/context/global-sync/event-reducer.ts b/packages/app/src/context/global-sync/event-reducer.ts index bbbbb3874..5ac775608 100644 --- a/packages/app/src/context/global-sync/event-reducer.ts +++ b/packages/app/src/context/global-sync/event-reducer.ts @@ -18,6 +18,46 @@ import { promptAdmissionClientMessageID } from "./prompt-admission" const SKIP_PARTS = new Set(["patch", "step-start", "step-finish"]) +type AssistantMessage = Extract +type ActivityProgress = NonNullable + +function activityProgress(value: unknown): ActivityProgress | undefined { + if (!value || typeof value !== "object") return + const marker = value as Record + if (typeof marker.activityID !== "string" || marker.activityID.length === 0) return + if (typeof marker.revision !== "number" || !Number.isInteger(marker.revision) || marker.revision < 0) return + if ( + !["provisional", "progress", "final", "interrupted", "recovery_required", "failed"].includes( + String(marker.state), + ) + ) + return + if (marker.terminalReason !== undefined && typeof marker.terminalReason !== "string") return + return value as ActivityProgress +} + +function mergeMessage(current: Message, incoming: Message) { + if (current.role !== "assistant" || incoming.role !== "assistant") return { message: incoming, conflict: false } + const existing = activityProgress(current.activityProgress) + const next = activityProgress(incoming.activityProgress) + if (incoming.activityProgress && !next) + return { + message: { ...incoming, activityProgress: existing }, + conflict: true, + } + if (!existing) return { message: incoming, conflict: false } + if (!next) return { message: { ...incoming, activityProgress: existing }, conflict: false } + if (existing.activityID !== next.activityID || existing.revision !== next.revision) + return { message: { ...incoming, activityProgress: existing }, conflict: true } + if (existing.state === next.state) return { message: incoming, conflict: false } + const rank = (progress: ActivityProgress) => + progress.state === "provisional" ? 0 : progress.state === "progress" ? 1 : 2 + if (rank(next) > rank(existing)) return { message: incoming, conflict: false } + if (rank(next) < rank(existing)) + return { message: { ...incoming, activityProgress: existing }, conflict: false } + return { message: { ...incoming, activityProgress: existing }, conflict: true } +} + export function applyGlobalEvent(input: { event: { type: string; properties?: unknown } project: Project[] @@ -91,6 +131,7 @@ export function applyDirectoryEvent(input: { options?: SessionPlanUpdateOptions, ) => void setSessionGoal?: (sessionID: string, goal: SessionGoal | undefined) => void + refetchSession?: (sessionID: string) => void retainedLimit?: number }) { const event = input.event @@ -251,7 +292,18 @@ export function applyDirectoryEvent(input: { } const result = Binary.search(messages, info.id, (m) => m.id) if (result.found) { - input.setStore("message", info.sessionID, result.index, reconcile(info)) + const existing = messages[result.index]! + const merged = mergeMessage(existing, info) + input.setStore("message", info.sessionID, result.index, reconcile(merged.message)) + if (merged.conflict) { + console.error("Conflicting activity progress update", { + sessionID: info.sessionID, + messageID: info.id, + existing: existing.role === "assistant" ? existing.activityProgress : undefined, + incoming: info.role === "assistant" ? info.activityProgress : undefined, + }) + input.refetchSession?.(info.sessionID) + } break } input.setStore( diff --git a/packages/app/src/context/server-sync.tsx b/packages/app/src/context/server-sync.tsx index 59952d9ee..322222d24 100644 --- a/packages/app/src/context/server-sync.tsx +++ b/packages/app/src/context/server-sync.tsx @@ -139,6 +139,7 @@ export function createServerSyncContextInner(_serverSDK?: ServerSDK) { const sessionLoads = new Map>() const planLoads = new Map>() const planReloads = new Set() + const sessionReloaders = new Map Promise>() const sessionMeta = new Map() const sdkFor = (directory: string) => { @@ -414,6 +415,7 @@ export function createServerSyncContextInner(_serverSDK?: ServerSDK) { queue.clear(key) sessionMeta.delete(key) sdkCache.delete(key) + sessionReloaders.delete(key) clearProviderRev(serverSDK.scope, key) clearSessionPrefetchDirectory(serverSDK.scope, key) }, @@ -597,6 +599,13 @@ export function createServerSyncContextInner(_serverSDK?: ServerSDK) { } }, setSessionGoal, + refetchSession: (sessionID) => { + const reload = sessionReloaders.get(key) + if (reload) + void reload(sessionID).catch((error) => { + console.error("Failed to recalibrate session progress", error) + }) + }, retainedLimit: sessionMeta.get(key)?.limit, vcsCache: children.vcsCache.get(key), loadLsp: () => { @@ -754,6 +763,12 @@ export function createServerSyncContextInner(_serverSDK?: ServerSDK) { goal: { set: setSessionGoal, }, + registerSessionReloader(directory: string, reload: (sessionID: string) => Promise) { + sessionReloaders.set(directoryKey(directory), reload) + }, + unregisterSessionReloader(directory: string) { + sessionReloaders.delete(directoryKey(directory)) + }, mcp: { toggle: async (directory: string, name: string) => { const key = directoryKey(directory) @@ -791,7 +806,10 @@ export function createServerSyncContext(_serverSDK?: ServerSDK) { return Object.assign(inner, { createDirSyncContext: createRefCountMap( (dir) => createDirSyncContext(dir, inner, _serverSDK), - (dir) => inner.disableMcp(dir), + (dir) => { + inner.disableMcp(dir) + inner.unregisterSessionReloader(dir) + }, directoryKey, ), }) diff --git a/packages/app/src/pages/session/message-timeline.data.test.ts b/packages/app/src/pages/session/message-timeline.data.test.ts index da70ed25e..2e7e40770 100644 --- a/packages/app/src/pages/session/message-timeline.data.test.ts +++ b/packages/app/src/pages/session/message-timeline.data.test.ts @@ -185,4 +185,57 @@ describe("message timeline activity progress", () => { rows.flatMap((row) => (row._tag === "AssistantPart" && row.group.type === "part" ? [row.group.ref.partID] : [])), ).toEqual(["prt_final_1", "prt_final_plain"]) }) + + test("uses the message marker to collapse reasoning-only revisions", async () => { + const { Timeline } = await import("./message-timeline.data") + const messages = [ + { + ...assistant("msg_reasoning_a0"), + activityProgress: { activityID: "activity-reasoning", revision: 0, state: "progress" as const }, + }, + { + ...assistant("msg_reasoning_a1"), + activityProgress: { activityID: "activity-reasoning", revision: 1, state: "progress" as const }, + }, + ] + const reasoning = (messageID: string, id: string) => + ({ id, sessionID: user.sessionID, messageID, type: "reasoning", text: id }) as Part + const parts = new Map([ + [messages[0].id, [reasoning(messages[0].id, "prt_reasoning_old")]], + [messages[1].id, [reasoning(messages[1].id, "prt_reasoning_latest")]], + ]) + + const rows = Timeline.constructMessageRows(user, (id) => parts.get(id) ?? [], messages, 0, true, "idle", false) + + expect( + rows.flatMap((row) => (row._tag === "AssistantPart" && row.group.type === "part" ? [row.group.ref.partID] : [])), + ).toEqual(["prt_reasoning_latest"]) + }) + + test("hides an older tool revision when the latest terminal message has no renderable parts", async () => { + const { Timeline } = await import("./message-timeline.data") + const old = { + ...assistant("msg_tool_old"), + activityProgress: { activityID: "activity-tool", revision: 0, state: "progress" as const }, + } + const terminal = { + ...assistant("msg_tool_terminal"), + activityProgress: { activityID: "activity-tool", revision: 1, state: "final" as const }, + } + const tool = { + id: "prt_tool_old", + sessionID: user.sessionID, + messageID: old.id, + type: "tool", + callID: "call-old", + tool: "bash", + state: { status: "completed", input: {}, output: "pending", title: "poll", time: { start: 1, end: 2 } }, + } as Part + const parts = new Map([[old.id, [tool]]]) + const visibility = Timeline.activityProgressVisibility([old, terminal], (id) => parts.get(id) ?? []) + + const rows = Timeline.constructMessageRows(user, (id) => parts.get(id) ?? [], [old], 0, true, "idle", false, visibility) + + expect(rows.some((row) => row._tag === "AssistantPart")).toBe(false) + }) }) diff --git a/packages/app/src/pages/session/message-timeline.data.ts b/packages/app/src/pages/session/message-timeline.data.ts index df35b4689..77b91613d 100644 --- a/packages/app/src/pages/session/message-timeline.data.ts +++ b/packages/app/src/pages/session/message-timeline.data.ts @@ -152,6 +152,7 @@ export namespace Timeline { const error = assistantMessages.find((m) => m.error && m.error.name !== "MessageAbortedError")?.error const assistantPartRefs = latestActivityProgress( + assistantMessages, assistantMessages.flatMap((message, messageIndex) => getMessageParts(message.id) .filter((part) => renderable(part, showReasoning)) @@ -287,30 +288,47 @@ export namespace Timeline { const refs = assistantMessages.flatMap((message, messageIndex) => getMessageParts(message.id).map((part) => ({ messageID: message.id, messageIndex, part })), ) - return new Set(latestActivityProgress(refs).map((ref) => `${ref.messageID}:${ref.part.id}`)) + return new Set( + latestActivityProgress(assistantMessages, refs).map((ref) => `${ref.messageID}:${ref.part.id}`), + ) } function latestActivityProgress( + assistantMessages: AssistantMessage[], refs: T[], visibility?: ReadonlySet, ) { - const progressByMessage = new Map>>() + const progressByMessage = new Map>>() + assistantMessages.forEach((message) => { + const marker = messageActivityProgress(message) + if (marker) progressByMessage.set(message.id, marker) + }) refs.forEach((ref) => { - const marker = activityProgress(ref.part) + const marker = partActivityProgress(ref.part) + const projected = progressByMessage.get(ref.messageID) + if ( + marker && + projected && + (marker.activityID !== projected.activityID || marker.revision !== projected.revision) + ) + console.error("Conflicting activity progress projection", { + messageID: ref.messageID, + projected, + legacy: marker, + }) + if (projected) return if (marker) progressByMessage.set(ref.messageID, marker) }) - const markerFor = (ref: T) => - activityProgress(ref.part) ?? (ref.part.type === "text" ? progressByMessage.get(ref.messageID) : undefined) + const markerFor = (ref: T) => progressByMessage.get(ref.messageID) ?? partActivityProgress(ref.part) if (visibility) return refs.filter((ref) => { if (!markerFor(ref)) return true return visibility.has(`${ref.messageID}:${ref.part.id}`) }) const selected = new Map() - refs.forEach((ref) => { - const marker = markerFor(ref) + progressByMessage.forEach((marker) => { if (!marker) return - const terminal = marker.state !== "progress" + const terminal = marker.state !== "provisional" && marker.state !== "progress" const current = selected.get(marker.activityID) if ( current && @@ -323,22 +341,37 @@ export namespace Timeline { const marker = markerFor(ref) if (!marker) return true const current = selected.get(marker.activityID) - return current?.revision === marker.revision && current.terminal === (marker.state !== "progress") + return ( + current?.revision === marker.revision && + current.terminal === (marker.state !== "provisional" && marker.state !== "progress") + ) }) } - function activityProgress(part: Part) { + function messageActivityProgress(message: AssistantMessage) { + const marker = message.activityProgress + if (!marker) return + if (!marker.activityID || !Number.isInteger(marker.revision) || marker.revision < 0) return + if ( + !["provisional", "progress", "final", "interrupted", "recovery_required", "failed"].includes(marker.state) + ) + return + return marker + } + + function partActivityProgress(part: Part) { if (part.type !== "text") return const value = part.metadata?.deepagent_activity_progress if (!value || typeof value !== "object") return const marker = value as Record if (typeof marker.activity_id !== "string" || marker.activity_id.length === 0) return if (typeof marker.revision !== "number" || !Number.isInteger(marker.revision) || marker.revision < 0) return - if (!["progress", "final", "interrupted", "recovery_required"].includes(String(marker.state))) return + if (!["provisional", "progress", "final", "interrupted", "recovery_required"].includes(String(marker.state))) + return return { activityID: marker.activity_id, revision: marker.revision, - state: marker.state as "progress" | "final" | "interrupted" | "recovery_required", + state: marker.state as "provisional" | "progress" | "final" | "interrupted" | "recovery_required", } } diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index e0dcc4c74..5ad0462ab 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -80,6 +80,10 @@ function sessionRow(info: SessionV1.SessionInfo): typeof SessionTable.$inferInse function messageData( info: (typeof SessionV1.Event.MessageUpdated.Type)["data"]["info"], ): typeof MessageTable.$inferInsert.data { + if (info.role === "assistant") { + const { id: _, sessionID: __, activityProgress: ___, ...rest } = info + return rest as DeepMutable + } const { id: _, sessionID: __, ...rest } = info return rest as DeepMutable } diff --git a/packages/core/src/v1/session.ts b/packages/core/src/v1/session.ts index 026af69f4..9594ee8b2 100644 --- a/packages/core/src/v1/session.ts +++ b/packages/core/src/v1/session.ts @@ -480,6 +480,21 @@ export const SubtaskPartInput = Schema.Struct({ }).annotate({ identifier: "SubtaskPartInput" }) export type SubtaskPartInput = Types.DeepMutable> +export const ActivityProgress = Schema.Struct({ + activityID: Schema.NonEmptyString, + revision: NonNegativeInt, + state: Schema.Literals([ + "provisional", + "progress", + "final", + "interrupted", + "recovery_required", + "failed", + ]), + terminalReason: Schema.optional(Schema.String), +}).annotate({ identifier: "ActivityProgress" }) +export type ActivityProgress = Types.DeepMutable> + export const Assistant = Schema.Struct({ ...messageBase, role: Schema.Literal("assistant"), @@ -513,6 +528,7 @@ export const Assistant = Schema.Struct({ structured: Schema.optional(Schema.Any), variant: Schema.optional(Schema.String), finish: Schema.optional(Schema.String), + activityProgress: Schema.optional(ActivityProgress), }).annotate({ identifier: "AssistantMessage" }) export type Assistant = Omit>, "error"> & { error?: AssistantError diff --git a/packages/core/test/session-projector.test.ts b/packages/core/test/session-projector.test.ts index 76c8f2154..651f1aad8 100644 --- a/packages/core/test/session-projector.test.ts +++ b/packages/core/test/session-projector.test.ts @@ -15,10 +15,11 @@ import { SessionMessage } from "@deepagent-code/core/session/message" import { Prompt } from "@deepagent-code/core/session/prompt" import { SessionMessageUpdater } from "@deepagent-code/core/session/message-updater" import { SessionProjector } from "@deepagent-code/core/session/projector" +import { SessionV1 } from "@deepagent-code/core/v1/session" import { SessionExecution } from "@deepagent-code/core/session/execution" import { SessionInput } from "@deepagent-code/core/session/input" import { SessionStore } from "@deepagent-code/core/session/store" -import { SessionInputTable, SessionMessageTable, SessionTable } from "@deepagent-code/core/session/sql" +import { MessageTable, SessionInputTable, SessionMessageTable, SessionTable } from "@deepagent-code/core/session/sql" import { testEffect } from "./lib/effect" const database = Database.layerFromPath(":memory:") @@ -44,6 +45,59 @@ const assistantRow = ( } describe("SessionProjector", () => { + it.effect("never persists computed V1 activity progress from a full message update", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "test", + directory: "/project", + title: "test", + version: "test", + }) + .run() + .pipe(Effect.orDie) + const events = yield* EventV2.Service + yield* events.publish(SessionV1.Event.MessageUpdated, { + sessionID, + info: { + id: SessionV1.MessageID.make("msg_v1_activity_progress"), + sessionID, + role: "assistant", + time: { created: 1 }, + parentID: SessionV1.MessageID.make("msg_v1_parent"), + modelID: ModelV2.ID.make("model"), + providerID: ProviderV2.ID.make("provider"), + mode: "build", + agent: "build", + path: { cwd: "/project", root: "/project" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + activityProgress: { activityID: "activity-1", revision: 3, state: "final" }, + }, + }) + + expect( + ( + yield* db + .select({ data: MessageTable.data }) + .from(MessageTable) + .where(eq(MessageTable.id, SessionV1.MessageID.make("msg_v1_activity_progress"))) + .get() + .pipe(Effect.orDie) + )?.data, + ).not.toHaveProperty("activityProgress") + }), + ) + it.effect("orders projected messages and context by durable aggregate sequence", () => Effect.gen(function* () { const { db } = yield* Database.Service diff --git a/packages/deepagent-code/src/session/compaction.ts b/packages/deepagent-code/src/session/compaction.ts index feee45881..02b844df6 100644 --- a/packages/deepagent-code/src/session/compaction.ts +++ b/packages/deepagent-code/src/session/compaction.ts @@ -919,7 +919,9 @@ export const layer = Layer.effect( time_created: continuation.message.info.time.created, time_updated: continuation.message.info.time.created, data: Object.fromEntries( - Object.entries(continuation.message.info).filter(([key]) => key !== "id" && key !== "sessionID"), + Object.entries(MessageV2.stripActivityProgress(continuation.message.info)).filter( + ([key]) => key !== "id" && key !== "sessionID", + ), ) as typeof MessageTable.$inferInsert.data, }) .run() diff --git a/packages/deepagent-code/src/session/message-v2.ts b/packages/deepagent-code/src/session/message-v2.ts index 6a437ce29..dab9ed1e9 100644 --- a/packages/deepagent-code/src/session/message-v2.ts +++ b/packages/deepagent-code/src/session/message-v2.ts @@ -44,6 +44,7 @@ import { SessionWorldStateBaselineTable, } from "@deepagent-code/core/session/sql" import { SessionPromptEpochTable } from "./prompt-epoch.sql" +import { SessionActivityProgressTable, SessionLegacyActivityTable } from "./activity-sql" import { ProviderError } from "@/provider/error" import { iife } from "@/util/iife" import { errorMessage } from "@/util/error" @@ -268,6 +269,26 @@ const info = (row: typeof MessageTable.$inferSelect) => sessionID: row.session_id, }) as Info +export function stripActivityProgress(message: T): T { + if (message.role !== "assistant" || !message.activityProgress) return message + const { activityProgress: _, ...rest } = message + return rest as T +} + +export function stripActivityProgressPart(messagePart: T): T { + if ( + messagePart.type !== "text" || + !messagePart.metadata || + !("deepagent_activity_progress" in messagePart.metadata) + ) + return messagePart + const { deepagent_activity_progress: _, ...metadata } = messagePart.metadata + return { + ...messagePart, + metadata: Object.keys(metadata).length > 0 ? metadata : undefined, + } as T +} + const part = (row: typeof PartTable.$inferSelect) => ({ ...row.data, @@ -282,31 +303,92 @@ const older = (row: Cursor) => function hydrate(db: Database.Interface["db"], rows: (typeof MessageTable.$inferSelect)[]) { const ids = rows.map((row) => row.id) const partByMessage = new Map() + const progressByMessage = new Map() return Effect.gen(function* () { if (ids.length > 0) { - const partRows = yield* db - .select({ part: PartTable }) - .from(PartTable) - .innerJoin( - MessageTable, - and(eq(MessageTable.id, PartTable.message_id), eq(MessageTable.session_id, PartTable.session_id)), - ) - .where(inArray(PartTable.message_id, ids)) - .orderBy(PartTable.message_id, PartTable.id) - .all() - .pipe(Effect.orDie) + const [partRows, progressRows] = yield* Effect.all([ + db + .select({ part: PartTable }) + .from(PartTable) + .innerJoin( + MessageTable, + and(eq(MessageTable.id, PartTable.message_id), eq(MessageTable.session_id, PartTable.session_id)), + ) + .where(inArray(PartTable.message_id, ids)) + .orderBy(PartTable.message_id, PartTable.id) + .all() + .pipe(Effect.orDie), + db + .select({ + assistantMessageID: SessionActivityProgressTable.assistant_message_id, + activityID: SessionActivityProgressTable.activity_id, + revision: SessionActivityProgressTable.revision, + progressState: SessionActivityProgressTable.state, + activityState: SessionLegacyActivityTable.state, + activitySessionID: SessionLegacyActivityTable.session_id, + terminalReason: SessionLegacyActivityTable.terminal_reason, + latestRevision: sql`( + SELECT max(latest.revision) + FROM session_activity_progress AS latest + WHERE latest.activity_id = ${SessionActivityProgressTable.activity_id} + )`, + messageData: MessageTable.data, + messageSessionID: MessageTable.session_id, + }) + .from(SessionActivityProgressTable) + .innerJoin( + SessionLegacyActivityTable, + eq(SessionLegacyActivityTable.activity_id, SessionActivityProgressTable.activity_id), + ) + .innerJoin(MessageTable, eq(MessageTable.id, SessionActivityProgressTable.assistant_message_id)) + .where(inArray(SessionActivityProgressTable.assistant_message_id, ids)) + .all() + .pipe(Effect.orDie), + ]) for (const row of partRows) { const next = part(row.part) const list = partByMessage.get(row.part.message_id) if (list) list.push(next) else partByMessage.set(row.part.message_id, [next]) } + for (const row of progressRows) { + if (row.activitySessionID !== row.messageSessionID || row.messageData.role !== "assistant") { + yield* Effect.logWarning("ignored activity progress with inconsistent Session ownership").pipe( + Effect.annotateLogs({ + activityID: row.activityID, + assistantMessageID: row.assistantMessageID, + activitySessionID: row.activitySessionID, + messageSessionID: row.messageSessionID, + messageRole: row.messageData.role, + }), + ) + continue + } + const state = + row.revision !== row.latestRevision || row.activityState === "active" + ? row.progressState + : row.activityState === "settled" + ? "final" + : row.activityState + progressByMessage.set(MessageID.make(row.assistantMessageID), { + activityID: row.activityID, + revision: row.revision, + state, + ...(row.revision === row.latestRevision && row.terminalReason + ? { terminalReason: row.terminalReason } + : {}), + }) + } } - return rows.map((row) => ({ - info: info(row), - parts: partByMessage.get(row.id) ?? [], - })) + return rows.map((row) => { + const message = info(row) + const activityProgress = progressByMessage.get(MessageID.make(row.id)) + return { + info: message.role === "assistant" && activityProgress ? { ...message, activityProgress } : message, + parts: partByMessage.get(row.id) ?? [], + } + }) }) } @@ -810,10 +892,8 @@ export const get = Effect.fn("MessageV2.get")(function* (input: { sessionID: Ses .get() .pipe(Effect.orDie) if (!row) return yield* new NotFoundError({ message: `Message not found: ${input.messageID}` }) - return { - info: info(row), - parts: yield* parts(input.messageID), - } + const result = yield* hydrate(db, [row]) + return result[0]! }) export function filterCompacted(msgs: Iterable) { diff --git a/packages/deepagent-code/src/session/prompt-intent.ts b/packages/deepagent-code/src/session/prompt-intent.ts index fab76dcbb..cab645c41 100644 --- a/packages/deepagent-code/src/session/prompt-intent.ts +++ b/packages/deepagent-code/src/session/prompt-intent.ts @@ -81,6 +81,12 @@ export type Progress = { readonly state: "provisional" | "progress" | "final" | "interrupted" | "recovery_required" } +export type ProjectionInvalidation = { + readonly activityID: string + readonly sessionID: SessionID + readonly assistantMessageID?: MessageID +} + const leaseDuration = 30_000 const fromRow = (row: typeof SessionIntentTable.$inferSelect): Receipt => ({ @@ -935,7 +941,10 @@ export const recoverActiveActivities = Effect.fn("SessionPromptIntent.recoverAct ) { const { db } = yield* Database.Service const active = yield* db - .select({ activityID: SessionLegacyActivityTable.activity_id }) + .select({ + activityID: SessionLegacyActivityTable.activity_id, + sessionID: SessionLegacyActivityTable.session_id, + }) .from(SessionLegacyActivityTable) .where( and( @@ -945,7 +954,7 @@ export const recoverActiveActivities = Effect.fn("SessionPromptIntent.recoverAct ) .all() .pipe(Effect.orDie) - yield* Effect.forEach( + return yield* Effect.forEach( active, (activity) => Effect.gen(function* () { @@ -973,7 +982,12 @@ export const recoverActiveActivities = Effect.fn("SessionPromptIntent.recoverAct assistantMessageID: MessageID.make(latest.assistant_message_id), }) : undefined - if (settled && settled.state !== "progress") return + if (settled && settled.state !== "progress") + return { + activityID: activity.activityID, + sessionID: SessionID.make(activity.sessionID), + assistantMessageID: settled.assistantMessageID, + } satisfies ProjectionInvalidation const recoveryProgressState = settled?.state ?? latest?.state const recoveryReason = settled ? "process restarted after settled activity progress" @@ -1015,19 +1029,44 @@ export const recoverActiveActivities = Effect.fn("SessionPromptIntent.recoverAct { behavior: "immediate" }, ) .pipe(Effect.orDie) + return { + activityID: activity.activityID, + sessionID: SessionID.make(activity.sessionID), + ...(latest ? { assistantMessageID: MessageID.make(latest.assistant_message_id) } : {}), + } satisfies ProjectionInvalidation }), - { discard: true }, ) - return active.length }) export const interruptActivity = Effect.fn("SessionPromptIntent.interruptActivity")(function* (activityID: string) { const { db } = yield* Database.Service - yield* db - .update(SessionLegacyActivityTable) - .set({ state: "interrupted", terminal_reason: "aborted_before_provider_settlement", settled_at: Date.now() }) - .where(and(eq(SessionLegacyActivityTable.activity_id, activityID), eq(SessionLegacyActivityTable.state, "active"))) - .run() + return yield* db + .transaction( + (tx) => + Effect.gen(function* () { + const updated = yield* tx + .update(SessionLegacyActivityTable) + .set({ state: "interrupted", terminal_reason: "aborted_before_provider_settlement", settled_at: Date.now() }) + .where( + and(eq(SessionLegacyActivityTable.activity_id, activityID), eq(SessionLegacyActivityTable.state, "active")), + ) + .returning({ sessionID: SessionLegacyActivityTable.session_id }) + .get() + if (!updated) return + const latest = yield* tx + .select({ assistantMessageID: SessionActivityProgressTable.assistant_message_id }) + .from(SessionActivityProgressTable) + .where(eq(SessionActivityProgressTable.activity_id, activityID)) + .orderBy(sql`${SessionActivityProgressTable.revision} DESC`) + .get() + return { + activityID, + sessionID: SessionID.make(updated.sessionID), + ...(latest ? { assistantMessageID: MessageID.make(latest.assistantMessageID) } : {}), + } satisfies ProjectionInvalidation + }), + { behavior: "immediate" }, + ) .pipe(Effect.orDie) }) @@ -1035,39 +1074,57 @@ export const retireDisabledSteerActivity = Effect.fn("SessionPromptIntent.retire sessionID: SessionID, ) { const { db } = yield* Database.Service - const activity = yield* db - .select({ activityID: SessionLegacyActivityTable.activity_id }) - .from(SessionLegacyActivityTable) - .innerJoin( - SessionActivityAdmissionTable, - eq(SessionActivityAdmissionTable.admission_id, SessionLegacyActivityTable.trigger_admission_id), - ) - .where( - and( - eq(SessionLegacyActivityTable.session_id, sessionID), - eq(SessionLegacyActivityTable.state, "active"), - eq(SessionActivityAdmissionTable.delivery, "steer"), - ), - ) - .get() - .pipe(Effect.orDie) - if (!activity) return false - yield* db - .update(SessionLegacyActivityTable) - .set({ - state: "interrupted", - terminal_reason: "steering_disabled_before_absorption", - settled_at: Date.now(), - }) - .where( - and( - eq(SessionLegacyActivityTable.activity_id, activity.activityID), - eq(SessionLegacyActivityTable.state, "active"), - ), + return yield* db + .transaction( + (tx) => + Effect.gen(function* () { + const activity = yield* tx + .select({ activityID: SessionLegacyActivityTable.activity_id }) + .from(SessionLegacyActivityTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.admission_id, SessionLegacyActivityTable.trigger_admission_id), + ) + .where( + and( + eq(SessionLegacyActivityTable.session_id, sessionID), + eq(SessionLegacyActivityTable.state, "active"), + eq(SessionActivityAdmissionTable.delivery, "steer"), + ), + ) + .get() + if (!activity) return + const updated = yield* tx + .update(SessionLegacyActivityTable) + .set({ + state: "interrupted", + terminal_reason: "steering_disabled_before_absorption", + settled_at: Date.now(), + }) + .where( + and( + eq(SessionLegacyActivityTable.activity_id, activity.activityID), + eq(SessionLegacyActivityTable.state, "active"), + ), + ) + .returning({ activityID: SessionLegacyActivityTable.activity_id }) + .get() + if (!updated) return + const latest = yield* tx + .select({ assistantMessageID: SessionActivityProgressTable.assistant_message_id }) + .from(SessionActivityProgressTable) + .where(eq(SessionActivityProgressTable.activity_id, activity.activityID)) + .orderBy(sql`${SessionActivityProgressTable.revision} DESC`) + .get() + return { + activityID: activity.activityID, + sessionID, + ...(latest ? { assistantMessageID: MessageID.make(latest.assistantMessageID) } : {}), + } satisfies ProjectionInvalidation + }), + { behavior: "immediate" }, ) - .run() .pipe(Effect.orDie) - return true }) const progress = (row: typeof SessionActivityProgressTable.$inferSelect): Progress => ({ diff --git a/packages/deepagent-code/src/session/prompt.ts b/packages/deepagent-code/src/session/prompt.ts index b9ffd3c14..51a18a82b 100644 --- a/packages/deepagent-code/src/session/prompt.ts +++ b/packages/deepagent-code/src/session/prompt.ts @@ -691,12 +691,20 @@ export const layer = Layer.effect( ) const database = yield* Database.Service const { db } = database + const publishActivityProjection = (input: SessionPromptIntent.ProjectionInvalidation) => + input.assistantMessageID + ? sessions.publishMessageProjection({ + sessionID: input.sessionID, + messageID: input.assistantMessageID, + }) + : Effect.void yield* recoverProviderReceiptsOnStartup() yield* SessionPromptIntent.recoverActiveActivities().pipe( Effect.provideService(Database.Service, database), - Effect.tap((count) => - count > 0 - ? Effect.logWarning(`marked ${count} legacy activities recovery_required after restart`) + Effect.tap((activities) => Effect.forEach(activities, publishActivityProjection, { discard: true })), + Effect.tap((activities) => + activities.length > 0 + ? Effect.logWarning(`marked ${activities.length} legacy activities recovery_required after restart`) : Effect.void, ), ) @@ -2223,6 +2231,7 @@ export const layer = Layer.effect( if (!flags.v4Steering) yield* SessionPromptIntent.retireDisabledSteerActivity(input.sessionID).pipe( Effect.provideService(Database.Service, database), + Effect.flatMap((activity) => (activity ? publishActivityProjection(activity) : Effect.void)), ) const messageID = input.messageID ?? MessageID.ascending() const intentID = input.intentID ?? `legacy-prompt:${input.sessionID}:${messageID}` @@ -2827,6 +2836,10 @@ export const layer = Layer.effect( : undefined const publishActivityProgress = (progress: SessionPromptIntent.Progress) => Effect.gen(function* () { + yield* sessions.publishMessageProjection({ + sessionID, + messageID: progress.assistantMessageID, + }) if (!progress.textPartID) return const messages = yield* sessions.messages({ sessionID }).pipe(Effect.orDie) const parts = messages @@ -3237,6 +3250,7 @@ export const layer = Layer.effect( if (legacyActivity && !activityProgressFinalizer.value) yield* SessionPromptIntent.interruptActivity(legacyActivity.activityID).pipe( Effect.provideService(Database.Service, database), + Effect.flatMap((activity) => (activity ? publishActivityProjection(activity) : Effect.void)), ) }), ) @@ -3569,7 +3583,10 @@ export const layer = Layer.effect( activityID: progressActivity.activityID, assistantMessageID: handle.message.id, providerReceiptID: receiptID, - }).pipe(Effect.provideService(Database.Service, database)) + }).pipe( + Effect.provideService(Database.Service, database), + Effect.tap(publishActivityProgress), + ) if (progressActivity) activityProgressFinalizer.value = () => SessionPromptIntent.settleProgress({ @@ -5093,7 +5110,12 @@ function stableJson(value: unknown, seen = new WeakSet()): string { /** @internal Exported for deterministic receipt verification. */ function providerResponseFingerprint(response: SessionV1.WithParts) { - return Hash.sha256(stableJson(response)) + return Hash.sha256( + stableJson({ + ...response, + info: MessageV2.stripActivityProgress(response.info), + }), + ) } /** @internal Exported for testing */ diff --git a/packages/deepagent-code/src/session/session.ts b/packages/deepagent-code/src/session/session.ts index 9b7675f61..11554f649 100644 --- a/packages/deepagent-code/src/session/session.ts +++ b/packages/deepagent-code/src/session/session.ts @@ -726,6 +726,10 @@ export interface Interface { readonly children: (parentID: SessionID) => Effect.Effect readonly remove: (sessionID: SessionID) => Effect.Effect readonly updateMessage: (msg: T) => Effect.Effect + readonly publishMessageProjection: (input: { + sessionID: SessionID + messageID: MessageID + }) => Effect.Effect readonly removeMessage: (input: { sessionID: SessionID; messageID: MessageID }) => Effect.Effect readonly removePart: (input: { sessionID: SessionID; messageID: MessageID; partID: PartID }) => Effect.Effect readonly getPart: (input: { @@ -925,10 +929,30 @@ export const layer: Layer.Layer< .pipe(Effect.orDie) if (existing && existing.session_id !== msg.sessionID) return yield* Effect.die(`Session.updateMessage: message ${msg.id} belongs to another Session`) - yield* events.publish(SessionV1.Event.MessageUpdated, { sessionID: msg.sessionID, info: msg }) + yield* events.publish(SessionV1.Event.MessageUpdated, { + sessionID: msg.sessionID, + info: MessageV2.stripActivityProgress(msg), + }) return msg }).pipe(Effect.withSpan("Session.updateMessage")) + const publishMessageProjection: Interface["publishMessageProjection"] = Effect.fn( + "Session.publishMessageProjection", + )(function* (input) { + const message = yield* MessageV2.get(input).pipe( + Effect.provideService(Database.Service, database), + Effect.catchIf(NotFoundError.isInstance, () => + Effect.logWarning("cannot publish activity progress for a missing assistant message").pipe( + Effect.annotateLogs(input), + Effect.as(undefined), + ), + ), + ) + if (!message) return + yield* events.publish(SessionV1.Event.MessageUpdated, { sessionID: input.sessionID, info: message.info }) + return message.info + }) + const updatePart = (part: T): Effect.Effect => Effect.gen(function* () { yield* requireMessageOwnership({ sessionID: part.sessionID, messageID: part.messageID }).pipe(Effect.orDie) @@ -1185,17 +1209,19 @@ export const layer: Layer.Layer< }) for (const row of rows) { + const message = MessageV2.stripActivityProgress(row.info) const messageEventID = EventV2.ID.make( `evt_${Hash.sha256(`fork-event:v1:${intentID}:message:${row.info.id}`).slice(0, 26)}`, ) yield* publishOnce((commit) => events.publish( SessionV1.Event.MessageUpdated, - { sessionID: target.id, info: row.info }, + { sessionID: target.id, info: message }, { id: messageEventID, idempotent: true, commit }, ), ) for (const part of row.parts) { + const projectedPart = MessageV2.stripActivityProgressPart(part) const partEventID = EventV2.ID.make( `evt_${Hash.sha256(`fork-event:v1:${intentID}:part:${part.id}`).slice(0, 26)}`, ) @@ -1204,8 +1230,11 @@ export const layer: Layer.Layer< SessionV1.Event.PartUpdated, { sessionID: target.id, - part, - time: part.type === "text" ? (part.time?.start ?? row.info.time.created) : row.info.time.created, + part: projectedPart, + time: + projectedPart.type === "text" + ? (projectedPart.time?.start ?? row.info.time.created) + : row.info.time.created, }, { id: partEventID, idempotent: true, commit }, ), @@ -2361,21 +2390,24 @@ export const layer: Layer.Layer< time_created: message.info.time.created, time_updated: message.info.time.created, data: Object.fromEntries( - Object.entries(message.info).filter(([key]) => key !== "id" && key !== "sessionID"), + Object.entries(MessageV2.stripActivityProgress(message.info)).filter( + ([key]) => key !== "id" && key !== "sessionID", + ), ) as typeof MessageTable.$inferInsert.data, }) .run() for (const part of message.parts) { + const clonedPart = MessageV2.stripActivityProgressPart(part) yield* db .insert(PartTable) .values({ - id: part.id, + id: clonedPart.id, message_id: message.info.id, session_id: session.id, time_created: message.info.time.created, time_updated: message.info.time.created, data: Object.fromEntries( - Object.entries(part).filter( + Object.entries(clonedPart).filter( ([key]) => key !== "id" && key !== "messageID" && key !== "sessionID", ), ) as typeof PartTable.$inferInsert.data, @@ -3159,6 +3191,7 @@ export const layer: Layer.Layer< children, remove, updateMessage, + publishMessageProjection, removeMessage, removePart, updatePart, diff --git a/packages/deepagent-code/test/session/message-v2.test.ts b/packages/deepagent-code/test/session/message-v2.test.ts index 99e322879..492aea951 100644 --- a/packages/deepagent-code/test/session/message-v2.test.ts +++ b/packages/deepagent-code/test/session/message-v2.test.ts @@ -113,6 +113,37 @@ function basePart(messageID: string, id: string) { } describe("session.message-v2.toModelMessage", () => { + test("does not expose computed activity progress to the provider projection", async () => { + const assistantID = "msg_activity_projection" + const plain: SessionV1.WithParts = { + info: assistantInfo(assistantID, "msg_parent"), + parts: [ + { + ...basePart(assistantID, "p1"), + type: "reasoning", + text: "inspect state", + time: { start: 0, end: 1 }, + }, + { ...basePart(assistantID, "p2"), type: "text", text: "working" }, + ] as SessionV1.Part[], + } + const projected: SessionV1.WithParts = { + ...plain, + info: { + ...plain.info, + activityProgress: { + activityID: "activity-1", + revision: 8, + state: "progress", + }, + } as SessionV1.Assistant, + } + + expect(await MessageV2.toModelMessages([projected], model)).toStrictEqual( + await MessageV2.toModelMessages([plain], model), + ) + }) + test("filters out messages with no parts", async () => { const input: SessionV1.WithParts[] = [ { diff --git a/packages/deepagent-code/test/session/prompt-intent.test.ts b/packages/deepagent-code/test/session/prompt-intent.test.ts index 71897900f..13c9e3e5c 100644 --- a/packages/deepagent-code/test/session/prompt-intent.test.ts +++ b/packages/deepagent-code/test/session/prompt-intent.test.ts @@ -15,6 +15,7 @@ import { import { SessionV1 } from "@deepagent-code/core/v1/session" import { eq } from "drizzle-orm" import { Effect } from "effect" +import { MessageV2 } from "../../src/session/message-v2" import { SessionMutationEpoch } from "../../src/session/mutation-epoch" import { SessionPromptIntent } from "../../src/session/prompt-intent" import { @@ -294,8 +295,10 @@ describe("SessionPromptIntent", () => { providerReceiptID: "receipt-progress-final", }) - expect(yield* SessionPromptIntent.recoverActiveActivities()).toBe(0) - expect(yield* SessionPromptIntent.recoverActiveActivities("next-process-owner")).toBe(1) + expect(yield* SessionPromptIntent.recoverActiveActivities()).toEqual([]) + expect(yield* SessionPromptIntent.recoverActiveActivities("next-process-owner")).toEqual([ + { activityID: activity.activityID, sessionID, assistantMessageID: assistantID }, + ]) expect( yield* db .select() @@ -434,7 +437,9 @@ describe("SessionPromptIntent", () => { providerReceiptID: "receipt-progress-tool", }) - expect(yield* SessionPromptIntent.recoverActiveActivities("next-process-owner")).toBe(1) + expect(yield* SessionPromptIntent.recoverActiveActivities("next-process-owner")).toEqual([ + { activityID: activity.activityID, sessionID, assistantMessageID: assistantID }, + ]) expect( yield* db .select() @@ -457,6 +462,108 @@ describe("SessionPromptIntent", () => { }), ) + it.effect("projects reasoning-only progress from durable authority without persisting it in message JSON", () => + Effect.gen(function* () { + yield* setup + const first = yield* claim({ + intentID: "intent_reasoning_projection", + messageID: MessageID.make("msg_reasoning_projection_user"), + }) + if (first.kind !== "claimed") return + yield* SessionPromptIntent.complete({ + intentID: first.receipt.intentID, + ownerToken: first.receipt.ownerToken, + messageID: first.receipt.messageID, + delivery: "turn", + }) + const activity = yield* SessionPromptIntent.activityForMessage({ + sessionID, + messageID: first.receipt.messageID, + }) + if (!activity) return + const assistantID = MessageID.make("msg_reasoning_projection_assistant") + const { db } = yield* Database.Service + yield* db + .insert(MessageTable) + .values({ + id: assistantID, + session_id: sessionID, + time_created: 20, + data: { + role: "assistant", + parentID: first.receipt.messageID, + mode: "build", + agent: "build", + path: { cwd: "/project", root: "/project" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + modelID: ModelV2.ID.make("test"), + providerID: ProviderV2.ID.make("test"), + time: { created: 20 }, + } as typeof MessageTable.$inferInsert.data, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(PartTable) + .values({ + id: PartID.make("prt_reasoning_projection"), + message_id: assistantID, + session_id: sessionID, + time_created: 20, + data: { type: "reasoning", text: "checking remote state" } as typeof PartTable.$inferInsert.data, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: "receipt-reasoning-projection", + request_ordinal: 1, + session_id: sessionID, + user_message_id: first.receipt.messageID, + assistant_message_id: assistantID, + provider_id: "test", + model_id: "test", + registry_tool_ids: [], + permission_filtered_tool_ids: [], + final_offered_tool_ids: [], + call_ids: [], + provider_state: "preparing", + request_state: "prepared", + created_at: 20, + }) + .run() + .pipe(Effect.orDie) + yield* SessionPromptIntent.beginProgress({ + activityID: activity.activityID, + assistantMessageID: assistantID, + providerReceiptID: "receipt-reasoning-projection", + }) + + expect((yield* MessageV2.get({ sessionID, messageID: assistantID })).info).toMatchObject({ + activityProgress: { activityID: activity.activityID, revision: 0, state: "provisional" }, + }) + expect( + (yield* MessageV2.page({ sessionID, limit: 10 })).items.find((item) => item.info.id === assistantID)?.info, + ).toMatchObject({ activityProgress: { activityID: activity.activityID, revision: 0, state: "provisional" } }) + + yield* SessionPromptIntent.interruptActivity(activity.activityID) + expect((yield* MessageV2.get({ sessionID, messageID: assistantID })).info).toMatchObject({ + activityProgress: { + activityID: activity.activityID, + revision: 0, + state: "interrupted", + terminalReason: "aborted_before_provider_settlement", + }, + }) + expect( + (yield* db.select({ data: MessageTable.data }).from(MessageTable).where(eq(MessageTable.id, assistantID)).get()) + ?.data, + ).not.toHaveProperty("activityProgress") + }), + ) + it.effect("a revert epoch prevents an old direct request from materializing any message", () => Effect.gen(function* () { yield* setup diff --git a/packages/sdk/js/src/gen/types.gen.ts b/packages/sdk/js/src/gen/types.gen.ts index 49a1b3740..d8d300136 100644 --- a/packages/sdk/js/src/gen/types.gen.ts +++ b/packages/sdk/js/src/gen/types.gen.ts @@ -376,6 +376,13 @@ export type PlanProtocolViolation = { } } +export type ActivityProgress = { + activityID: string + revision: number + state: "provisional" | "progress" | "final" | "interrupted" | "recovery_required" | "failed" + terminalReason?: string +} + export type AssistantMessage = { id: string sessionID: string @@ -421,6 +428,7 @@ export type AssistantMessage = { structured?: unknown variant?: string finish?: string + activityProgress?: ActivityProgress } export type Message = UserMessage | AssistantMessage diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index 49a1b3740..d8d300136 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -376,6 +376,13 @@ export type PlanProtocolViolation = { } } +export type ActivityProgress = { + activityID: string + revision: number + state: "provisional" | "progress" | "final" | "interrupted" | "recovery_required" | "failed" + terminalReason?: string +} + export type AssistantMessage = { id: string sessionID: string @@ -421,6 +428,7 @@ export type AssistantMessage = { structured?: unknown variant?: string finish?: string + activityProgress?: ActivityProgress } export type Message = UserMessage | AssistantMessage From 1a9e440ed026756be35e34c604f393c2e865b8fd Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 12:19:34 +0800 Subject: [PATCH 02/44] fix(app): preserve activity projection on stale pages --- .../app/src/context/directory-sync.test.ts | 78 +++++++++++++++++++ packages/app/src/context/directory-sync.ts | 41 ++++++++-- .../src/context/global-sync/event-reducer.ts | 2 +- packages/sdk/openapi.json | 25 ++++++ 4 files changed, 139 insertions(+), 7 deletions(-) diff --git a/packages/app/src/context/directory-sync.test.ts b/packages/app/src/context/directory-sync.test.ts index 0c1ef71d2..62fe24c87 100644 --- a/packages/app/src/context/directory-sync.test.ts +++ b/packages/app/src/context/directory-sync.test.ts @@ -23,6 +23,27 @@ const userMessage = (id: string, sessionID: string): Message => ({ model: { providerID: "openai", modelID: "gpt" }, }) +const assistantMessage = ( + id: string, + sessionID: string, + activityProgress?: Extract["activityProgress"], +): Extract => + ({ + id, + sessionID, + role: "assistant", + parentID: "msg_user", + time: { created: 1 }, + modelID: "gpt", + providerID: "openai", + mode: "build", + agent: "assistant", + path: { cwd: "/tmp", root: "/tmp" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + activityProgress, + }) as Extract + describe("directory optimistic targeting", () => { test("writes and removes an explicit worktree optimistic message in that child store", () => createRoot((dispose) => { @@ -218,6 +239,63 @@ describe("directory optimistic targeting", () => { expect(current[0].message[sessionID]?.map((message) => message.id)).toEqual([canonical.id]) }) + + test("preserves an activity marker when a stale page returns without the computed projection", async () => { + let releaseStalePage: (() => void) | undefined + const current = state() + const serverSync = { + child() { + return current + }, + plan: { + async sync() {}, + }, + } as unknown as Parameters[1] + const sync = createDirSyncContext("/repo/main", serverSync, { + scope: ServerScope.local, + createClient() { + return { + session: { + async get() { + return { data: { id: "ses_1" } } + }, + async messages() { + await new Promise((resolve) => { + releaseStalePage = resolve + }) + return { + data: [ + { + info: assistantMessage("msg_assistant", "ses_1"), + parts: [], + }, + ], + response: { headers: new Headers() }, + } + }, + }, + } + }, + } as unknown as Parameters[2]) + const sessionID = "ses_1" + const marker = { + activityID: "activity-1", + revision: 2, + state: "interrupted" as const, + terminalReason: "AbortError", + } + current[1]("session", [current[0].session.length], { id: sessionID }) + const stale = sync.session.sync(sessionID, { force: true }) + await Promise.resolve() + expect(releaseStalePage).toBeDefined() + current[1]("message", sessionID, [assistantMessage("msg_assistant", sessionID, marker)]) + releaseStalePage?.() + await stale + + const result = current[0].message[sessionID]?.[0] + expect(result?.role).toBe("assistant") + if (result?.role === "assistant") expect(result.activityProgress).toEqual(marker) + }) }) describe("directory forced session synchronization", () => { diff --git a/packages/app/src/context/directory-sync.ts b/packages/app/src/context/directory-sync.ts index ccda49edc..81fb9bce3 100644 --- a/packages/app/src/context/directory-sync.ts +++ b/packages/app/src/context/directory-sync.ts @@ -14,6 +14,7 @@ import { diffs as list, message as clean } from "@/utils/diffs" import { createServerSdkContext, useServerSDK } from "./server-sdk" import { type createServerSyncContextInner } from "./server-sync" import { promptAdmissionClientMessageID } from "./global-sync/prompt-admission" +import { mergeMessage } from "./global-sync/event-reducer" const SKIP_PARTS = new Set(["patch", "step-start", "step-finish"]) @@ -53,10 +54,28 @@ const isNotFound = (error: unknown) => error.cause !== null && (error.cause as { status?: unknown }).status === 404 -function merge(a: readonly T[], b: readonly T[]) { - const map = new Map(a.map((item) => [item.id, item] as const)) - for (const item of b) map.set(item.id, item) - return [...map.values()].sort((x, y) => cmp(x.id, y.id)) +function mergePageMessages( + base: readonly Message[], + incoming: readonly Message[], + previous: readonly Message[], +) { + const messages = new Map(base.map((item) => [item.id, item] as const)) + const previousByID = new Map(previous.map((item) => [item.id, item] as const)) + let conflict = false + for (const item of incoming) { + const current = previousByID.get(item.id) ?? messages.get(item.id) + if (!current) { + messages.set(item.id, item) + continue + } + const result = mergeMessage(current, item) + messages.set(item.id, result.message) + conflict ||= result.conflict + } + return { + messages: [...messages.values()].sort((a, b) => cmp(a.id, b.id)), + conflict, + } } type OptimisticStore = { @@ -228,6 +247,7 @@ export const createDirSyncContext = ( const optimistic = new Map>() const maxDirs = 30 const seen = new Map>() + let requestSessionSync: ((sessionID: string) => void) | undefined const [meta, setMeta] = createStore({ limit: {} as Record, cursor: {} as Record, @@ -381,8 +401,11 @@ export const createDirSyncContext = ( clearOptimistic(input.directory, input.sessionID, messageID) } const [store] = serverSync.child(input.directory, { bootstrap: false }) - const cached = input.mode === "prepend" ? (store.message[input.sessionID] ?? []) : optimistic.canonical - const message = merge(cached, next.session) + const previous = store.message[input.sessionID] ?? [] + const cached = input.mode === "prepend" ? previous : optimistic.canonical + const merged = mergePageMessages(cached, next.session, previous) + if (merged.conflict) requestSessionSync?.(input.sessionID) + const message = merged.messages batch(() => { input.setStore("message", input.sessionID, reconcile(message, { key: "id" })) for (const p of next.part) { @@ -665,6 +688,12 @@ export const createDirSyncContext = ( return current()[0].path.directory }, } + + requestSessionSync = (sessionID) => { + void result.session.sync(sessionID, { force: true }).catch((error) => { + console.error("Failed to reload session after an activity progress conflict", error) + }) + } serverSync.registerSessionReloader?.(directory, (sessionID) => result.session.sync(sessionID, { force: true }), ) diff --git a/packages/app/src/context/global-sync/event-reducer.ts b/packages/app/src/context/global-sync/event-reducer.ts index 5ac775608..bb97d4796 100644 --- a/packages/app/src/context/global-sync/event-reducer.ts +++ b/packages/app/src/context/global-sync/event-reducer.ts @@ -36,7 +36,7 @@ function activityProgress(value: unknown): ActivityProgress | undefined { return value as ActivityProgress } -function mergeMessage(current: Message, incoming: Message) { +export function mergeMessage(current: Message, incoming: Message) { if (current.role !== "assistant" || incoming.role !== "assistant") return { message: incoming, conflict: false } const existing = activityProgress(current.activityProgress) const next = activityProgress(incoming.activityProgress) diff --git a/packages/sdk/openapi.json b/packages/sdk/openapi.json index 4b446b5f5..438bf6483 100644 --- a/packages/sdk/openapi.json +++ b/packages/sdk/openapi.json @@ -15739,6 +15739,28 @@ "required": ["name", "data"], "additionalProperties": false }, + "ActivityProgress": { + "type": "object", + "properties": { + "activityID": { + "type": "string", + "minLength": 1 + }, + "revision": { + "type": "integer", + "minimum": 0 + }, + "state": { + "type": "string", + "enum": ["provisional", "progress", "final", "interrupted", "recovery_required", "failed"] + }, + "terminalReason": { + "type": "string" + } + }, + "required": ["activityID", "revision", "state"], + "additionalProperties": false + }, "AssistantMessage": { "type": "object", "properties": { @@ -15867,6 +15889,9 @@ }, "finish": { "type": "string" + }, + "activityProgress": { + "$ref": "#/components/schemas/ActivityProgress" } }, "required": [ From d42a38cf0654e5f191fc215a5ea3b2b9478d0b41 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 12:27:34 +0800 Subject: [PATCH 03/44] fix(app): bound activity projection refetch --- .../app/src/context/directory-sync.test.ts | 106 ++++++++++++++++++ packages/app/src/context/directory-sync.ts | 12 +- 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/packages/app/src/context/directory-sync.test.ts b/packages/app/src/context/directory-sync.test.ts index 62fe24c87..2fe12c801 100644 --- a/packages/app/src/context/directory-sync.test.ts +++ b/packages/app/src/context/directory-sync.test.ts @@ -296,6 +296,112 @@ describe("directory optimistic targeting", () => { expect(result?.role).toBe("assistant") if (result?.role === "assistant") expect(result.activityProgress).toEqual(marker) }) + + test("does not recursively refetch when a forced page keeps returning a conflicting marker", async () => { + let calls = 0 + const current = state() + const serverSync = { + child() { + return current + }, + plan: { + async sync() {}, + }, + } as unknown as Parameters[1] + const sync = createDirSyncContext("/repo/main", serverSync, { + scope: ServerScope.local, + createClient() { + return { + session: { + async get() { + return { data: { id: "ses_1" } } + }, + async messages() { + calls += 1 + return { + data: [ + { + info: assistantMessage("msg_assistant", "ses_1", { + activityID: "activity-conflict", + revision: 0, + state: "progress", + }), + parts: [], + }, + ], + response: { headers: new Headers() }, + } + }, + }, + } + }, + } as unknown as Parameters[2]) + const sessionID = "ses_1" + const marker = { activityID: "activity-1", revision: 2, state: "final" as const } + current[1]("session", [current[0].session.length], { id: sessionID }) + current[1]("message", sessionID, [assistantMessage("msg_assistant", sessionID, marker)]) + + await sync.session.sync(sessionID, { force: true }) + await Promise.resolve() + + expect(calls).toBe(1) + const result = current[0].message[sessionID]?.[0] + expect(result?.role).toBe("assistant") + if (result?.role === "assistant") expect(result.activityProgress).toEqual(marker) + }) + + test("runs a forced message fetch after a history page conflicts", async () => { + let calls = 0 + let forcedStarted: (() => void) | undefined + const current = state() + const canonical = { activityID: "activity-1", revision: 2, state: "final" as const } + const serverSync = { + child() { + return current + }, + plan: { + async sync() {}, + }, + } as unknown as Parameters[1] + const sync = createDirSyncContext("/repo/main", serverSync, { + scope: ServerScope.local, + createClient() { + return { + session: { + async get() { + return { data: { id: "ses_1" } } + }, + async messages() { + calls += 1 + if (calls === 3) forcedStarted?.() + const marker = + calls === 2 + ? { activityID: "activity-conflict", revision: 0, state: "progress" as const } + : canonical + return { + data: [{ info: assistantMessage("msg_assistant", "ses_1", marker), parts: [] }], + response: { headers: new Headers(calls === 1 ? { "x-next-cursor": "older" } : {}) }, + } + }, + }, + } + }, + } as unknown as Parameters[2]) + const sessionID = "ses_1" + current[1]("session", [current[0].session.length], { id: sessionID }) + await sync.session.sync(sessionID, { force: true }) + + const forced = new Promise((resolve) => { + forcedStarted = resolve + }) + await sync.session.history.loadMore(sessionID) + await forced + + expect(calls).toBe(3) + const result = current[0].message[sessionID]?.[0] + expect(result?.role).toBe("assistant") + if (result?.role === "assistant") expect(result.activityProgress).toEqual(canonical) + }) }) describe("directory forced session synchronization", () => { diff --git a/packages/app/src/context/directory-sync.ts b/packages/app/src/context/directory-sync.ts index 81fb9bce3..1970dfb0c 100644 --- a/packages/app/src/context/directory-sync.ts +++ b/packages/app/src/context/directory-sync.ts @@ -387,10 +387,12 @@ export const createDirSyncContext = ( limit: number before?: string mode?: "replace" | "prepend" + refetchOnConflict?: boolean }) => { const key = keyFor(input.directory, input.sessionID) if (meta.loading[key]) return + let conflict = false setMeta("loading", key, true) await fetchMessages(input) .then((page) => { @@ -404,7 +406,13 @@ export const createDirSyncContext = ( const previous = store.message[input.sessionID] ?? [] const cached = input.mode === "prepend" ? previous : optimistic.canonical const merged = mergePageMessages(cached, next.session, previous) - if (merged.conflict) requestSessionSync?.(input.sessionID) + conflict = merged.conflict + if (conflict) + console.error("Conflicting activity progress page", { + sessionID: input.sessionID, + mode: input.mode ?? "replace", + refetchOnConflict: input.refetchOnConflict !== false, + }) const message = merged.messages batch(() => { input.setStore("message", input.sessionID, reconcile(message, { key: "id" })) @@ -439,6 +447,7 @@ export const createDirSyncContext = ( draft.loading[key] = false }), ) + if (conflict && input.refetchOnConflict !== false) requestSessionSync?.(input.sessionID) }) } @@ -594,6 +603,7 @@ export const createDirSyncContext = ( setStore, sessionID, limit, + refetchOnConflict: opts?.force !== true, }) await Promise.all([sessionReq, messagesReq, planReq]) From f4b00c3997bdd62e2cb6e2aa5dd52e72a93a0f22 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 12:40:13 +0800 Subject: [PATCH 04/44] fix(app): reconcile authoritative activity refresh --- .../app/src/context/directory-sync.test.ts | 67 ++++++++- packages/app/src/context/directory-sync.ts | 135 ++++++++++-------- 2 files changed, 142 insertions(+), 60 deletions(-) diff --git a/packages/app/src/context/directory-sync.test.ts b/packages/app/src/context/directory-sync.test.ts index 2fe12c801..c00011030 100644 --- a/packages/app/src/context/directory-sync.test.ts +++ b/packages/app/src/context/directory-sync.test.ts @@ -297,7 +297,7 @@ describe("directory optimistic targeting", () => { if (result?.role === "assistant") expect(result.activityProgress).toEqual(marker) }) - test("does not recursively refetch when a forced page keeps returning a conflicting marker", async () => { + test("uses a forced page as authority without recursively refetching", async () => { let calls = 0 const current = state() const serverSync = { @@ -337,9 +337,10 @@ describe("directory optimistic targeting", () => { }, } as unknown as Parameters[2]) const sessionID = "ses_1" - const marker = { activityID: "activity-1", revision: 2, state: "final" as const } + const existing = { activityID: "activity-existing", revision: 2, state: "final" as const } + const canonical = { activityID: "activity-conflict", revision: 0, state: "progress" as const } current[1]("session", [current[0].session.length], { id: sessionID }) - current[1]("message", sessionID, [assistantMessage("msg_assistant", sessionID, marker)]) + current[1]("message", sessionID, [assistantMessage("msg_assistant", sessionID, existing)]) await sync.session.sync(sessionID, { force: true }) await Promise.resolve() @@ -347,7 +348,7 @@ describe("directory optimistic targeting", () => { expect(calls).toBe(1) const result = current[0].message[sessionID]?.[0] expect(result?.role).toBe("assistant") - if (result?.role === "assistant") expect(result.activityProgress).toEqual(marker) + if (result?.role === "assistant") expect(result.activityProgress).toEqual(canonical) }) test("runs a forced message fetch after a history page conflicts", async () => { @@ -402,6 +403,64 @@ describe("directory optimistic targeting", () => { expect(result?.role).toBe("assistant") if (result?.role === "assistant") expect(result.activityProgress).toEqual(canonical) }) + + test("does not let an older forced response overwrite a marker received during the request", async () => { + let release: (() => void) | undefined + const current = state() + const serverSync = { + child() { + return current + }, + plan: { + async sync() {}, + }, + } as unknown as Parameters[1] + const sync = createDirSyncContext("/repo/main", serverSync, { + scope: ServerScope.local, + createClient() { + return { + session: { + async get() { + return { data: { id: "ses_1" } } + }, + async messages() { + await new Promise((resolve) => { + release = resolve + }) + return { + data: [ + { + info: assistantMessage("msg_assistant", "ses_1", { + activityID: "activity-http", + revision: 1, + state: "progress", + }), + parts: [], + }, + ], + response: { headers: new Headers() }, + } + }, + }, + } + }, + } as unknown as Parameters[2]) + const sessionID = "ses_1" + const initial = { activityID: "activity-initial", revision: 0, state: "progress" as const } + const event = { activityID: "activity-event", revision: 2, state: "final" as const } + current[1]("session", [current[0].session.length], { id: sessionID }) + current[1]("message", sessionID, [assistantMessage("msg_assistant", sessionID, initial)]) + + const forced = sync.session.sync(sessionID, { force: true }) + await Promise.resolve() + current[1]("message", sessionID, 0, assistantMessage("msg_assistant", sessionID, event)) + release?.() + await forced + + const result = current[0].message[sessionID]?.[0] + expect(result?.role).toBe("assistant") + if (result?.role === "assistant") expect(result.activityProgress).toEqual(event) + }) }) describe("directory forced session synchronization", () => { diff --git a/packages/app/src/context/directory-sync.ts b/packages/app/src/context/directory-sync.ts index 1970dfb0c..255656e70 100644 --- a/packages/app/src/context/directory-sync.ts +++ b/packages/app/src/context/directory-sync.ts @@ -58,6 +58,8 @@ function mergePageMessages( base: readonly Message[], incoming: readonly Message[], previous: readonly Message[], + baseline: ReadonlyMap, + authoritative: boolean, ) { const messages = new Map(base.map((item) => [item.id, item] as const)) const previousByID = new Map(previous.map((item) => [item.id, item] as const)) @@ -68,6 +70,10 @@ function mergePageMessages( messages.set(item.id, item) continue } + if (authoritative && activityProgressKey(current) === baseline.get(item.id)) { + messages.set(item.id, item) + continue + } const result = mergeMessage(current, item) messages.set(item.id, result.message) conflict ||= result.conflict @@ -78,6 +84,11 @@ function mergePageMessages( } } +function activityProgressKey(message: Message) { + if (message.role !== "assistant" || !message.activityProgress) return "" + return JSON.stringify(message.activityProgress) +} + type OptimisticStore = { message: Record part: Record @@ -242,6 +253,8 @@ export const createDirSyncContext = ( const historyMessagePageSize = 200 const inflight = new Map>() const trailing = new Set() + const inflightMessages = new Map>() + const trailingMessages = new Set() const inflightDiff = new Map>() const trailingDiff = new Set() const optimistic = new Map>() @@ -387,68 +400,76 @@ export const createDirSyncContext = ( limit: number before?: string mode?: "replace" | "prepend" + force?: boolean + authoritative?: boolean refetchOnConflict?: boolean }) => { const key = keyFor(input.directory, input.sessionID) - if (meta.loading[key]) return - - let conflict = false - setMeta("loading", key, true) - await fetchMessages(input) - .then((page) => { - if (!tracked(input.directory, input.sessionID)) return - const optimistic = getOptimistic(input.directory, input.sessionID) - const next = mergeOptimisticPage(page, optimistic.pending) - for (const messageID of next.confirmed) { - clearOptimistic(input.directory, input.sessionID, messageID) - } - const [store] = serverSync.child(input.directory, { bootstrap: false }) - const previous = store.message[input.sessionID] ?? [] - const cached = input.mode === "prepend" ? previous : optimistic.canonical - const merged = mergePageMessages(cached, next.session, previous) - conflict = merged.conflict - if (conflict) - console.error("Conflicting activity progress page", { - sessionID: input.sessionID, - mode: input.mode ?? "replace", - refetchOnConflict: input.refetchOnConflict !== false, - }) - const message = merged.messages - batch(() => { - input.setStore("message", input.sessionID, reconcile(message, { key: "id" })) - for (const p of next.part) { - const filtered = p.part.filter((x) => !SKIP_PARTS.has(x.type)) - if (filtered.length) input.setStore("part", p.id, filtered) + return runInflight(inflightMessages, trailingMessages, key, input.force === true, async () => { + const baseline = new Map( + (serverSync.child(input.directory, { bootstrap: false })[0].message[input.sessionID] ?? []).map((message) => [ + message.id, + activityProgressKey(message), + ]), + ) + let conflict = false + setMeta("loading", key, true) + await fetchMessages(input) + .then((page) => { + if (!tracked(input.directory, input.sessionID)) return + const optimistic = getOptimistic(input.directory, input.sessionID) + const next = mergeOptimisticPage(page, optimistic.pending) + for (const messageID of next.confirmed) { + clearOptimistic(input.directory, input.sessionID, messageID) } - setMeta("limit", key, message.length) - setMeta("cursor", key, next.cursor) - setMeta("complete", key, next.complete) - setSessionPrefetch({ - scope: serverSDK.scope, - directory: input.directory, - sessionID: input.sessionID, - limit: message.length, - cursor: next.cursor, - complete: next.complete, + const [store] = serverSync.child(input.directory, { bootstrap: false }) + const previous = store.message[input.sessionID] ?? [] + const cached = input.mode === "prepend" ? previous : optimistic.canonical + const merged = mergePageMessages(cached, next.session, previous, baseline, input.authoritative === true) + conflict = merged.conflict + if (conflict) + console.error("Conflicting activity progress page", { + sessionID: input.sessionID, + mode: input.mode ?? "replace", + refetchOnConflict: input.refetchOnConflict !== false, + }) + const message = merged.messages + batch(() => { + input.setStore("message", input.sessionID, reconcile(message, { key: "id" })) + for (const p of next.part) { + const filtered = p.part.filter((x) => !SKIP_PARTS.has(x.type)) + if (filtered.length) input.setStore("part", p.id, filtered) + } + setMeta("limit", key, message.length) + setMeta("cursor", key, next.cursor) + setMeta("complete", key, next.complete) + setSessionPrefetch({ + scope: serverSDK.scope, + directory: input.directory, + sessionID: input.sessionID, + limit: message.length, + cursor: next.cursor, + complete: next.complete, + }) }) }) - }) - .catch((error) => { - if (isNotFound(error) && !tracked(input.directory, input.sessionID)) return - throw error - }) - .finally(() => { - setMeta( - produce((draft) => { - if (!tracked(input.directory, input.sessionID)) { - delete draft.loading[key] - return - } - draft.loading[key] = false - }), - ) - if (conflict && input.refetchOnConflict !== false) requestSessionSync?.(input.sessionID) - }) + .catch((error) => { + if (isNotFound(error) && !tracked(input.directory, input.sessionID)) return + throw error + }) + .finally(() => { + setMeta( + produce((draft) => { + if (!tracked(input.directory, input.sessionID)) { + delete draft.loading[key] + return + } + draft.loading[key] = false + }), + ) + if (conflict && input.refetchOnConflict !== false) requestSessionSync?.(input.sessionID) + }) + }) } const loadPlan = (sessionID: string) => serverSync.plan.sync(directory, sessionID) @@ -603,6 +624,8 @@ export const createDirSyncContext = ( setStore, sessionID, limit, + force: opts?.force === true, + authoritative: opts?.force === true, refetchOnConflict: opts?.force !== true, }) From 146bfaa2eed8ad13ab8d39725b27749d3d4a7697 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 12:58:40 +0800 Subject: [PATCH 05/44] fix(app): preserve message deltas during refresh --- .../app/src/context/directory-sync.test.ts | 134 +++++++++++++++++- packages/app/src/context/directory-sync.ts | 29 +++- 2 files changed, 156 insertions(+), 7 deletions(-) diff --git a/packages/app/src/context/directory-sync.test.ts b/packages/app/src/context/directory-sync.test.ts index c00011030..0791d8ab3 100644 --- a/packages/app/src/context/directory-sync.test.ts +++ b/packages/app/src/context/directory-sync.test.ts @@ -355,6 +355,7 @@ describe("directory optimistic targeting", () => { let calls = 0 let forcedStarted: (() => void) | undefined const current = state() + const initial = { activityID: "activity-1", revision: 1, state: "progress" as const } const canonical = { activityID: "activity-1", revision: 2, state: "final" as const } const serverSync = { child() { @@ -378,7 +379,9 @@ describe("directory optimistic targeting", () => { const marker = calls === 2 ? { activityID: "activity-conflict", revision: 0, state: "progress" as const } - : canonical + : calls === 3 + ? canonical + : initial return { data: [{ info: assistantMessage("msg_assistant", "ses_1", marker), parts: [] }], response: { headers: new Headers(calls === 1 ? { "x-next-cursor": "older" } : {}) }, @@ -399,6 +402,11 @@ describe("directory optimistic targeting", () => { await forced expect(calls).toBe(3) + for (let attempt = 0; attempt < 10; attempt += 1) { + const message = current[0].message[sessionID]?.[0] + if (message?.role === "assistant" && message.activityProgress?.revision === canonical.revision) break + await Promise.resolve() + } const result = current[0].message[sessionID]?.[0] expect(result?.role).toBe("assistant") if (result?.role === "assistant") expect(result.activityProgress).toEqual(canonical) @@ -461,6 +469,130 @@ describe("directory optimistic targeting", () => { expect(result?.role).toBe("assistant") if (result?.role === "assistant") expect(result.activityProgress).toEqual(event) }) + + test("does not let an older forced page remove a new revision received during the request", async () => { + let release: (() => void) | undefined + const current = state() + const serverSync = { + child() { + return current + }, + plan: { + async sync() {}, + }, + } as unknown as Parameters[1] + const sync = createDirSyncContext("/repo/main", serverSync, { + scope: ServerScope.local, + createClient() { + return { + session: { + async get() { + return { data: { id: "ses_1" } } + }, + async messages() { + await new Promise((resolve) => { + release = resolve + }) + return { + data: [ + { + info: assistantMessage("msg_revision_1", "ses_1", { + activityID: "activity-1", + revision: 1, + state: "progress", + }), + parts: [], + }, + ], + response: { headers: new Headers() }, + } + }, + }, + } + }, + } as unknown as Parameters[2]) + const sessionID = "ses_1" + current[1]("session", [current[0].session.length], { id: sessionID }) + current[1]("message", sessionID, [ + assistantMessage("msg_revision_1", sessionID, { + activityID: "activity-1", + revision: 1, + state: "progress", + }), + ]) + + const forced = sync.session.sync(sessionID, { force: true }) + await Promise.resolve() + current[1]("message", sessionID, 1, assistantMessage("msg_revision_2", sessionID, { + activityID: "activity-1", + revision: 2, + state: "final", + })) + release?.() + await forced + + expect(current[0].message[sessionID]?.map((message) => message.id)).toEqual(["msg_revision_1", "msg_revision_2"]) + }) + + test("does not let an older forced page restore a message removed during the request", async () => { + let release: (() => void) | undefined + const current = state() + const serverSync = { + child() { + return current + }, + plan: { + async sync() {}, + }, + } as unknown as Parameters[1] + const sync = createDirSyncContext("/repo/main", serverSync, { + scope: ServerScope.local, + createClient() { + return { + session: { + async get() { + return { data: { id: "ses_1" } } + }, + async messages() { + await new Promise((resolve) => { + release = resolve + }) + return { + data: [ + { + info: assistantMessage("msg_removed", "ses_1", { + activityID: "activity-1", + revision: 1, + state: "progress", + }), + parts: [], + }, + ], + response: { headers: new Headers() }, + } + }, + }, + } + }, + } as unknown as Parameters[2]) + const sessionID = "ses_1" + current[1]("session", [current[0].session.length], { id: sessionID }) + current[1]("message", sessionID, [ + assistantMessage("msg_removed", sessionID, { + activityID: "activity-1", + revision: 1, + state: "progress", + }), + ]) + + const forced = sync.session.sync(sessionID, { force: true }) + await Promise.resolve() + current[1]("message", sessionID, []) + release?.() + await forced + + expect(current[0].message[sessionID]).toEqual([]) + }) }) describe("directory forced session synchronization", () => { diff --git a/packages/app/src/context/directory-sync.ts b/packages/app/src/context/directory-sync.ts index 255656e70..202c635b2 100644 --- a/packages/app/src/context/directory-sync.ts +++ b/packages/app/src/context/directory-sync.ts @@ -17,6 +17,7 @@ import { promptAdmissionClientMessageID } from "./global-sync/prompt-admission" import { mergeMessage } from "./global-sync/event-reducer" const SKIP_PARTS = new Set(["patch", "step-start", "step-finish"]) +type ActivityProgress = NonNullable["activityProgress"]> function sortParts(parts: Part[]) { return parts.filter((part) => !!part?.id).sort((a, b) => cmp(a.id, b.id)) @@ -58,19 +59,25 @@ function mergePageMessages( base: readonly Message[], incoming: readonly Message[], previous: readonly Message[], - baseline: ReadonlyMap, + baseline: ReadonlyMap, authoritative: boolean, ) { const messages = new Map(base.map((item) => [item.id, item] as const)) const previousByID = new Map(previous.map((item) => [item.id, item] as const)) + if (authoritative) { + for (const item of previous) { + if (!baseline.has(item.id) || !sameActivityProgress(item, baseline.get(item.id))) messages.set(item.id, item) + } + } let conflict = false for (const item of incoming) { + if (authoritative && baseline.has(item.id) && !previousByID.has(item.id)) continue const current = previousByID.get(item.id) ?? messages.get(item.id) if (!current) { messages.set(item.id, item) continue } - if (authoritative && activityProgressKey(current) === baseline.get(item.id)) { + if (authoritative && sameActivityProgress(current, baseline.get(item.id))) { messages.set(item.id, item) continue } @@ -84,9 +91,19 @@ function mergePageMessages( } } -function activityProgressKey(message: Message) { - if (message.role !== "assistant" || !message.activityProgress) return "" - return JSON.stringify(message.activityProgress) +function activityProgress(message: Message): ActivityProgress | undefined { + if (message.role !== "assistant" || !message.activityProgress) return + return { ...message.activityProgress } +} + +function sameActivityProgress(message: Message, baseline: ActivityProgress | undefined) { + const marker = activityProgress(message) + return ( + marker?.activityID === baseline?.activityID && + marker?.revision === baseline?.revision && + marker?.state === baseline?.state && + marker?.terminalReason === baseline?.terminalReason + ) } type OptimisticStore = { @@ -409,7 +426,7 @@ export const createDirSyncContext = ( const baseline = new Map( (serverSync.child(input.directory, { bootstrap: false })[0].message[input.sessionID] ?? []).map((message) => [ message.id, - activityProgressKey(message), + activityProgress(message), ]), ) let conflict = false From 013c1c3b74db75905cf591600bd8816374e63314 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 20:23:25 +0800 Subject: [PATCH 06/44] fix(session): make legacy activity lifecycle durable --- packages/core/src/database/migration.gen.ts | 1 + ...130000_legacy_activity_lifecycle_expand.ts | 479 +++++ packages/core/src/session/sql.ts | 7 + ...egacy-activity-lifecycle-migration.test.ts | 388 ++++ packages/deepagent-code/src/effect/runner.ts | 17 + .../src/session/activity-crash-test.ts | 6 +- .../src/session/activity-sql.ts | 84 +- .../deepagent-code/src/session/compaction.ts | 15 +- .../deepagent-code/src/session/processor.ts | 74 +- .../src/session/prompt-intent.ts | 1812 ++++++++++++++--- packages/deepagent-code/src/session/prompt.ts | 453 ++++- .../deepagent-code/src/session/run-state.ts | 37 +- packages/deepagent-code/src/session/steer.ts | 69 + .../deepagent-code/test/effect/runner.test.ts | 23 + .../test/session/compaction.test.ts | 5 +- .../test/session/processor-effect.test.ts | 165 +- .../test/session/prompt-intent.test.ts | 392 +++- .../test/session/prompt.test.ts | 430 +++- .../deepagent-code/test/session/steer.test.ts | 69 +- 19 files changed, 4166 insertions(+), 360 deletions(-) create mode 100644 packages/core/src/database/migration/20260812130000_legacy_activity_lifecycle_expand.ts create mode 100644 packages/core/test/legacy-activity-lifecycle-migration.test.ts diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index 036b2a521..cda868844 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -80,5 +80,6 @@ export const migrations = ( import("./migration/20260810170000_part_integrity_backfill"), import("./migration/20260811090000_legacy_activity_progress"), import("./migration/20260811100000_legacy_activity_owner"), + import("./migration/20260812130000_legacy_activity_lifecycle_expand"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260812130000_legacy_activity_lifecycle_expand.ts b/packages/core/src/database/migration/20260812130000_legacy_activity_lifecycle_expand.ts new file mode 100644 index 000000000..d3d8dd1b5 --- /dev/null +++ b/packages/core/src/database/migration/20260812130000_legacy_activity_lifecycle_expand.ts @@ -0,0 +1,479 @@ +import { sql } from "drizzle-orm" +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" +import { Hash } from "../../util/hash" + +const migrationID = "20260812130000_legacy_activity_lifecycle_expand" +const classifierVersion = "legacy-terminal-state-v1" + +export default { + id: migrationID, + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + ALTER TABLE session_intent + ADD COLUMN execution_mode TEXT NOT NULL DEFAULT 'legacy' + CHECK (execution_mode IN ('legacy','run_now','deferred')) + `) + yield* tx.run(` + ALTER TABLE session_intent + ADD COLUMN execution_state TEXT NOT NULL DEFAULT 'legacy' + CHECK (execution_state IN ('legacy','pending','claimed','absorbed','canceled')) + `) + yield* tx.run(`ALTER TABLE session_intent ADD COLUMN execution_claim_id TEXT`) + yield* tx.run(`ALTER TABLE session_intent ADD COLUMN execution_claimed_at INTEGER`) + yield* tx.run(` + CREATE TRIGGER session_intent_execution_validate_insert + BEFORE INSERT ON session_intent + WHEN NOT ( + (NEW.execution_mode = 'legacy' AND NEW.execution_state = 'legacy' + AND NEW.execution_claim_id IS NULL AND NEW.execution_claimed_at IS NULL) OR + (NEW.execution_mode IN ('run_now','deferred') AND NEW.execution_state = 'pending' + AND NEW.execution_claim_id IS NULL AND NEW.execution_claimed_at IS NULL) OR + (NEW.execution_mode IN ('run_now','deferred') AND NEW.execution_state = 'claimed' + AND NEW.execution_claim_id IS NOT NULL AND NEW.execution_claimed_at IS NOT NULL) OR + (NEW.execution_mode = 'deferred' AND NEW.execution_state = 'absorbed' + AND NEW.execution_claim_id IS NOT NULL AND NEW.execution_claimed_at IS NOT NULL) OR + (NEW.execution_mode IN ('run_now','deferred') AND NEW.execution_state = 'canceled' + AND NEW.execution_claim_id IS NULL AND NEW.execution_claimed_at IS NULL) + ) + BEGIN + SELECT RAISE(ABORT, 'invalid session intent execution identity'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_intent_execution_validate_update + BEFORE UPDATE ON session_intent + WHEN NOT ( + (NEW.execution_mode = 'legacy' AND NEW.execution_state = 'legacy' + AND NEW.execution_claim_id IS NULL AND NEW.execution_claimed_at IS NULL) OR + (NEW.execution_mode IN ('run_now','deferred') AND NEW.execution_state = 'pending' + AND NEW.execution_claim_id IS NULL AND NEW.execution_claimed_at IS NULL) OR + (NEW.execution_mode IN ('run_now','deferred') AND NEW.execution_state = 'claimed' + AND NEW.execution_claim_id IS NOT NULL AND NEW.execution_claimed_at IS NOT NULL) OR + (NEW.execution_mode = 'deferred' AND NEW.execution_state = 'absorbed' + AND NEW.execution_claim_id IS NOT NULL AND NEW.execution_claimed_at IS NOT NULL) OR + (NEW.execution_mode IN ('run_now','deferred') AND NEW.execution_state = 'canceled' + AND NEW.execution_claim_id IS NULL AND NEW.execution_claimed_at IS NULL) + ) OR NOT ( + (NEW.execution_mode = OLD.execution_mode AND NEW.execution_state = OLD.execution_state + AND NEW.execution_claim_id IS OLD.execution_claim_id + AND NEW.execution_claimed_at IS OLD.execution_claimed_at) OR + (OLD.execution_state = 'pending' AND NEW.execution_mode = OLD.execution_mode + AND NEW.execution_state IN ('claimed','absorbed','canceled')) OR + (OLD.execution_mode = 'legacy' AND OLD.execution_state = 'legacy' + AND NEW.execution_mode IN ('run_now','deferred') + AND NEW.execution_state IN ('pending','claimed','absorbed','canceled')) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal session intent execution transition'); + END + `) + + yield* tx.run(` + ALTER TABLE session_activity_admission + ADD COLUMN execution_mode TEXT NOT NULL DEFAULT 'legacy' + CHECK (execution_mode IN ('legacy','run_now','deferred')) + `) + yield* tx.run(` + CREATE TRIGGER session_activity_admission_execution_mode_validate_insert + BEFORE INSERT ON session_activity_admission + BEGIN + SELECT CASE WHEN NEW.source_kind = 'legacy_intent' AND NOT EXISTS ( + SELECT 1 FROM session_intent intent + WHERE intent.intent_id = NEW.legacy_intent_id + AND intent.session_id = NEW.session_id + AND intent.execution_mode = NEW.execution_mode + ) THEN RAISE(ABORT, 'legacy activity admission execution mode mismatch') END; + SELECT CASE WHEN NEW.source_kind = 'session_input' AND NEW.execution_mode != 'legacy' + THEN RAISE(ABORT, 'v2 activity admission cannot use legacy execution modes') END; + END + `) + + yield* tx.run(`DROP TRIGGER session_legacy_activity_admission_validate_insert`) + yield* tx.run(`DROP TRIGGER session_legacy_activity_admission_immutable`) + yield* tx.run(`ALTER TABLE session_legacy_activity_admission RENAME TO session_legacy_activity_admission_old`) + yield* tx.run(` + CREATE TABLE session_legacy_activity_admission ( + activity_id TEXT NOT NULL REFERENCES session_legacy_activity(activity_id) ON DELETE CASCADE, + admission_id TEXT NOT NULL UNIQUE REFERENCES session_activity_admission(admission_id), + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + role TEXT NOT NULL CHECK (role IN ('trigger', 'steer', 'deferred_context')), + attached_at INTEGER NOT NULL, + PRIMARY KEY (activity_id, admission_id), + UNIQUE (activity_id, ordinal), + CHECK ( + (role = 'trigger' AND ordinal = 0) OR + (role IN ('steer', 'deferred_context') AND ordinal > 0) + ) + ) + `) + yield* tx.run(` + INSERT INTO session_legacy_activity_admission ( + activity_id, admission_id, ordinal, role, attached_at + ) + SELECT activity_id, admission_id, ordinal, role, attached_at + FROM session_legacy_activity_admission_old + `) + yield* tx.run(`DROP TABLE session_legacy_activity_admission_old`) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_admission_validate_insert + BEFORE INSERT ON session_legacy_activity_admission + BEGIN + SELECT CASE WHEN NEW.role = 'trigger' AND NOT EXISTS ( + SELECT 1 FROM session_legacy_activity activity + WHERE activity.activity_id = NEW.activity_id + AND activity.trigger_admission_id = NEW.admission_id + ) THEN RAISE(ABORT, 'legacy activity trigger admission mismatch') END; + SELECT CASE WHEN NEW.role = 'steer' AND NOT EXISTS ( + SELECT 1 + FROM session_legacy_activity activity + JOIN session_activity_admission admission ON admission.admission_id = NEW.admission_id + WHERE activity.activity_id = NEW.activity_id + AND activity.state = 'active' + AND admission.session_id = activity.session_id + AND admission.source_kind = 'legacy_intent' + AND admission.delivery = 'steer' + ) THEN RAISE(ABORT, 'legacy steer admission is not owned by active activity') END; + SELECT CASE WHEN NEW.role = 'deferred_context' AND NOT EXISTS ( + SELECT 1 + FROM session_legacy_activity activity + JOIN session_activity_admission admission ON admission.admission_id = NEW.admission_id + JOIN session_intent intent ON intent.intent_id = admission.legacy_intent_id + JOIN session_activity_admission trigger_admission + ON trigger_admission.admission_id = activity.trigger_admission_id + JOIN session_intent trigger_intent ON trigger_intent.intent_id = trigger_admission.legacy_intent_id + WHERE activity.activity_id = NEW.activity_id + AND activity.state = 'active' + AND admission.session_id = activity.session_id + AND admission.source_kind = 'legacy_intent' + AND admission.execution_mode = 'deferred' + AND intent.execution_mode = 'deferred' + AND intent.execution_state = 'absorbed' + AND intent.execution_claim_id IS NOT NULL + AND intent.execution_claimed_at IS NOT NULL + AND intent.execution_claim_id = trigger_intent.execution_claim_id + AND intent.mutation_epoch = trigger_intent.mutation_epoch + ) THEN RAISE(ABORT, 'legacy deferred context admission is not bound to active run') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_admission_immutable + BEFORE UPDATE ON session_legacy_activity_admission + BEGIN + SELECT RAISE(ABORT, 'session_legacy_activity_admission is immutable'); + END + `) + + yield* tx.run(` + ALTER TABLE session_activity_progress + ADD COLUMN input_membership_ordinal INTEGER NOT NULL DEFAULT 0 + CHECK (input_membership_ordinal >= 0) + `) + yield* tx.run(` + DROP TRIGGER session_activity_progress_legal_update + `) + yield* tx.run(` + CREATE TRIGGER session_activity_progress_legal_update + BEFORE UPDATE ON session_activity_progress + WHEN NEW.activity_id != OLD.activity_id + OR NEW.revision != OLD.revision + OR NEW.assistant_message_id != OLD.assistant_message_id + OR NEW.provider_receipt_id != OLD.provider_receipt_id + OR NEW.input_membership_ordinal != OLD.input_membership_ordinal + OR NEW.created_at != OLD.created_at + OR OLD.state != 'provisional' + OR NEW.state NOT IN ('progress', 'final', 'interrupted', 'recovery_required') + OR NEW.settled_at IS NULL + BEGIN + SELECT RAISE(ABORT, 'illegal session_activity_progress transition'); + END + `) + + yield* tx.run(` + CREATE TABLE session_legacy_activity_run ( + run_id TEXT PRIMARY KEY, + activity_id TEXT NOT NULL REFERENCES session_legacy_activity(activity_id) ON DELETE CASCADE, + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + mutation_epoch INTEGER NOT NULL CHECK (mutation_epoch >= 0), + generation INTEGER NOT NULL CHECK (generation >= 0), + owner_token TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ( + 'running','finalizing','completed','failed','interrupted','recovery_required' + )), + started_at INTEGER NOT NULL, + terminal_at INTEGER, + terminal_reason TEXT, + CHECK ( + (state IN ('running','finalizing') AND terminal_at IS NULL AND terminal_reason IS NULL) OR + (state IN ('completed','failed','interrupted','recovery_required') AND + terminal_at IS NOT NULL AND terminal_reason IS NOT NULL) + ) + ) + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_legacy_activity_run_generation_idx + ON session_legacy_activity_run(session_id, mutation_epoch, generation) + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_legacy_activity_live_run_idx + ON session_legacy_activity_run(activity_id) + WHERE state IN ('running','finalizing') + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_run_validate_insert + BEFORE INSERT ON session_legacy_activity_run + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 + FROM session_legacy_activity activity + JOIN session_activity_admission admission + ON admission.admission_id = activity.trigger_admission_id + JOIN session_intent intent ON intent.intent_id = admission.legacy_intent_id + WHERE activity.activity_id = NEW.activity_id + AND activity.session_id = NEW.session_id + AND activity.state = 'active' + AND activity.owner_token = NEW.owner_token + AND intent.session_id = NEW.session_id + AND intent.mutation_epoch = NEW.mutation_epoch + AND intent.execution_state = 'claimed' + AND intent.execution_claim_id = NEW.run_id + ) THEN RAISE(ABORT, 'legacy activity run identity mismatch') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_run_legal_update + BEFORE UPDATE ON session_legacy_activity_run + WHEN NEW.run_id != OLD.run_id + OR NEW.activity_id != OLD.activity_id + OR NEW.session_id != OLD.session_id + OR NEW.mutation_epoch != OLD.mutation_epoch + OR NEW.generation != OLD.generation + OR NEW.owner_token != OLD.owner_token + OR NEW.started_at != OLD.started_at + OR OLD.state NOT IN ('running','finalizing') + OR (OLD.state = 'running' AND NEW.state NOT IN ( + 'finalizing','completed','failed','interrupted','recovery_required' + )) + OR (OLD.state = 'finalizing' AND NEW.state NOT IN ( + 'running','completed','failed','interrupted','recovery_required' + )) + OR (NEW.state IN ('running','finalizing') AND + (NEW.terminal_at IS NOT NULL OR NEW.terminal_reason IS NOT NULL)) + OR (NEW.state IN ('completed','failed','interrupted','recovery_required') AND + (NEW.terminal_at IS NULL OR NEW.terminal_reason IS NULL)) + BEGIN + SELECT RAISE(ABORT, 'illegal session_legacy_activity_run transition'); + END + `) + + yield* tx.run(` + CREATE TABLE session_legacy_activity_terminal ( + activity_id TEXT PRIMARY KEY REFERENCES session_legacy_activity(activity_id) ON DELETE CASCADE, + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + mutation_epoch INTEGER NOT NULL CHECK (mutation_epoch >= 0), + state TEXT NOT NULL CHECK (state IN ('settled','failed','interrupted','recovery_required')), + reason_code TEXT NOT NULL, + source TEXT NOT NULL CHECK (source IN ( + 'provider_final','host_stop','cancel','compaction','restart_recovery', + 'same_process_recovery','migration_repair','migration_backfill' + )), + operation_id TEXT NOT NULL, + run_id TEXT REFERENCES session_legacy_activity_run(run_id), + assistant_message_id TEXT, + progress_revision INTEGER, + membership_ordinal INTEGER NOT NULL CHECK (membership_ordinal >= 0), + owner_token TEXT NOT NULL, + created_at INTEGER NOT NULL, + CHECK ( + (assistant_message_id IS NULL AND progress_revision IS NULL) OR + (assistant_message_id IS NOT NULL AND progress_revision IS NOT NULL) + ), + CHECK ( + (source IN ('provider_final','host_stop','cancel','compaction') AND run_id IS NOT NULL) OR + source IN ('restart_recovery','same_process_recovery','migration_repair','migration_backfill') + ) + ) + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_legacy_activity_terminal_operation_idx + ON session_legacy_activity_terminal(operation_id) + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_terminal_validate_insert + BEFORE INSERT ON session_legacy_activity_terminal + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 + FROM session_legacy_activity activity + JOIN session_activity_admission admission + ON admission.admission_id = activity.trigger_admission_id + JOIN session_intent intent ON intent.intent_id = admission.legacy_intent_id + WHERE activity.activity_id = NEW.activity_id + AND activity.session_id = NEW.session_id + AND activity.state = NEW.state + AND activity.owner_token = NEW.owner_token + AND intent.mutation_epoch = NEW.mutation_epoch + ) THEN RAISE(ABORT, 'legacy activity terminal identity mismatch') END; + SELECT CASE WHEN NEW.run_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_legacy_activity_run run + WHERE run.run_id = NEW.run_id + AND run.activity_id = NEW.activity_id + AND run.session_id = NEW.session_id + AND run.mutation_epoch = NEW.mutation_epoch + AND run.owner_token = NEW.owner_token + AND ( + (NEW.state = 'settled' AND run.state = 'completed') OR + (NEW.state = 'failed' AND run.state = 'failed') OR + (NEW.state = 'interrupted' AND run.state = 'interrupted') OR + (NEW.state = 'recovery_required' AND run.state = 'recovery_required') + ) + ) THEN RAISE(ABORT, 'legacy activity terminal run mismatch') END; + SELECT CASE WHEN NEW.assistant_message_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_activity_progress progress + WHERE progress.activity_id = NEW.activity_id + AND progress.revision = NEW.progress_revision + AND progress.assistant_message_id = NEW.assistant_message_id + AND progress.input_membership_ordinal = NEW.membership_ordinal + AND progress.state != 'provisional' + ) THEN RAISE(ABORT, 'legacy activity terminal progress mismatch') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_terminal_immutable_update + BEFORE UPDATE ON session_legacy_activity_terminal + BEGIN + SELECT RAISE(ABORT, 'session_legacy_activity_terminal is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_terminal_immutable_delete + BEFORE DELETE ON session_legacy_activity_terminal + WHEN EXISTS (SELECT 1 FROM session WHERE id = OLD.session_id) + BEGIN + SELECT RAISE(ABORT, 'session_legacy_activity_terminal is immutable'); + END + `) + + yield* tx.run(` + CREATE TABLE session_legacy_activity_migration_receipt ( + receipt_id TEXT PRIMARY KEY, + batch_id TEXT NOT NULL, + activity_id TEXT NOT NULL REFERENCES session_legacy_activity(activity_id) ON DELETE CASCADE, + classifier_version TEXT NOT NULL, + before_state TEXT NOT NULL CHECK (before_state IN ( + 'active','settled','failed','interrupted','recovery_required' + )), + after_state TEXT NOT NULL CHECK (after_state IN ( + 'settled','failed','interrupted','recovery_required' + )), + evidence_hash TEXT NOT NULL, + terminal_operation_id TEXT NOT NULL + REFERENCES session_legacy_activity_terminal(operation_id) ON DELETE CASCADE, + error_code TEXT, + created_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_legacy_activity_migration_batch_idx + ON session_legacy_activity_migration_receipt(batch_id, activity_id) + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_legacy_activity_migration_terminal_idx + ON session_legacy_activity_migration_receipt(terminal_operation_id) + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_migration_receipt_immutable_update + BEFORE UPDATE ON session_legacy_activity_migration_receipt + BEGIN + SELECT RAISE(ABORT, 'session_legacy_activity_migration_receipt is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_migration_receipt_immutable_delete + BEFORE DELETE ON session_legacy_activity_migration_receipt + WHEN EXISTS ( + SELECT 1 + FROM session_legacy_activity activity + JOIN session ON session.id = activity.session_id + WHERE activity.activity_id = OLD.activity_id + ) + BEGIN + SELECT RAISE(ABORT, 'session_legacy_activity_migration_receipt is immutable'); + END + `) + + const terminalRows = yield* tx.all<{ + activity_id: string + session_id: string + state: "settled" | "failed" | "interrupted" | "recovery_required" + terminal_reason: string + settled_at: number + owner_token: string + mutation_epoch: number + membership_ordinal: number + }>(` + SELECT + activity.activity_id, + activity.session_id, + activity.state, + activity.terminal_reason, + activity.settled_at, + activity.owner_token, + intent.mutation_epoch, + COALESCE(MAX(membership.ordinal), 0) AS membership_ordinal + FROM session_legacy_activity activity + JOIN session_activity_admission admission + ON admission.admission_id = activity.trigger_admission_id + JOIN session_intent intent ON intent.intent_id = admission.legacy_intent_id + LEFT JOIN session_legacy_activity_admission membership + ON membership.activity_id = activity.activity_id + WHERE activity.state != 'active' + GROUP BY activity.activity_id + `) + yield* Effect.forEach( + terminalRows, + (row) => { + const operationID = Hash.sha256(`legacy-activity-terminal-backfill:v1:${row.activity_id}`) + const receiptID = Hash.sha256(`legacy-activity-migration-receipt:v1:${row.activity_id}`) + const evidenceHash = Hash.sha256( + JSON.stringify({ + activityID: row.activity_id, + sessionID: row.session_id, + mutationEpoch: row.mutation_epoch, + state: row.state, + reason: row.terminal_reason, + settledAt: row.settled_at, + ownerToken: row.owner_token, + membershipOrdinal: row.membership_ordinal, + }), + ) + return Effect.gen(function* () { + yield* tx.run(sql` + INSERT INTO session_legacy_activity_terminal ( + activity_id, session_id, mutation_epoch, state, reason_code, source, + operation_id, run_id, assistant_message_id, progress_revision, + membership_ordinal, owner_token, created_at + ) VALUES ( + ${row.activity_id}, ${row.session_id}, ${row.mutation_epoch}, + ${row.state}, ${row.terminal_reason}, 'migration_backfill', + ${operationID}, NULL, NULL, NULL, ${row.membership_ordinal}, + ${row.owner_token}, ${row.settled_at} + ) + `) + yield* tx.run(sql` + INSERT INTO session_legacy_activity_migration_receipt ( + receipt_id, batch_id, activity_id, classifier_version, before_state, after_state, + evidence_hash, terminal_operation_id, error_code, created_at + ) VALUES ( + ${receiptID}, ${migrationID}, ${row.activity_id}, ${classifierVersion}, + ${row.state}, ${row.state}, ${evidenceHash}, ${operationID}, NULL, ${row.settled_at} + ) + `) + }) + }, + { discard: true }, + ) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/session/sql.ts b/packages/core/src/session/sql.ts index 4f409e4c0..c76ddbb54 100644 --- a/packages/core/src/session/sql.ts +++ b/packages/core/src/session/sql.ts @@ -299,6 +299,13 @@ export const SessionIntentTable = sqliteTable( correlation_id: text(), owner_token: text(), lease_expires_at: integer(), + execution_mode: text().$type<"legacy" | "run_now" | "deferred">().notNull().default("legacy"), + execution_state: text() + .$type<"legacy" | "pending" | "claimed" | "absorbed" | "canceled">() + .notNull() + .default("legacy"), + execution_claim_id: text(), + execution_claimed_at: integer(), mutation_epoch: integer().notNull().default(0), version: integer().notNull().default(0), time_created: integer().notNull(), diff --git a/packages/core/test/legacy-activity-lifecycle-migration.test.ts b/packages/core/test/legacy-activity-lifecycle-migration.test.ts new file mode 100644 index 000000000..841f48ad2 --- /dev/null +++ b/packages/core/test/legacy-activity-lifecycle-migration.test.ts @@ -0,0 +1,388 @@ +import { describe, expect, test } from "bun:test" +import { SqliteClient } from "@effect/sql-sqlite-bun" +import { DatabaseMigration } from "@deepagent-code/core/database/migration" +import { migrations } from "@deepagent-code/core/database/migration.gen" +import { EffectDrizzleSqlite } from "@deepagent-code/effect-drizzle-sqlite" +import { Effect, Exit } from "effect" + +describe("legacy activity lifecycle migration", () => { + test("backfills terminal authority and enforces atomic run and terminal transitions", async () => { + await Effect.runPromise( + Effect.gen(function* () { + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + const lifecycleIndex = migrations.findIndex( + (migration) => migration.id === "20260812130000_legacy_activity_lifecycle_expand", + ) + expect(lifecycleIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, lifecycleIndex)) + + yield* db.run(` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('project-lifecycle', '/repo', '[]', 1, 1) + `) + yield* db.run(` + INSERT INTO session ( + id, project_id, slug, directory, title, version, mutation_epoch, time_created, time_updated + ) VALUES ( + 'session-lifecycle-backfill', 'project-lifecycle', 'lifecycle-backfill', '/repo', + 'Lifecycle backfill', '1', 2, 1, 1 + ) + `) + yield* db.run(` + INSERT INTO session_intent ( + intent_id, session_id, source, state, selected_variant, selected_payload_hash, + delivery, admitted_message_id, mutation_epoch, + time_created, time_selected, time_admitted, time_updated + ) VALUES ( + 'intent-lifecycle-backfill', 'session-lifecycle-backfill', 'composer', 'admitted', + 'original', 'payload-lifecycle-backfill', 'turn', 'message-lifecycle-backfill', 2, + 1, 1, 1, 1 + ) + `) + yield* db.run(` + INSERT INTO session_activity_admission ( + admission_id, session_id, source_kind, legacy_intent_id, admitted_message_id, + delivery, payload_fingerprint_kind, payload_fingerprint, created_at + ) VALUES ( + 'admission-lifecycle-backfill', 'session-lifecycle-backfill', 'legacy_intent', + 'intent-lifecycle-backfill', 'message-lifecycle-backfill', 'turn', 'payload_hash', + 'payload-lifecycle-backfill', 1 + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity ( + activity_id, session_id, ordinal, trigger_admission_id, owner_token, + state, terminal_reason, created_at, settled_at + ) VALUES ( + 'activity-lifecycle-backfill', 'session-lifecycle-backfill', 0, + 'admission-lifecycle-backfill', 'owner-lifecycle-backfill', + 'recovery_required', 'legacy outcome unknown', 1, 2 + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity_admission ( + activity_id, admission_id, ordinal, role, attached_at + ) VALUES ( + 'activity-lifecycle-backfill', 'admission-lifecycle-backfill', 0, 'trigger', 1 + ) + `) + + yield* DatabaseMigration.applyOnly(db, migrations.slice(lifecycleIndex, lifecycleIndex + 1)) + + expect( + yield* db.all(` + SELECT name FROM pragma_table_info('session_intent') + WHERE name IN ('execution_mode', 'execution_state', 'execution_claim_id', 'execution_claimed_at') + ORDER BY name + `), + ).toEqual([ + { name: "execution_claim_id" }, + { name: "execution_claimed_at" }, + { name: "execution_mode" }, + { name: "execution_state" }, + ]) + expect( + yield* db.all(` + SELECT name FROM sqlite_master + WHERE type = 'table' AND name IN ( + 'session_legacy_activity_run', + 'session_legacy_activity_terminal', + 'session_legacy_activity_migration_receipt' + ) + ORDER BY name + `), + ).toEqual([ + { name: "session_legacy_activity_migration_receipt" }, + { name: "session_legacy_activity_run" }, + { name: "session_legacy_activity_terminal" }, + ]) + expect( + yield* db.get(` + SELECT state, reason_code, source, run_id, membership_ordinal + FROM session_legacy_activity_terminal + WHERE activity_id = 'activity-lifecycle-backfill' + `), + ).toEqual({ + state: "recovery_required", + reason_code: "legacy outcome unknown", + source: "migration_backfill", + run_id: null, + membership_ordinal: 0, + }) + expect( + yield* db.get(` + SELECT before_state, after_state, classifier_version + FROM session_legacy_activity_migration_receipt + WHERE activity_id = 'activity-lifecycle-backfill' + `), + ).toEqual({ + before_state: "recovery_required", + after_state: "recovery_required", + classifier_version: "legacy-terminal-state-v1", + }) + + yield* db.run(` + INSERT INTO session ( + id, project_id, slug, directory, title, version, mutation_epoch, time_created, time_updated + ) VALUES ( + 'session-lifecycle-runtime', 'project-lifecycle', 'lifecycle-runtime', '/repo', + 'Lifecycle runtime', '1', 3, 1, 1 + ) + `) + yield* db.run(` + INSERT INTO session_intent ( + intent_id, session_id, source, state, selected_variant, selected_payload_hash, + delivery, admitted_message_id, execution_mode, execution_state, mutation_epoch, + time_created, time_selected, time_admitted, time_updated + ) VALUES ( + 'intent-lifecycle-runtime', 'session-lifecycle-runtime', 'composer', 'admitted', + 'original', 'payload-lifecycle-runtime', 'turn', 'message-lifecycle-runtime', + 'run_now', 'pending', 3, 1, 1, 1, 1 + ) + `) + yield* db.run(` + INSERT INTO session_activity_admission ( + admission_id, session_id, source_kind, legacy_intent_id, admitted_message_id, + delivery, payload_fingerprint_kind, payload_fingerprint, execution_mode, created_at + ) VALUES ( + 'admission-lifecycle-runtime', 'session-lifecycle-runtime', 'legacy_intent', + 'intent-lifecycle-runtime', 'message-lifecycle-runtime', 'turn', 'payload_hash', + 'payload-lifecycle-runtime', 'run_now', 1 + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity ( + activity_id, session_id, ordinal, trigger_admission_id, owner_token, + state, terminal_reason, created_at, settled_at + ) VALUES ( + 'activity-lifecycle-runtime', 'session-lifecycle-runtime', 0, + 'admission-lifecycle-runtime', 'owner-lifecycle-runtime', 'active', NULL, 1, NULL + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity_admission ( + activity_id, admission_id, ordinal, role, attached_at + ) VALUES ( + 'activity-lifecycle-runtime', 'admission-lifecycle-runtime', 0, 'trigger', 1 + ) + `) + + const failedClaim = yield* db + .transaction((tx) => + Effect.gen(function* () { + yield* tx.run(` + UPDATE session_intent + SET execution_state = 'claimed', execution_claim_id = 'run-lifecycle-bad', + execution_claimed_at = 2 + WHERE intent_id = 'intent-lifecycle-runtime' + `) + yield* tx.run(` + INSERT INTO session_legacy_activity_run ( + run_id, activity_id, session_id, mutation_epoch, generation, + owner_token, state, started_at + ) VALUES ( + 'run-lifecycle-bad', 'activity-lifecycle-runtime', 'session-lifecycle-runtime', + 3, 0, 'wrong-owner', 'running', 2 + ) + `) + }), + ) + .pipe(Effect.exit) + expect(Exit.isFailure(failedClaim)).toBe(true) + expect( + yield* db.get(` + SELECT execution_state, execution_claim_id, execution_claimed_at + FROM session_intent WHERE intent_id = 'intent-lifecycle-runtime' + `), + ).toEqual({ execution_state: "pending", execution_claim_id: null, execution_claimed_at: null }) + expect(yield* db.get(`SELECT count(*) AS count FROM session_legacy_activity_run`)).toEqual({ count: 0 }) + + yield* db.transaction((tx) => + Effect.gen(function* () { + yield* tx.run(` + UPDATE session_intent + SET execution_state = 'claimed', execution_claim_id = 'run-lifecycle-runtime', + execution_claimed_at = 3 + WHERE intent_id = 'intent-lifecycle-runtime' + `) + yield* tx.run(` + INSERT INTO session_legacy_activity_run ( + run_id, activity_id, session_id, mutation_epoch, generation, + owner_token, state, started_at + ) VALUES ( + 'run-lifecycle-runtime', 'activity-lifecycle-runtime', 'session-lifecycle-runtime', + 3, 0, 'owner-lifecycle-runtime', 'running', 3 + ) + `) + }), + ) + + const failedTerminal = yield* db + .transaction((tx) => + Effect.gen(function* () { + yield* tx.run(` + UPDATE session_legacy_activity_run + SET state = 'completed', terminal_at = 4, terminal_reason = 'assistant_completed' + WHERE run_id = 'run-lifecycle-runtime' + `) + yield* tx.run(` + UPDATE session_legacy_activity + SET state = 'settled', terminal_reason = 'assistant_completed', settled_at = 4 + WHERE activity_id = 'activity-lifecycle-runtime' + `) + yield* tx.run(` + INSERT INTO session_legacy_activity_terminal ( + activity_id, session_id, mutation_epoch, state, reason_code, source, + operation_id, run_id, assistant_message_id, progress_revision, + membership_ordinal, owner_token, created_at + ) VALUES ( + 'activity-lifecycle-runtime', 'session-lifecycle-runtime', 3, 'settled', + 'assistant_completed', 'provider_final', 'terminal-lifecycle-bad', + 'run-lifecycle-runtime', NULL, NULL, 0, 'wrong-owner', 4 + ) + `) + }), + ) + .pipe(Effect.exit) + expect(Exit.isFailure(failedTerminal)).toBe(true) + expect( + yield* db.get(` + SELECT state, terminal_reason, settled_at + FROM session_legacy_activity WHERE activity_id = 'activity-lifecycle-runtime' + `), + ).toEqual({ state: "active", terminal_reason: null, settled_at: null }) + expect( + yield* db.get(` + SELECT state, terminal_reason, terminal_at + FROM session_legacy_activity_run WHERE run_id = 'run-lifecycle-runtime' + `), + ).toEqual({ state: "running", terminal_reason: null, terminal_at: null }) + + yield* db.transaction((tx) => + Effect.gen(function* () { + yield* tx.run(` + UPDATE session_legacy_activity_run + SET state = 'completed', terminal_at = 5, terminal_reason = 'assistant_completed' + WHERE run_id = 'run-lifecycle-runtime' + `) + yield* tx.run(` + UPDATE session_legacy_activity + SET state = 'settled', terminal_reason = 'assistant_completed', settled_at = 5 + WHERE activity_id = 'activity-lifecycle-runtime' + `) + yield* tx.run(` + INSERT INTO session_legacy_activity_terminal ( + activity_id, session_id, mutation_epoch, state, reason_code, source, + operation_id, run_id, assistant_message_id, progress_revision, + membership_ordinal, owner_token, created_at + ) VALUES ( + 'activity-lifecycle-runtime', 'session-lifecycle-runtime', 3, 'settled', + 'assistant_completed', 'provider_final', 'terminal-lifecycle-runtime', + 'run-lifecycle-runtime', NULL, NULL, 0, 'owner-lifecycle-runtime', 5 + ) + `) + }), + ) + expect( + yield* db.get(` + SELECT state, reason_code, source, run_id, membership_ordinal + FROM session_legacy_activity_terminal + WHERE activity_id = 'activity-lifecycle-runtime' + `), + ).toEqual({ + state: "settled", + reason_code: "assistant_completed", + source: "provider_final", + run_id: "run-lifecycle-runtime", + membership_ordinal: 0, + }) + expect( + Exit.isFailure( + yield* db + .run(` + UPDATE session_legacy_activity_terminal SET reason_code = 'changed' + WHERE activity_id = 'activity-lifecycle-runtime' + `) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run(` + DELETE FROM session_legacy_activity_terminal + WHERE activity_id = 'activity-lifecycle-runtime' + `) + .pipe(Effect.exit), + ), + ).toBe(true) + + yield* db.run(` + INSERT INTO session ( + id, project_id, slug, directory, title, version, mutation_epoch, time_created, time_updated + ) VALUES ( + 'session-lifecycle-old-writer', 'project-lifecycle', 'lifecycle-old-writer', '/repo', + 'Lifecycle old writer', '1', 0, 1, 1 + ) + `) + yield* db.run(` + INSERT INTO session_intent ( + intent_id, session_id, source, state, selected_variant, selected_payload_hash, + delivery, admitted_message_id, mutation_epoch, + time_created, time_selected, time_admitted, time_updated + ) VALUES ( + 'intent-lifecycle-old-writer', 'session-lifecycle-old-writer', 'composer', 'admitted', + 'original', 'payload-lifecycle-old-writer', 'turn', 'message-lifecycle-old-writer', 0, + 1, 1, 1, 1 + ) + `) + yield* db.run(` + INSERT INTO session_activity_admission ( + admission_id, session_id, source_kind, legacy_intent_id, admitted_message_id, + delivery, payload_fingerprint_kind, payload_fingerprint, created_at + ) VALUES ( + 'admission-lifecycle-old-writer', 'session-lifecycle-old-writer', 'legacy_intent', + 'intent-lifecycle-old-writer', 'message-lifecycle-old-writer', 'turn', 'payload_hash', + 'payload-lifecycle-old-writer', 1 + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity ( + activity_id, session_id, ordinal, trigger_admission_id, owner_token, + state, terminal_reason, created_at, settled_at + ) VALUES ( + 'activity-lifecycle-old-writer', 'session-lifecycle-old-writer', 0, + 'admission-lifecycle-old-writer', 'owner-lifecycle-old-writer', + 'active', NULL, 1, NULL + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity_admission ( + activity_id, admission_id, ordinal, role, attached_at + ) VALUES ( + 'activity-lifecycle-old-writer', 'admission-lifecycle-old-writer', 0, 'trigger', 1 + ) + `) + yield* db.run(` + UPDATE session_legacy_activity + SET state = 'interrupted', terminal_reason = 'old writer stop', settled_at = 2 + WHERE activity_id = 'activity-lifecycle-old-writer' + `) + expect( + yield* db.get(` + SELECT state, terminal_reason + FROM session_legacy_activity WHERE activity_id = 'activity-lifecycle-old-writer' + `), + ).toEqual({ state: "interrupted", terminal_reason: "old writer stop" }) + expect( + yield* db.get(` + SELECT count(*) AS count FROM session_legacy_activity_terminal + WHERE activity_id = 'activity-lifecycle-old-writer' + `), + ).toEqual({ count: 0 }) + expect(yield* db.get(`PRAGMA integrity_check`)).toEqual({ integrity_check: "ok" }) + expect(yield* db.all(`PRAGMA foreign_key_check`)).toEqual([]) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), Effect.scoped), + ) + }) +}) diff --git a/packages/deepagent-code/src/effect/runner.ts b/packages/deepagent-code/src/effect/runner.ts index 363c6eb70..d8c96b04d 100644 --- a/packages/deepagent-code/src/effect/runner.ts +++ b/packages/deepagent-code/src/effect/runner.ts @@ -4,6 +4,7 @@ export interface Runner { readonly state: State readonly busy: boolean readonly ensureRunning: (work: Effect.Effect, onRunning?: Effect.Effect) => Effect.Effect + readonly startRunning: (work: Effect.Effect) => Effect.Effect readonly startShell: (work: Effect.Effect, ready?: Latch.Latch) => Effect.Effect readonly cancel: Effect.Effect } @@ -139,6 +140,21 @@ export const make = ( }), ).pipe(Effect.flatten) + const startRunning = (work: Effect.Effect): Effect.Effect => + SynchronizedRef.modifyEffect( + ref, + Effect.fnUntraced(function* (st) { + if (st._tag !== "Idle") { + const reject: Effect.Effect = Effect.fail(new Busy()) + return [reject, st] as const + } + const done = yield* Deferred.make() + const run = yield* startRun(work, done) + const awaitRun: Effect.Effect = awaitDone(done) + return [awaitRun, { _tag: "Running", run }] as const + }), + ).pipe(Effect.flatten) + const startShell = (work: Effect.Effect, ready?: Latch.Latch): Effect.Effect => SynchronizedRef.modifyEffect( ref, @@ -211,6 +227,7 @@ export const make = ( return state()._tag !== "Idle" }, ensureRunning, + startRunning, startShell, cancel, } diff --git a/packages/deepagent-code/src/session/activity-crash-test.ts b/packages/deepagent-code/src/session/activity-crash-test.ts index 3f134cfbd..ff7279363 100644 --- a/packages/deepagent-code/src/session/activity-crash-test.ts +++ b/packages/deepagent-code/src/session/activity-crash-test.ts @@ -2,7 +2,11 @@ import { realpath, writeFile } from "node:fs/promises" import path from "node:path" import { Effect } from "effect" -export type Point = "after_provider_streaming" | "after_provider_receipt_terminal" | "after_progress_settled" +export type Point = + | "after_admit_and_bind" + | "after_provider_streaming" + | "after_provider_receipt_terminal" + | "after_progress_settled" export function pause(point: Point) { if (process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT !== point) return Effect.void diff --git a/packages/deepagent-code/src/session/activity-sql.ts b/packages/deepagent-code/src/session/activity-sql.ts index 62888049f..87f1a54e9 100644 --- a/packages/deepagent-code/src/session/activity-sql.ts +++ b/packages/deepagent-code/src/session/activity-sql.ts @@ -1,3 +1,4 @@ +import { sql } from "drizzle-orm" import { integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" export const SessionActivityAdmissionTable = sqliteTable( @@ -12,6 +13,7 @@ export const SessionActivityAdmissionTable = sqliteTable( delivery: text().$type<"turn" | "steer" | "queue" | "goal_steer">().notNull(), payload_fingerprint_kind: text().$type<"payload_hash" | "source_identity">().notNull(), payload_fingerprint: text().notNull(), + execution_mode: text().$type<"legacy" | "run_now" | "deferred">().notNull().default("legacy"), created_at: integer().notNull(), }, (table) => [ @@ -26,7 +28,7 @@ export const SessionLegacyActivityAdmissionTable = sqliteTable( activity_id: text().notNull(), admission_id: text().notNull(), ordinal: integer().notNull(), - role: text().$type<"trigger" | "steer">().notNull(), + role: text().$type<"trigger" | "steer" | "deferred_context">().notNull(), attached_at: integer().notNull(), }, (table) => [ @@ -59,6 +61,7 @@ export const SessionActivityProgressTable = sqliteTable( assistant_message_id: text().notNull(), text_part_id: text(), provider_receipt_id: text().notNull(), + input_membership_ordinal: integer().notNull().default(0), state: text().$type<"provisional" | "progress" | "final" | "interrupted" | "recovery_required">().notNull(), finish_observed: text(), response_fingerprint: text(), @@ -71,3 +74,82 @@ export const SessionActivityProgressTable = sqliteTable( uniqueIndex("session_activity_progress_receipt_idx").on(table.provider_receipt_id), ], ) + +export const SessionLegacyActivityRunTable = sqliteTable( + "session_legacy_activity_run", + { + run_id: text().primaryKey(), + activity_id: text().notNull(), + session_id: text().notNull(), + mutation_epoch: integer().notNull(), + generation: integer().notNull(), + owner_token: text().notNull(), + state: text() + .$type<"running" | "finalizing" | "completed" | "failed" | "interrupted" | "recovery_required">() + .notNull(), + started_at: integer().notNull(), + terminal_at: integer(), + terminal_reason: text(), + }, + (table) => [ + uniqueIndex("session_legacy_activity_run_generation_idx").on( + table.session_id, + table.mutation_epoch, + table.generation, + ), + uniqueIndex("session_legacy_activity_live_run_idx") + .on(table.activity_id) + .where(sql`${table.state} IN ('running','finalizing')`), + ], +) + +export const SessionLegacyActivityTerminalTable = sqliteTable( + "session_legacy_activity_terminal", + { + activity_id: text().primaryKey(), + session_id: text().notNull(), + mutation_epoch: integer().notNull(), + state: text().$type<"settled" | "failed" | "interrupted" | "recovery_required">().notNull(), + reason_code: text().notNull(), + source: text() + .$type< + | "provider_final" + | "host_stop" + | "cancel" + | "compaction" + | "restart_recovery" + | "same_process_recovery" + | "migration_repair" + | "migration_backfill" + >() + .notNull(), + operation_id: text().notNull(), + run_id: text(), + assistant_message_id: text(), + progress_revision: integer(), + membership_ordinal: integer().notNull(), + owner_token: text().notNull(), + created_at: integer().notNull(), + }, + (table) => [uniqueIndex("session_legacy_activity_terminal_operation_idx").on(table.operation_id)], +) + +export const SessionLegacyActivityMigrationReceiptTable = sqliteTable( + "session_legacy_activity_migration_receipt", + { + receipt_id: text().primaryKey(), + batch_id: text().notNull(), + activity_id: text().notNull(), + classifier_version: text().notNull(), + before_state: text().$type<"active" | "settled" | "failed" | "interrupted" | "recovery_required">().notNull(), + after_state: text().$type<"settled" | "failed" | "interrupted" | "recovery_required">().notNull(), + evidence_hash: text().notNull(), + terminal_operation_id: text().notNull(), + error_code: text(), + created_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("session_legacy_activity_migration_batch_idx").on(table.batch_id, table.activity_id), + uniqueIndex("session_legacy_activity_migration_terminal_idx").on(table.terminal_operation_id), + ], +) diff --git a/packages/deepagent-code/src/session/compaction.ts b/packages/deepagent-code/src/session/compaction.ts index 02b844df6..2f385dae4 100644 --- a/packages/deepagent-code/src/session/compaction.ts +++ b/packages/deepagent-code/src/session/compaction.ts @@ -1289,7 +1289,10 @@ export const layer = Layer.effect( let dispatchCount = 0 let previousAttemptID: string | undefined let currentProcessor = yield* processors.create({ assistantMessage: msg, sessionID: input.sessionID, model }) - let result: "continue" | "compact" | "stop" = "stop" + let result: SessionProcessor.ProcessorDecision = { + action: "stop", + reason: { code: "assistant_error", errorName: "CompactionNotStarted" }, + } while (true) { dispatchCount++ @@ -1306,7 +1309,7 @@ export const layer = Layer.effect( const dispatchResult = yield* Effect.exit(currentProcessor.processSummary(summaryStreamInput, attempt)) if (Exit.isSuccess(dispatchResult)) { result = dispatchResult.value - if (result === "stop") { + if (result.action === "stop") { yield* failRun(run.run_id, "summary_provider_error") return "stop" } @@ -1344,7 +1347,7 @@ export const layer = Layer.effect( currentProcessor = yield* processors.create({ assistantMessage: retryMsg, sessionID: input.sessionID, model }) } - if (result === "compact") { + if (result.action === "compact") { currentProcessor.message.error = new SessionV1.ContextOverflowError({ message: "Session too large to compact - context exceeds model limit even after stripping media", }).toObject() @@ -1355,7 +1358,7 @@ export const layer = Layer.effect( } const continuation = yield* Effect.gen(function* () { - if (result !== "continue" || !input.auto) return + if (result.action !== "continue" || !input.auto) return const info = yield* provider.getProvider(userMessage.model.providerID) if ( (yield* plugin.trigger( @@ -1435,7 +1438,7 @@ export const layer = Layer.effect( yield* failRun(run.run_id, "summary_provider_error") return "stop" } - if (result === "continue") { + if (result.action === "continue") { const persisted = yield* session.messages({ sessionID: input.sessionID }).pipe(Effect.orDie) const checkpointIndex = persisted.findIndex((item) => item.info.id === currentProcessor.message.id) const checkpoint = persisted[checkpointIndex] ?? { @@ -1533,7 +1536,7 @@ export const layer = Layer.effect( yield* publishCommittedRun(run.run_id) } } - return result + return result.action }) const processCompaction = Effect.fn("SessionCompaction.process")(function* ( diff --git a/packages/deepagent-code/src/session/processor.ts b/packages/deepagent-code/src/session/processor.ts index c1446db63..2b3610168 100644 --- a/packages/deepagent-code/src/session/processor.ts +++ b/packages/deepagent-code/src/session/processor.ts @@ -466,7 +466,16 @@ class DegenerationDetector { } } -export type Result = "compact" | "stop" | "continue" +export type ProcessorStopReason = + | { readonly code: "user_rejected_question"; readonly callID: string } + | { readonly code: "user_rejected_permission"; readonly callID: string } + | { readonly code: "task_no_progress_budget_exhausted"; readonly limit: number; readonly used: number } + | { readonly code: "assistant_error"; readonly errorName: string } + +export type ProcessorDecision = + | { readonly action: "compact" } + | { readonly action: "stop"; readonly reason: ProcessorStopReason } + | { readonly action: "continue" } export interface Handle { readonly message: SessionV1.Assistant @@ -492,7 +501,7 @@ export interface Handle { readonly settled: Effect.Effect readonly failed: (error: unknown) => Effect.Effect }, - ) => Effect.Effect + ) => Effect.Effect readonly processSummary: ( streamInput: LLM.StreamInput, providerAttempt: { @@ -502,7 +511,7 @@ export interface Handle { readonly settled: Effect.Effect readonly failed: (error: unknown) => Effect.Effect }, - ) => Effect.Effect + ) => Effect.Effect } type Input = { @@ -539,7 +548,7 @@ interface ProcessorContext extends Input { toolcalls: Record shouldBreak: boolean snapshot: string | undefined - blocked: boolean + stopReason: ProcessorStopReason | undefined needsCompaction: boolean currentText: SessionV1.TextPart | undefined currentTextID: string | undefined @@ -589,7 +598,7 @@ export const layer = Layer.effect( toolcalls: {}, shouldBreak: false, snapshot: initialSnapshot, - blocked: false, + stopReason: undefined, needsCompaction: false, currentText: undefined, currentTextID: undefined, @@ -873,21 +882,31 @@ export const layer = Layer.effect( ) { const match = yield* readToolCall(toolCallID) if (!match || match.part.state.status !== "running") return false + const rejection = + error instanceof Question.RejectedError + ? ({ code: "user_rejected_question", callID: toolCallID } as const) + : error instanceof PermissionV1.RejectedError + ? ({ code: "user_rejected_permission", callID: toolCallID } as const) + : undefined yield* session.updatePart({ ...match.part, state: { status: "error", input: match.part.state.input, error: errorMessage(error), - ...(match.part.state.metadata || metadata - ? { metadata: { ...match.part.state.metadata, ...metadata } } + ...(match.part.state.metadata || metadata || rejection + ? { + metadata: { + ...match.part.state.metadata, + ...metadata, + ...(rejection ? { failureCode: rejection.code } : {}), + }, + } : {}), time: { start: match.part.state.time.start, end: Date.now() }, }, }) - if (error instanceof PermissionV1.RejectedError || error instanceof Question.RejectedError) { - ctx.blocked = ctx.shouldBreak - } + if (rejection && ctx.shouldBreak) ctx.stopReason ??= rejection yield* settleToolCall(toolCallID) return true }) @@ -1985,15 +2004,42 @@ export const layer = Layer.effect( : completed.pipe(Effect.catch(halt)) ).pipe(Effect.ensuring(cleanup())) - if (ctx.needsCompaction) return "compact" - if (ctx.blocked || ctx.assistantMessage.error) return "stop" - return "continue" + if (ctx.needsCompaction) return { action: "compact" } as const + if (ctx.stopReason) return { action: "stop", reason: ctx.stopReason } as const + if ( + ctx.assistantMessage.error?.name === "TaskBudgetExceededError" && + ctx.assistantMessage.error.data.budget === "no_progress" + ) + return { + action: "stop", + reason: { + code: "task_no_progress_budget_exhausted", + limit: ctx.assistantMessage.error.data.limit, + used: ctx.assistantMessage.error.data.used, + }, + } as const + if (ctx.assistantMessage.error) + return { + action: "stop", + reason: { + code: "assistant_error", + errorName: ctx.assistantMessage.error.name, + }, + } as const + return { action: "continue" } as const }) }) const process = Effect.fn("SessionProcessor.process")((streamInput: LLM.StreamInput, providerAttempt) => processInternal(streamInput, providerAttempt).pipe( - Effect.catchTag("SummaryProtocolViolation", (error) => halt(error).pipe(Effect.as("stop" as const))), + Effect.catchTag("SummaryProtocolViolation", (error) => + halt(error).pipe( + Effect.as({ + action: "stop", + reason: { code: "assistant_error", errorName: "SummaryProtocolViolation" }, + } as const), + ), + ), ), ) diff --git a/packages/deepagent-code/src/session/prompt-intent.ts b/packages/deepagent-code/src/session/prompt-intent.ts index cab645c41..387844633 100644 --- a/packages/deepagent-code/src/session/prompt-intent.ts +++ b/packages/deepagent-code/src/session/prompt-intent.ts @@ -8,7 +8,7 @@ import { SessionTable, } from "@deepagent-code/core/session/sql" import { SessionV1 } from "@deepagent-code/core/v1/session" -import { and, eq, max, sql } from "drizzle-orm" +import { and, asc, eq, inArray, isNull, lte, max, sql } from "drizzle-orm" import { Data, Effect, Types } from "effect" import { randomUUID } from "node:crypto" import { MessageID, SessionID } from "./schema" @@ -17,7 +17,9 @@ import { SessionActivityAdmissionTable, SessionActivityProgressTable, SessionLegacyActivityAdmissionTable, + SessionLegacyActivityRunTable, SessionLegacyActivityTable, + SessionLegacyActivityTerminalTable, } from "./activity-sql" import { SessionToolRequestReceiptTable } from "./tool-request-receipt.sql" import { SessionActivityOwner } from "./activity-owner" @@ -25,6 +27,17 @@ import { SessionActivityOwner } from "./activity-owner" export type Source = "composer" | "intelligence" | "followup" | "rewrite" export type Variant = "original" | "rewritten" export type Delivery = "turn" | "steer" | "queue" | "goal_steer" +export type ExecutionMode = "run_now" | "deferred" +export type ActivityTerminalState = "settled" | "failed" | "interrupted" | "recovery_required" +export type ActivityTerminalSource = + | "provider_final" + | "host_stop" + | "cancel" + | "compaction" + | "restart_recovery" + | "same_process_recovery" + | "migration_repair" + | "migration_backfill" export class Conflict extends Data.TaggedError("SessionPromptIntent.Conflict")<{ readonly intentID: string @@ -48,6 +61,9 @@ export type Receipt = { readonly messageID?: MessageID readonly correlationID?: MessageID readonly ownerToken?: string + readonly executionMode?: "legacy" | ExecutionMode + readonly executionState?: "legacy" | "pending" | "claimed" | "absorbed" | "canceled" + readonly executionClaimID?: string readonly mutationEpoch: number readonly version: number } @@ -73,6 +89,44 @@ export type Activity = { readonly state: "active" | "settled" | "failed" | "interrupted" | "recovery_required" } +export type RunIdentity = { + readonly runID: string + readonly activityID: string + readonly sessionID: SessionID + readonly mutationEpoch: number + readonly generation: number + readonly ownerToken: string +} + +export type ProviderInputBoundary = RunIdentity & { + readonly membershipOrdinal: number +} + +export type ProviderInputBoundaryResult = + | { readonly kind: "ready"; readonly boundary: ProviderInputBoundary } + | { readonly kind: "pending_steer" } + +export type MaterializedTurn = Receipt & { + readonly state: "admitted" + readonly messageID: MessageID + readonly admissionID: string + readonly executionMode: ExecutionMode + readonly run?: RunIdentity +} + +export type ActivityTerminalDecision = { + readonly state: ActivityTerminalState + readonly reasonCode: string + readonly source: Exclude + readonly operationID: string + readonly ownerToken: string +} + +export type FinalizeResult = + | { readonly kind: "terminal_committed"; readonly invalidation: ProjectionInvalidation } + | { readonly kind: "exact_replay"; readonly invalidation: ProjectionInvalidation } + | { readonly kind: "follow_up_required"; readonly membershipOrdinal: number } + export type Progress = { readonly activityID: string readonly revision: number @@ -88,6 +142,7 @@ export type ProjectionInvalidation = { } const leaseDuration = 30_000 +type Transaction = Parameters[0]>[0] const fromRow = (row: typeof SessionIntentTable.$inferSelect): Receipt => ({ intentID: row.intent_id, @@ -100,6 +155,9 @@ const fromRow = (row: typeof SessionIntentTable.$inferSelect): Receipt => ({ ...(row.admitted_message_id ? { messageID: MessageID.make(row.admitted_message_id) } : {}), ...(row.correlation_id ? { correlationID: MessageID.make(row.correlation_id) } : {}), ...(row.owner_token ? { ownerToken: row.owner_token } : {}), + executionMode: row.execution_mode, + executionState: row.execution_state, + ...(row.execution_claim_id ? { executionClaimID: row.execution_claim_id } : {}), mutationEpoch: row.mutation_epoch, version: row.version, }) @@ -168,10 +226,12 @@ export const claim = Effect.fn("SessionPromptIntent.claim")(function* (input: { readonly variant: Variant readonly payloadHash: string readonly messageID: MessageID + readonly executionMode?: ExecutionMode }) { const { db } = yield* Database.Service const now = Date.now() const ownerToken = randomUUID() + const executionMode = input.executionMode ?? "run_now" return yield* db .transaction( (tx) => @@ -196,6 +256,8 @@ export const claim = Effect.fn("SessionPromptIntent.claim")(function* (input: { correlation_id: input.messageID, owner_token: ownerToken, lease_expires_at: now + leaseDuration, + execution_mode: executionMode, + execution_state: "pending", mutation_epoch: session.mutationEpoch, version: 1, time_created: now, @@ -225,7 +287,8 @@ export const claim = Effect.fn("SessionPromptIntent.claim")(function* (input: { existing.session_id !== input.sessionID || existing.source !== input.source || (existing.selected_variant !== null && existing.selected_variant !== input.variant) || - (existing.selected_payload_hash !== null && existing.selected_payload_hash !== input.payloadHash) + (existing.selected_payload_hash !== null && existing.selected_payload_hash !== input.payloadHash) || + (existing.execution_mode !== "legacy" && existing.execution_mode !== executionMode) ) { return yield* Effect.fail( new Conflict({ intentID: input.intentID, reason: "intent payload or selected variant conflicts" }), @@ -308,6 +371,8 @@ export const claim = Effect.fn("SessionPromptIntent.claim")(function* (input: { correlation_id: messageID, owner_token: ownerToken, lease_expires_at: now + leaseDuration, + execution_mode: executionMode, + execution_state: "pending", time_selected: existing.time_selected ?? now, time_updated: now, version: existing.version + 1, @@ -421,6 +486,8 @@ export const materializeTurn = Effect.fn("SessionPromptIntent.materializeTurn")( readonly messageID: MessageID } readonly message: { readonly info: SessionV1.User; readonly parts: ReadonlyArray } + readonly executionMode?: ExecutionMode + readonly run?: { readonly runID: string; readonly generation: number; readonly ownerToken: string } }) { const { db } = yield* Database.Service if (input.message.info.id !== input.receipt.messageID || input.message.info.sessionID !== input.receipt.sessionID) @@ -469,6 +536,11 @@ export const materializeTurn = Effect.fn("SessionPromptIntent.materializeTurn")( return yield* Effect.fail( new Conflict({ intentID: input.receipt.intentID, reason: "intent admission ownership was lost" }), ) + const executionMode = input.executionMode ?? (intent.execution_mode === "deferred" ? "deferred" : "run_now") + if (intent.execution_mode !== "legacy" && intent.execution_mode !== executionMode) + return yield* Effect.fail( + new Conflict({ intentID: input.receipt.intentID, reason: "intent execution mode conflicts" }), + ) const storedMessage = yield* tx .select() .from(MessageTable) @@ -539,6 +611,10 @@ export const materializeTurn = Effect.fn("SessionPromptIntent.materializeTurn")( admitted_message_id: input.message.info.id, owner_token: null, lease_expires_at: null, + execution_mode: executionMode, + execution_state: "pending", + execution_claim_id: null, + execution_claimed_at: null, time_admitted: now, time_updated: now, version: intent.version + 1, @@ -573,6 +649,7 @@ export const materializeTurn = Effect.fn("SessionPromptIntent.materializeTurn")( delivery: "turn", payload_fingerprint_kind: "payload_hash", payload_fingerprint: intent.selected_payload_hash, + execution_mode: executionMode, created_at: intent.time_created, }) .onConflictDoNothing() @@ -589,12 +666,21 @@ export const materializeTurn = Effect.fn("SessionPromptIntent.materializeTurn")( admission.session_id !== input.receipt.sessionID || admission.admitted_message_id !== input.message.info.id || admission.delivery !== "turn" || + admission.execution_mode !== executionMode || admission.payload_fingerprint_kind !== "payload_hash" || admission.payload_fingerprint !== intent.selected_payload_hash ) return yield* Effect.fail( new Conflict({ intentID: input.receipt.intentID, reason: "activity admission identity conflicts" }), ) + if (executionMode === "deferred") + return { + ...fromRow(admitted), + state: "admitted" as const, + messageID: input.message.info.id, + admissionID, + executionMode, + } satisfies MaterializedTurn const existingActivity = yield* tx .select() .from(SessionLegacyActivityTable) @@ -602,6 +688,21 @@ export const materializeTurn = Effect.fn("SessionPromptIntent.materializeTurn")( .get() .pipe(Effect.orDie) const activityID = existingActivity?.activity_id ?? Hash.sha256(`session-legacy-activity:v1:${admissionID}`) + const runID = input.run?.runID ?? Hash.sha256(`session-legacy-activity-run:v1:${activityID}`) + const generation = + input.run?.generation ?? + ((yield* tx + .select({ generation: max(SessionLegacyActivityRunTable.generation) }) + .from(SessionLegacyActivityRunTable) + .where( + and( + eq(SessionLegacyActivityRunTable.session_id, input.receipt.sessionID), + eq(SessionLegacyActivityRunTable.mutation_epoch, session.mutationEpoch), + ), + ) + .get() + .pipe(Effect.orDie))?.generation ?? -1) + 1 + const runOwnerToken = input.run?.ownerToken ?? SessionActivityOwner.processOwnerToken if (!existingActivity) { const active = yield* tx .select() @@ -634,7 +735,7 @@ export const materializeTurn = Effect.fn("SessionPromptIntent.materializeTurn")( session_id: input.receipt.sessionID, ordinal: (latest?.ordinal ?? -1) + 1, trigger_admission_id: admissionID, - owner_token: SessionActivityOwner.processOwnerToken, + owner_token: runOwnerToken, state: "active", terminal_reason: null, created_at: now, @@ -665,7 +766,431 @@ export const materializeTurn = Effect.fn("SessionPromptIntent.materializeTurn")( return yield* Effect.fail( new Conflict({ intentID: input.receipt.intentID, reason: "activity trigger membership conflicts" }), ) - return fromRow(admitted) + const claimed = yield* tx + .update(SessionIntentTable) + .set({ execution_state: "claimed", execution_claim_id: runID, execution_claimed_at: now }) + .where( + and( + eq(SessionIntentTable.intent_id, input.receipt.intentID), + eq(SessionIntentTable.execution_state, "pending"), + isNull(SessionIntentTable.execution_claim_id), + eq(SessionIntentTable.mutation_epoch, session.mutationEpoch), + ), + ) + .returning() + .get() + .pipe(Effect.orDie) + if (!claimed) + return yield* Effect.fail( + new Conflict({ intentID: input.receipt.intentID, reason: "activity execution claim was lost" }), + ) + yield* tx + .insert(SessionLegacyActivityRunTable) + .values({ + run_id: runID, + activity_id: activityID, + session_id: input.receipt.sessionID, + mutation_epoch: session.mutationEpoch, + generation, + owner_token: runOwnerToken, + state: "running", + started_at: now, + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + const run = yield* tx + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, runID)) + .get() + .pipe(Effect.orDie) + if ( + !run || + run.activity_id !== activityID || + run.session_id !== input.receipt.sessionID || + run.mutation_epoch !== session.mutationEpoch || + run.generation !== generation || + run.owner_token !== runOwnerToken || + run.state !== "running" + ) + return yield* Effect.fail( + new Conflict({ intentID: input.receipt.intentID, reason: "activity run identity conflicts" }), + ) + const deferred = yield* tx + .select({ + intentID: SessionIntentTable.intent_id, + admissionID: SessionActivityAdmissionTable.admission_id, + }) + .from(SessionIntentTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.legacy_intent_id, SessionIntentTable.intent_id), + ) + .where( + and( + eq(SessionIntentTable.session_id, input.receipt.sessionID), + eq(SessionIntentTable.mutation_epoch, session.mutationEpoch), + eq(SessionIntentTable.state, "admitted"), + eq(SessionIntentTable.execution_mode, "deferred"), + eq(SessionIntentTable.execution_state, "pending"), + isNull(SessionIntentTable.execution_claim_id), + lte(SessionIntentTable.time_created, intent.time_created), + ), + ) + .orderBy(asc(SessionIntentTable.time_created), asc(SessionIntentTable.intent_id)) + .all() + .pipe(Effect.orDie) + yield* Effect.forEach( + deferred, + (candidate, index) => + Effect.gen(function* () { + const absorbed = yield* tx + .update(SessionIntentTable) + .set({ execution_state: "absorbed", execution_claim_id: runID, execution_claimed_at: now }) + .where( + and( + eq(SessionIntentTable.intent_id, candidate.intentID), + eq(SessionIntentTable.execution_state, "pending"), + isNull(SessionIntentTable.execution_claim_id), + eq(SessionIntentTable.mutation_epoch, session.mutationEpoch), + ), + ) + .returning({ intentID: SessionIntentTable.intent_id }) + .get() + .pipe(Effect.orDie) + if (!absorbed) + return yield* Effect.fail( + new Conflict({ intentID: candidate.intentID, reason: "deferred execution claim was lost" }), + ) + yield* tx + .insert(SessionLegacyActivityAdmissionTable) + .values({ + activity_id: activityID, + admission_id: candidate.admissionID, + ordinal: index + 1, + role: "deferred_context", + attached_at: now, + }) + .run() + .pipe(Effect.orDie) + }), + { discard: true }, + ) + return { + ...fromRow(claimed), + state: "admitted" as const, + messageID: input.message.info.id, + admissionID, + executionMode, + run: { + runID, + activityID, + sessionID: input.receipt.sessionID, + mutationEpoch: session.mutationEpoch, + generation, + ownerToken: runOwnerToken, + }, + } satisfies MaterializedTurn + }), + { behavior: "immediate" }, + ) + .pipe(Effect.catchTag("SqlError", Effect.die)) +}) + +export const claimDeferredActivity = Effect.fn("SessionPromptIntent.claimDeferredActivity")(function* (input: { + readonly sessionID: SessionID + readonly messageID?: MessageID + readonly run?: { readonly runID: string; readonly generation: number; readonly ownerToken: string } +}) { + const { db } = yield* Database.Service + return yield* db + .transaction( + (tx) => + Effect.gen(function* () { + const session = yield* tx + .select({ mutationEpoch: SessionTable.mutation_epoch }) + .from(SessionTable) + .where(eq(SessionTable.id, input.sessionID)) + .get() + .pipe(Effect.orDie) + if (!session) return yield* Effect.die(`Session not found: ${input.sessionID}`) + const candidates = yield* tx + .select({ + intentID: SessionIntentTable.intent_id, + admissionID: SessionActivityAdmissionTable.admission_id, + messageID: SessionActivityAdmissionTable.admitted_message_id, + timeCreated: SessionIntentTable.time_created, + }) + .from(SessionIntentTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.legacy_intent_id, SessionIntentTable.intent_id), + ) + .where( + and( + eq(SessionIntentTable.session_id, input.sessionID), + eq(SessionIntentTable.mutation_epoch, session.mutationEpoch), + eq(SessionIntentTable.state, "admitted"), + eq(SessionIntentTable.execution_mode, "deferred"), + eq(SessionIntentTable.execution_state, "pending"), + isNull(SessionIntentTable.execution_claim_id), + ...(input.messageID ? [eq(SessionActivityAdmissionTable.admitted_message_id, input.messageID)] : []), + ), + ) + .orderBy(asc(SessionIntentTable.time_created), asc(SessionIntentTable.intent_id)) + .all() + .pipe(Effect.orDie) + if (candidates.length === 0) return undefined + if (candidates.length > 1) + return yield* Effect.fail( + new Conflict({ + intentID: candidates[0]!.intentID, + reason: "multiple deferred prompts require an explicit message identity", + }), + ) + const candidate = candidates[0]! + const activityID = Hash.sha256(`session-legacy-activity:v1:${candidate.admissionID}`) + const runID = input.run?.runID ?? Hash.sha256(`session-legacy-activity-run:v1:${activityID}`) + const generation = + input.run?.generation ?? + ((yield* tx + .select({ generation: max(SessionLegacyActivityRunTable.generation) }) + .from(SessionLegacyActivityRunTable) + .where( + and( + eq(SessionLegacyActivityRunTable.session_id, input.sessionID), + eq(SessionLegacyActivityRunTable.mutation_epoch, session.mutationEpoch), + ), + ) + .get() + .pipe(Effect.orDie))?.generation ?? -1) + 1 + const ownerToken = input.run?.ownerToken ?? SessionActivityOwner.processOwnerToken + const active = yield* tx + .select({ activityID: SessionLegacyActivityTable.activity_id }) + .from(SessionLegacyActivityTable) + .where( + and( + eq(SessionLegacyActivityTable.session_id, input.sessionID), + eq(SessionLegacyActivityTable.state, "active"), + ), + ) + .get() + .pipe(Effect.orDie) + if (active) + return yield* Effect.fail( + new Conflict({ + intentID: candidate.intentID, + reason: `legacy activity ${active.activityID} requires recovery before deferred execution`, + }), + ) + const now = Date.now() + yield* tx + .insert(SessionLegacyActivityTable) + .values({ + activity_id: activityID, + session_id: input.sessionID, + ordinal: + ((yield* tx + .select({ ordinal: max(SessionLegacyActivityTable.ordinal) }) + .from(SessionLegacyActivityTable) + .where(eq(SessionLegacyActivityTable.session_id, input.sessionID)) + .get() + .pipe(Effect.orDie))?.ordinal ?? -1) + 1, + trigger_admission_id: candidate.admissionID, + owner_token: ownerToken, + state: "active", + created_at: now, + }) + .run() + .pipe(Effect.orDie) + yield* tx + .insert(SessionLegacyActivityAdmissionTable) + .values({ + activity_id: activityID, + admission_id: candidate.admissionID, + ordinal: 0, + role: "trigger", + attached_at: now, + }) + .run() + .pipe(Effect.orDie) + const claimed = yield* tx + .update(SessionIntentTable) + .set({ execution_state: "claimed", execution_claim_id: runID, execution_claimed_at: now }) + .where( + and( + eq(SessionIntentTable.intent_id, candidate.intentID), + eq(SessionIntentTable.execution_state, "pending"), + isNull(SessionIntentTable.execution_claim_id), + eq(SessionIntentTable.mutation_epoch, session.mutationEpoch), + ), + ) + .returning({ intentID: SessionIntentTable.intent_id }) + .get() + .pipe(Effect.orDie) + if (!claimed) + return yield* Effect.fail( + new Conflict({ intentID: candidate.intentID, reason: "deferred execution claim was lost" }), + ) + yield* tx + .insert(SessionLegacyActivityRunTable) + .values({ + run_id: runID, + activity_id: activityID, + session_id: input.sessionID, + mutation_epoch: session.mutationEpoch, + generation, + owner_token: ownerToken, + state: "running", + started_at: now, + }) + .run() + .pipe(Effect.orDie) + return { + runID, + activityID, + sessionID: input.sessionID, + mutationEpoch: session.mutationEpoch, + generation, + ownerToken, + } satisfies RunIdentity + }), + { behavior: "immediate" }, + ) + .pipe(Effect.catchTag("SqlError", Effect.die)) +}) + +export const claimActiveActivityRun = Effect.fn("SessionPromptIntent.claimActiveActivityRun")(function* (input: { + readonly sessionID: SessionID +}) { + const { db } = yield* Database.Service + return yield* db + .transaction( + (tx) => + Effect.gen(function* () { + const session = yield* tx + .select({ mutationEpoch: SessionTable.mutation_epoch }) + .from(SessionTable) + .where(eq(SessionTable.id, input.sessionID)) + .get() + .pipe(Effect.orDie) + if (!session) return yield* Effect.die(`Session not found: ${input.sessionID}`) + const active = yield* tx + .select({ + activityID: SessionLegacyActivityTable.activity_id, + ownerToken: SessionLegacyActivityTable.owner_token, + intentID: SessionIntentTable.intent_id, + executionMode: SessionIntentTable.execution_mode, + executionState: SessionIntentTable.execution_state, + executionClaimID: SessionIntentTable.execution_claim_id, + }) + .from(SessionLegacyActivityTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.admission_id, SessionLegacyActivityTable.trigger_admission_id), + ) + .innerJoin(SessionIntentTable, eq(SessionIntentTable.intent_id, SessionActivityAdmissionTable.legacy_intent_id)) + .where( + and( + eq(SessionLegacyActivityTable.session_id, input.sessionID), + eq(SessionLegacyActivityTable.state, "active"), + eq(SessionIntentTable.mutation_epoch, session.mutationEpoch), + ), + ) + .get() + .pipe(Effect.orDie) + if (!active) return undefined + if (active.executionMode !== "run_now") return undefined + const existing = yield* tx + .select() + .from(SessionLegacyActivityRunTable) + .where( + and( + eq(SessionLegacyActivityRunTable.activity_id, active.activityID), + eq(SessionLegacyActivityRunTable.session_id, input.sessionID), + eq(SessionLegacyActivityRunTable.mutation_epoch, session.mutationEpoch), + inArray(SessionLegacyActivityRunTable.state, ["running", "finalizing"]), + ), + ) + .get() + .pipe(Effect.orDie) + if (existing) + return { + runID: existing.run_id, + activityID: existing.activity_id, + sessionID: SessionID.make(existing.session_id), + mutationEpoch: existing.mutation_epoch, + generation: existing.generation, + ownerToken: existing.owner_token, + } satisfies RunIdentity + const runID = Hash.sha256(`session-legacy-activity-run:v1:${active.activityID}`) + if (active.executionState === "claimed" && active.executionClaimID !== runID) + return yield* Effect.fail( + new Conflict({ intentID: active.intentID, reason: "active activity claim is owned by another run" }), + ) + if (active.executionState !== "claimed") { + if (active.executionState !== "pending") + return yield* Effect.fail( + new Conflict({ intentID: active.intentID, reason: "active activity execution cannot be claimed" }), + ) + const claimed = yield* tx + .update(SessionIntentTable) + .set({ + execution_state: "claimed", + execution_claim_id: runID, + execution_claimed_at: Date.now(), + }) + .where( + and( + eq(SessionIntentTable.intent_id, active.intentID), + eq(SessionIntentTable.execution_mode, "run_now"), + eq(SessionIntentTable.execution_state, "pending"), + eq(SessionIntentTable.mutation_epoch, session.mutationEpoch), + ), + ) + .returning({ intentID: SessionIntentTable.intent_id }) + .get() + .pipe(Effect.orDie) + if (!claimed) + return yield* Effect.fail( + new Conflict({ intentID: active.intentID, reason: "active activity execution claim was lost" }), + ) + } + const generation = + ((yield* tx + .select({ generation: max(SessionLegacyActivityRunTable.generation) }) + .from(SessionLegacyActivityRunTable) + .where( + and( + eq(SessionLegacyActivityRunTable.session_id, input.sessionID), + eq(SessionLegacyActivityRunTable.mutation_epoch, session.mutationEpoch), + ), + ) + .get() + .pipe(Effect.orDie))?.generation ?? -1) + 1 + yield* tx + .insert(SessionLegacyActivityRunTable) + .values({ + run_id: runID, + activity_id: active.activityID, + session_id: input.sessionID, + mutation_epoch: session.mutationEpoch, + generation, + owner_token: active.ownerToken, + state: "running", + started_at: Date.now(), + }) + .run() + .pipe(Effect.orDie) + return { + runID, + activityID: active.activityID, + sessionID: input.sessionID, + mutationEpoch: session.mutationEpoch, + generation, + ownerToken: active.ownerToken, + } satisfies RunIdentity }), { behavior: "immediate" }, ) @@ -710,11 +1235,79 @@ export const activityForMessage = Effect.fn("SessionPromptIntent.activityForMess } satisfies Activity }) -export const beginProgress = Effect.fn("SessionPromptIntent.beginProgress")(function* (input: { - readonly activityID: string - readonly assistantMessageID: MessageID - readonly providerReceiptID: string -}) { +export const freezeProviderInputBoundary = Effect.fn("SessionPromptIntent.freezeProviderInputBoundary")(function* ( + run: RunIdentity, + options?: { readonly includePendingSteers?: boolean }, +) { + const { db } = yield* Database.Service + return yield* db + .transaction( + (tx) => + Effect.gen(function* () { + const session = yield* tx + .select({ mutationEpoch: SessionTable.mutation_epoch }) + .from(SessionTable) + .where(eq(SessionTable.id, run.sessionID)) + .get() + .pipe(Effect.orDie) + const current = yield* tx + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, run.runID)) + .get() + .pipe(Effect.orDie) + if ( + !session || + session.mutationEpoch !== run.mutationEpoch || + !current || + current.activity_id !== run.activityID || + current.session_id !== run.sessionID || + current.mutation_epoch !== run.mutationEpoch || + current.generation !== run.generation || + current.owner_token !== run.ownerToken || + !["running", "finalizing"].includes(current.state) + ) + return yield* Effect.fail(new Conflict({ intentID: run.runID, reason: "activity run ownership is stale" })) + const pendingSteer = + options?.includePendingSteers === false + ? undefined + : yield* tx + .select({ id: SessionSteerTable.id }) + .from(SessionSteerTable) + .where( + and( + eq(SessionSteerTable.session_id, run.sessionID), + eq(SessionSteerTable.mutation_epoch, run.mutationEpoch), + eq(SessionSteerTable.delivery, "steer"), + isNull(SessionSteerTable.consumed_seq), + isNull(SessionSteerTable.superseded_at), + ), + ) + .get() + .pipe(Effect.orDie) + if (pendingSteer) return { kind: "pending_steer" as const } + const membership = yield* tx + .select({ ordinal: max(SessionLegacyActivityAdmissionTable.ordinal) }) + .from(SessionLegacyActivityAdmissionTable) + .where(eq(SessionLegacyActivityAdmissionTable.activity_id, run.activityID)) + .get() + .pipe(Effect.orDie) + return { + kind: "ready" as const, + boundary: { ...run, membershipOrdinal: membership?.ordinal ?? 0 }, + } satisfies ProviderInputBoundaryResult + }), + { behavior: "immediate" }, + ) + .pipe(Effect.catchTag("SqlError", Effect.die)) +}) + +export const beginProgress = Effect.fn("SessionPromptIntent.beginProgress")(function* (input: { + readonly activityID: string + readonly assistantMessageID: MessageID + readonly providerReceiptID: string + readonly membershipOrdinal?: number +}) { const { db } = yield* Database.Service return yield* db .transaction( @@ -748,6 +1341,7 @@ export const beginProgress = Effect.fn("SessionPromptIntent.beginProgress")(func assistant_message_id: input.assistantMessageID, text_part_id: null, provider_receipt_id: input.providerReceiptID, + input_membership_ordinal: input.membershipOrdinal ?? 0, state: "provisional" as const, finish_observed: null, response_fingerprint: null, @@ -762,184 +1356,787 @@ export const beginProgress = Effect.fn("SessionPromptIntent.beginProgress")(func .pipe(Effect.orDie) }) +const settleProgressInTransaction = Effect.fn("SessionPromptIntent.settleProgressInTransaction")(function* ( + tx: Transaction, + input: { + readonly activityID: string + readonly assistantMessageID: MessageID + }, + settleActivity: boolean, +) { + const current = yield* tx + .select() + .from(SessionActivityProgressTable) + .where(eq(SessionActivityProgressTable.assistant_message_id, input.assistantMessageID)) + .get() + if (!current || current.activity_id !== input.activityID) + return yield* Effect.die(new Error(`activity progress is missing: ${input.assistantMessageID}`)) + if (current.state !== "provisional") return { value: progress(current), row: current } + const receipt = yield* tx + .select() + .from(SessionToolRequestReceiptTable) + .where(eq(SessionToolRequestReceiptTable.receipt_id, current.provider_receipt_id)) + .get() + if (!receipt) return yield* Effect.die(new Error(`provider receipt is missing: ${current.provider_receipt_id}`)) + if (!["settled", "failed", "indeterminate_after_crash"].includes(receipt.provider_state)) + return yield* Effect.die( + new Error(`provider receipt is not terminal: ${current.provider_receipt_id}: ${receipt.provider_state}`), + ) + const assistant = yield* tx.select().from(MessageTable).where(eq(MessageTable.id, input.assistantMessageID)).get() + if (!assistant || assistant.session_id !== receipt.session_id || assistant.data.role !== "assistant") + return yield* Effect.die(new Error(`assistant response ownership mismatch: ${input.assistantMessageID}`)) + const assistantData = assistant.data as Omit + const latestAdmission = yield* tx + .select({ ordinal: max(SessionLegacyActivityAdmissionTable.ordinal) }) + .from(SessionLegacyActivityAdmissionTable) + .where(eq(SessionLegacyActivityAdmissionTable.activity_id, input.activityID)) + .get() + const pendingAdmission = + typeof latestAdmission?.ordinal === "number" ? current.input_membership_ordinal < latestAdmission.ordinal : false + const parts = yield* tx + .select() + .from(PartTable) + .where( + and( + eq(PartTable.message_id, input.assistantMessageID), + eq(PartTable.session_id, SessionID.make(receipt.session_id)), + ), + ) + .all() + const textParts = parts.filter((part) => part.data.type === "text") + const text = textParts.findLast((part) => { + if (part.data.type !== "text") return false + return (part.data as Omit).text.trim() !== "" + }) + const hasToolCalls = parts.some((part) => { + if (part.data.type !== "tool") return false + const data = part.data as Omit + if (data.metadata?.providerExecuted) return false + return !(data.state.status === "error" && data.state.metadata?.interrupted === true) + }) + const state = + receipt.provider_state === "indeterminate_after_crash" + ? "recovery_required" + : receipt.provider_state === "failed" + ? receipt.request_error_code === "AbortError" + ? "interrupted" + : "recovery_required" + : !assistantData.time.completed || !assistantData.finish + ? "recovery_required" + : assistantData.finish === "tool-calls" || + assistantData.finish === "length" || + hasToolCalls || + pendingAdmission + ? "progress" + : "final" + const now = Date.now() + const updated = yield* tx + .update(SessionActivityProgressTable) + .set({ + text_part_id: text?.id ?? null, + state, + finish_observed: assistantData.finish ?? receipt.request_error_code ?? null, + response_fingerprint: receipt.response_fingerprint, + settled_at: now, + }) + .where( + and( + eq(SessionActivityProgressTable.activity_id, input.activityID), + eq(SessionActivityProgressTable.revision, current.revision), + eq(SessionActivityProgressTable.state, "provisional"), + ), + ) + .returning() + .get() + if (!updated) return yield* Effect.die(new Error(`activity progress settlement CAS lost: ${input.activityID}`)) + yield* Effect.forEach( + textParts, + (part) => { + const data = part.data as Omit + return tx + .update(PartTable) + .set({ + data: { + ...data, + metadata: { + ...(data.metadata ?? {}), + deepagent_activity_progress: { + activity_id: input.activityID, + revision: current.revision, + state, + }, + }, + } as typeof PartTable.$inferInsert.data, + }) + .where( + and( + eq(PartTable.id, part.id), + eq(PartTable.message_id, input.assistantMessageID), + eq(PartTable.session_id, SessionID.make(receipt.session_id)), + ), + ) + .run() + }, + { discard: true }, + ) + if (settleActivity && state !== "progress") { + const activityState = state === "final" ? "settled" : state === "interrupted" ? "interrupted" : "recovery_required" + const terminal = yield* tx + .update(SessionLegacyActivityTable) + .set({ + state: activityState, + terminal_reason: assistantData.finish ?? receipt.request_error_code ?? state, + settled_at: now, + }) + .where( + and( + eq(SessionLegacyActivityTable.activity_id, input.activityID), + eq(SessionLegacyActivityTable.state, "active"), + ), + ) + .returning({ activityID: SessionLegacyActivityTable.activity_id }) + .get() + if (!terminal) return yield* Effect.die(new Error(`legacy activity settlement CAS lost: ${input.activityID}`)) + } + return { value: progress(updated), row: updated } +}) + export const settleProgress = Effect.fn("SessionPromptIntent.settleProgress")(function* (input: { readonly activityID: string readonly assistantMessageID: MessageID }) { + const { db } = yield* Database.Service + return yield* db + .transaction((tx) => settleProgressInTransaction(tx, input, true).pipe(Effect.map((settled) => settled.value)), { + behavior: "immediate", + }) + .pipe(Effect.orDie) +}) + +export const settleProgressOnly = Effect.fn("SessionPromptIntent.settleProgressOnly")(function* (input: { + readonly activityID: string + readonly assistantMessageID: MessageID +}) { + const { db } = yield* Database.Service + return yield* db + .transaction((tx) => settleProgressInTransaction(tx, input, false).pipe(Effect.map((settled) => settled.value)), { + behavior: "immediate", + }) + .pipe(Effect.orDie) +}) + +export const finalizeActivityWithRevision = Effect.fn("SessionPromptIntent.finalizeActivityWithRevision")( + function* (input: { + readonly run: RunIdentity + readonly assistantMessageID: MessageID + readonly decision: ActivityTerminalDecision + }) { + const { db } = yield* Database.Service + return yield* db + .transaction( + (tx) => + Effect.gen(function* () { + const existing = yield* tx + .select() + .from(SessionLegacyActivityTerminalTable) + .where(eq(SessionLegacyActivityTerminalTable.activity_id, input.run.activityID)) + .get() + if (existing) { + const replayProgress = yield* tx + .select({ + revision: SessionActivityProgressTable.revision, + membershipOrdinal: SessionActivityProgressTable.input_membership_ordinal, + state: SessionActivityProgressTable.state, + }) + .from(SessionActivityProgressTable) + .where( + and( + eq(SessionActivityProgressTable.activity_id, input.run.activityID), + eq(SessionActivityProgressTable.assistant_message_id, input.assistantMessageID), + ), + ) + .get() + const replayDecision = replayProgress + ? terminalDecisionForProgress(input.decision, replayProgress.state) + : input.decision + if ( + existing.session_id !== input.run.sessionID || + existing.mutation_epoch !== input.run.mutationEpoch || + existing.state !== replayDecision.state || + existing.reason_code !== replayDecision.reasonCode || + existing.source !== replayDecision.source || + existing.operation_id !== replayDecision.operationID || + existing.run_id !== input.run.runID || + existing.assistant_message_id !== input.assistantMessageID || + existing.progress_revision !== replayProgress?.revision || + existing.membership_ordinal !== replayProgress?.membershipOrdinal || + existing.owner_token !== input.decision.ownerToken + ) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity terminal replay diverged" }), + ) + return { + kind: "exact_replay" as const, + invalidation: { + activityID: input.run.activityID, + sessionID: input.run.sessionID, + assistantMessageID: input.assistantMessageID, + }, + } satisfies FinalizeResult + } + if (input.decision.ownerToken !== input.run.ownerToken) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity terminal owner conflicts" }), + ) + const session = yield* tx + .select({ mutationEpoch: SessionTable.mutation_epoch }) + .from(SessionTable) + .where(eq(SessionTable.id, input.run.sessionID)) + .get() + const run = yield* tx + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, input.run.runID)) + .get() + const activity = yield* tx + .select() + .from(SessionLegacyActivityTable) + .where(eq(SessionLegacyActivityTable.activity_id, input.run.activityID)) + .get() + if ( + !session || + session.mutationEpoch !== input.run.mutationEpoch || + !run || + run.activity_id !== input.run.activityID || + run.session_id !== input.run.sessionID || + run.mutation_epoch !== input.run.mutationEpoch || + run.generation !== input.run.generation || + run.owner_token !== input.run.ownerToken || + !["running", "finalizing"].includes(run.state) || + !activity || + activity.session_id !== input.run.sessionID || + activity.owner_token !== input.run.ownerToken || + activity.state !== "active" + ) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity terminal ownership is stale" }), + ) + const settled = yield* settleProgressInTransaction( + tx, + { activityID: input.run.activityID, assistantMessageID: input.assistantMessageID }, + false, + ) + const decision = terminalDecisionForProgress(input.decision, settled.row.state) + const latestMembership = + (yield* tx + .select({ ordinal: max(SessionLegacyActivityAdmissionTable.ordinal) }) + .from(SessionLegacyActivityAdmissionTable) + .where(eq(SessionLegacyActivityAdmissionTable.activity_id, input.run.activityID)) + .get())?.ordinal ?? 0 + const now = Date.now() + if (decision.source === "cancel") yield* cancelPendingActivitySteers(tx, input.run, now) + const pendingSteer = yield* tx + .select({ id: SessionSteerTable.id }) + .from(SessionSteerTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.admitted_message_id, SessionSteerTable.id), + ) + .innerJoin( + SessionLegacyActivityAdmissionTable, + eq(SessionLegacyActivityAdmissionTable.admission_id, SessionActivityAdmissionTable.admission_id), + ) + .where( + and( + eq(SessionSteerTable.session_id, input.run.sessionID), + eq(SessionSteerTable.mutation_epoch, input.run.mutationEpoch), + eq(SessionSteerTable.delivery, "steer"), + eq(SessionLegacyActivityAdmissionTable.activity_id, input.run.activityID), + isNull(SessionSteerTable.consumed_seq), + isNull(SessionSteerTable.superseded_at), + ), + ) + .get() + if (decision.source !== "cancel" && (latestMembership > settled.row.input_membership_ordinal || pendingSteer)) + return { + kind: "follow_up_required" as const, + membershipOrdinal: latestMembership, + } satisfies FinalizeResult + const runState = + decision.state === "settled" + ? "completed" + : decision.state === "failed" + ? "failed" + : decision.state === "interrupted" + ? "interrupted" + : "recovery_required" + const terminalRun = yield* tx + .update(SessionLegacyActivityRunTable) + .set({ state: runState, terminal_at: now, terminal_reason: decision.reasonCode }) + .where( + and( + eq(SessionLegacyActivityRunTable.run_id, input.run.runID), + eq(SessionLegacyActivityRunTable.owner_token, input.run.ownerToken), + inArray(SessionLegacyActivityRunTable.state, ["running", "finalizing"]), + ), + ) + .returning({ runID: SessionLegacyActivityRunTable.run_id }) + .get() + if (!terminalRun) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity run terminal CAS lost" }), + ) + const terminalActivity = yield* tx + .update(SessionLegacyActivityTable) + .set({ state: decision.state, terminal_reason: decision.reasonCode, settled_at: now }) + .where( + and( + eq(SessionLegacyActivityTable.activity_id, input.run.activityID), + eq(SessionLegacyActivityTable.owner_token, input.run.ownerToken), + eq(SessionLegacyActivityTable.state, "active"), + ), + ) + .returning({ activityID: SessionLegacyActivityTable.activity_id }) + .get() + if (!terminalActivity) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity terminal CAS lost" }), + ) + yield* tx + .insert(SessionLegacyActivityTerminalTable) + .values({ + activity_id: input.run.activityID, + session_id: input.run.sessionID, + mutation_epoch: input.run.mutationEpoch, + state: decision.state, + reason_code: decision.reasonCode, + source: decision.source, + operation_id: decision.operationID, + run_id: input.run.runID, + assistant_message_id: input.assistantMessageID, + progress_revision: settled.row.revision, + membership_ordinal: settled.row.input_membership_ordinal, + owner_token: input.run.ownerToken, + created_at: now, + }) + .run() + return { + kind: "terminal_committed" as const, + invalidation: { + activityID: input.run.activityID, + sessionID: input.run.sessionID, + assistantMessageID: input.assistantMessageID, + }, + } satisfies FinalizeResult + }), + { behavior: "immediate" }, + ) + .pipe(Effect.catchTag("EffectDrizzleQueryError", Effect.die)) + .pipe(Effect.catchTag("SqlError", Effect.die)) + }, +) + +export const finalizeActivityWithoutRevision = Effect.fn("SessionPromptIntent.finalizeActivityWithoutRevision")( + function* (input: { + readonly run: RunIdentity + readonly membershipOrdinal: number + readonly decision: ActivityTerminalDecision + }) { + const { db } = yield* Database.Service + return yield* db + .transaction( + (tx) => + Effect.gen(function* () { + const existing = yield* tx + .select() + .from(SessionLegacyActivityTerminalTable) + .where(eq(SessionLegacyActivityTerminalTable.activity_id, input.run.activityID)) + .get() + if (existing) { + if ( + existing.session_id !== input.run.sessionID || + existing.mutation_epoch !== input.run.mutationEpoch || + existing.state !== input.decision.state || + existing.reason_code !== input.decision.reasonCode || + existing.source !== input.decision.source || + existing.operation_id !== input.decision.operationID || + existing.run_id !== input.run.runID || + existing.assistant_message_id !== null || + existing.progress_revision !== null || + existing.membership_ordinal !== input.membershipOrdinal || + existing.owner_token !== input.decision.ownerToken + ) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity terminal replay diverged" }), + ) + return { + kind: "exact_replay" as const, + invalidation: { activityID: input.run.activityID, sessionID: input.run.sessionID }, + } satisfies FinalizeResult + } + if (input.decision.ownerToken !== input.run.ownerToken) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity terminal owner conflicts" }), + ) + const session = yield* tx + .select({ mutationEpoch: SessionTable.mutation_epoch }) + .from(SessionTable) + .where(eq(SessionTable.id, input.run.sessionID)) + .get() + const run = yield* tx + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, input.run.runID)) + .get() + const activity = yield* tx + .select() + .from(SessionLegacyActivityTable) + .where(eq(SessionLegacyActivityTable.activity_id, input.run.activityID)) + .get() + if ( + !session || + session.mutationEpoch !== input.run.mutationEpoch || + !run || + run.activity_id !== input.run.activityID || + run.session_id !== input.run.sessionID || + run.mutation_epoch !== input.run.mutationEpoch || + run.generation !== input.run.generation || + run.owner_token !== input.run.ownerToken || + !["running", "finalizing"].includes(run.state) || + !activity || + activity.session_id !== input.run.sessionID || + activity.owner_token !== input.run.ownerToken || + activity.state !== "active" + ) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity terminal ownership is stale" }), + ) + const membershipOrdinal = + (yield* tx + .select({ ordinal: max(SessionLegacyActivityAdmissionTable.ordinal) }) + .from(SessionLegacyActivityAdmissionTable) + .where(eq(SessionLegacyActivityAdmissionTable.activity_id, input.run.activityID)) + .get())?.ordinal ?? 0 + const now = Date.now() + if (input.decision.source === "cancel") yield* cancelPendingActivitySteers(tx, input.run, now) + const pendingSteer = yield* tx + .select({ id: SessionSteerTable.id }) + .from(SessionSteerTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.admitted_message_id, SessionSteerTable.id), + ) + .innerJoin( + SessionLegacyActivityAdmissionTable, + eq(SessionLegacyActivityAdmissionTable.admission_id, SessionActivityAdmissionTable.admission_id), + ) + .where( + and( + eq(SessionSteerTable.session_id, input.run.sessionID), + eq(SessionSteerTable.mutation_epoch, input.run.mutationEpoch), + eq(SessionSteerTable.delivery, "steer"), + eq(SessionLegacyActivityAdmissionTable.activity_id, input.run.activityID), + isNull(SessionSteerTable.consumed_seq), + isNull(SessionSteerTable.superseded_at), + ), + ) + .get() + if (input.decision.source !== "cancel" && (membershipOrdinal > input.membershipOrdinal || pendingSteer)) + return { kind: "follow_up_required" as const, membershipOrdinal } satisfies FinalizeResult + const runState = terminalRunState(input.decision.state) + const terminalRun = yield* tx + .update(SessionLegacyActivityRunTable) + .set({ state: runState, terminal_at: now, terminal_reason: input.decision.reasonCode }) + .where( + and( + eq(SessionLegacyActivityRunTable.run_id, input.run.runID), + eq(SessionLegacyActivityRunTable.owner_token, input.run.ownerToken), + inArray(SessionLegacyActivityRunTable.state, ["running", "finalizing"]), + ), + ) + .returning({ runID: SessionLegacyActivityRunTable.run_id }) + .get() + if (!terminalRun) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity run terminal CAS lost" }), + ) + const terminalActivity = yield* tx + .update(SessionLegacyActivityTable) + .set({ state: input.decision.state, terminal_reason: input.decision.reasonCode, settled_at: now }) + .where( + and( + eq(SessionLegacyActivityTable.activity_id, input.run.activityID), + eq(SessionLegacyActivityTable.owner_token, input.run.ownerToken), + eq(SessionLegacyActivityTable.state, "active"), + ), + ) + .returning({ activityID: SessionLegacyActivityTable.activity_id }) + .get() + if (!terminalActivity) + return yield* Effect.fail( + new Conflict({ intentID: input.run.runID, reason: "activity terminal CAS lost" }), + ) + yield* tx + .insert(SessionLegacyActivityTerminalTable) + .values({ + activity_id: input.run.activityID, + session_id: input.run.sessionID, + mutation_epoch: input.run.mutationEpoch, + state: input.decision.state, + reason_code: input.decision.reasonCode, + source: input.decision.source, + operation_id: input.decision.operationID, + run_id: input.run.runID, + membership_ordinal: input.membershipOrdinal, + owner_token: input.run.ownerToken, + created_at: now, + }) + .run() + return { + kind: "terminal_committed" as const, + invalidation: { activityID: input.run.activityID, sessionID: input.run.sessionID }, + } satisfies FinalizeResult + }), + { behavior: "immediate" }, + ) + .pipe(Effect.catchTag("EffectDrizzleQueryError", Effect.die)) + .pipe(Effect.catchTag("SqlError", Effect.die)) + }, +) + +export const finalizeCancellationBeforeProgress = Effect.fn( + "SessionPromptIntent.finalizeCancellationBeforeProgress", +)(function* (run: RunIdentity) { const { db } = yield* Database.Service return yield* db .transaction( (tx) => Effect.gen(function* () { - const current = yield* tx + const existing = yield* tx .select() + .from(SessionLegacyActivityTerminalTable) + .where(eq(SessionLegacyActivityTerminalTable.activity_id, run.activityID)) + .get() + if (existing) { + if ( + existing.session_id !== run.sessionID || + existing.mutation_epoch !== run.mutationEpoch || + existing.state !== "interrupted" || + existing.reason_code !== "user_cancelled" || + existing.source !== "cancel" || + existing.operation_id !== `${run.runID}:terminal` || + existing.run_id !== run.runID || + existing.owner_token !== run.ownerToken + ) + return yield* Effect.fail( + new Conflict({ intentID: run.runID, reason: "activity cancellation replay diverged" }), + ) + return { + kind: "exact_replay" as const, + invalidation: { + activityID: run.activityID, + sessionID: run.sessionID, + ...(existing.assistant_message_id + ? { assistantMessageID: MessageID.make(existing.assistant_message_id) } + : {}), + }, + } satisfies FinalizeResult + } + const progress = yield* tx + .select({ assistantMessageID: SessionActivityProgressTable.assistant_message_id }) .from(SessionActivityProgressTable) - .where(eq(SessionActivityProgressTable.assistant_message_id, input.assistantMessageID)) + .where(eq(SessionActivityProgressTable.activity_id, run.activityID)) + .limit(1) .get() - if (!current || current.activity_id !== input.activityID) - return yield* Effect.die(new Error(`activity progress is missing: ${input.assistantMessageID}`)) - if (current.state !== "provisional") return progress(current) - const receipt = yield* tx + if (progress) return undefined + const session = yield* tx + .select({ mutationEpoch: SessionTable.mutation_epoch }) + .from(SessionTable) + .where(eq(SessionTable.id, run.sessionID)) + .get() + const currentRun = yield* tx .select() - .from(SessionToolRequestReceiptTable) - .where(eq(SessionToolRequestReceiptTable.receipt_id, current.provider_receipt_id)) + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, run.runID)) .get() - if (!receipt) - return yield* Effect.die(new Error(`provider receipt is missing: ${current.provider_receipt_id}`)) - if (!["settled", "failed", "indeterminate_after_crash"].includes(receipt.provider_state)) - return yield* Effect.die( - new Error(`provider receipt is not terminal: ${current.provider_receipt_id}: ${receipt.provider_state}`), - ) - const assistant = yield* tx + const activity = yield* tx .select() - .from(MessageTable) - .where(eq(MessageTable.id, input.assistantMessageID)) + .from(SessionLegacyActivityTable) + .where(eq(SessionLegacyActivityTable.activity_id, run.activityID)) .get() - if (!assistant || assistant.session_id !== receipt.session_id || assistant.data.role !== "assistant") - return yield* Effect.die(new Error(`assistant response ownership mismatch: ${input.assistantMessageID}`)) - const assistantData = assistant.data as Omit - const currentAdmission = yield* tx - .select({ ordinal: SessionLegacyActivityAdmissionTable.ordinal }) - .from(SessionLegacyActivityAdmissionTable) - .innerJoin( - SessionActivityAdmissionTable, - eq(SessionActivityAdmissionTable.admission_id, SessionLegacyActivityAdmissionTable.admission_id), + if ( + !session || + session.mutationEpoch !== run.mutationEpoch || + !currentRun || + currentRun.activity_id !== run.activityID || + currentRun.session_id !== run.sessionID || + currentRun.mutation_epoch !== run.mutationEpoch || + currentRun.generation !== run.generation || + currentRun.owner_token !== run.ownerToken || + !["running", "finalizing"].includes(currentRun.state) || + !activity || + activity.session_id !== run.sessionID || + activity.owner_token !== run.ownerToken || + activity.state !== "active" + ) + return yield* Effect.fail( + new Conflict({ intentID: run.runID, reason: "activity cancellation ownership is stale" }), ) + const membershipOrdinal = + (yield* tx + .select({ ordinal: max(SessionLegacyActivityAdmissionTable.ordinal) }) + .from(SessionLegacyActivityAdmissionTable) + .where(eq(SessionLegacyActivityAdmissionTable.activity_id, run.activityID)) + .get())?.ordinal ?? 0 + const now = Date.now() + yield* cancelPendingActivitySteers(tx, run, now) + const terminalRun = yield* tx + .update(SessionLegacyActivityRunTable) + .set({ state: "interrupted", terminal_at: now, terminal_reason: "user_cancelled" }) .where( and( - eq(SessionLegacyActivityAdmissionTable.activity_id, input.activityID), - eq(SessionActivityAdmissionTable.admitted_message_id, assistantData.parentID), + eq(SessionLegacyActivityRunTable.run_id, run.runID), + eq(SessionLegacyActivityRunTable.owner_token, run.ownerToken), + inArray(SessionLegacyActivityRunTable.state, ["running", "finalizing"]), ), ) + .returning({ runID: SessionLegacyActivityRunTable.run_id }) .get() - const latestAdmission = yield* tx - .select({ ordinal: max(SessionLegacyActivityAdmissionTable.ordinal) }) - .from(SessionLegacyActivityAdmissionTable) - .where(eq(SessionLegacyActivityAdmissionTable.activity_id, input.activityID)) - .get() - const pendingAdmission = - currentAdmission && typeof latestAdmission?.ordinal === "number" - ? currentAdmission.ordinal < latestAdmission.ordinal - : false - const parts = yield* tx - .select() - .from(PartTable) - .where( - and( - eq(PartTable.message_id, input.assistantMessageID), - eq(PartTable.session_id, SessionID.make(receipt.session_id)), - ), + if (!terminalRun) + return yield* Effect.fail( + new Conflict({ intentID: run.runID, reason: "activity cancellation run CAS lost" }), ) - .all() - const textParts = parts.filter((part) => part.data.type === "text") - const text = textParts.findLast((part) => { - if (part.data.type !== "text") return false - return (part.data as Omit).text.trim() !== "" - }) - const hasToolCalls = parts.some((part) => { - if (part.data.type !== "tool") return false - const data = part.data as Omit - if (data.metadata?.providerExecuted) return false - return !(data.state.status === "error" && data.state.metadata?.interrupted === true) - }) - const state = - receipt.provider_state === "indeterminate_after_crash" - ? "recovery_required" - : receipt.provider_state === "failed" - ? receipt.request_error_code === "AbortError" - ? "interrupted" - : "recovery_required" - : !assistantData.time.completed || !assistantData.finish - ? "recovery_required" - : assistantData.finish === "tool-calls" || - assistantData.finish === "length" || - hasToolCalls || - pendingAdmission - ? "progress" - : "final" - const now = Date.now() - const updated = yield* tx - .update(SessionActivityProgressTable) - .set({ - text_part_id: text?.id ?? null, - state, - finish_observed: assistantData.finish ?? receipt.request_error_code ?? null, - response_fingerprint: receipt.response_fingerprint, - settled_at: now, - }) + const terminalActivity = yield* tx + .update(SessionLegacyActivityTable) + .set({ state: "interrupted", terminal_reason: "user_cancelled", settled_at: now }) .where( and( - eq(SessionActivityProgressTable.activity_id, input.activityID), - eq(SessionActivityProgressTable.revision, current.revision), - eq(SessionActivityProgressTable.state, "provisional"), + eq(SessionLegacyActivityTable.activity_id, run.activityID), + eq(SessionLegacyActivityTable.owner_token, run.ownerToken), + eq(SessionLegacyActivityTable.state, "active"), ), ) - .returning() + .returning({ activityID: SessionLegacyActivityTable.activity_id }) .get() - if (!updated) - return yield* Effect.die(new Error(`activity progress settlement CAS lost: ${input.activityID}`)) - yield* Effect.forEach( - textParts, - (part) => { - const data = part.data as Omit - return tx - .update(PartTable) - .set({ - data: { - ...data, - metadata: { - ...(data.metadata ?? {}), - deepagent_activity_progress: { - activity_id: input.activityID, - revision: current.revision, - state, - }, - }, - } as typeof PartTable.$inferInsert.data, - }) - .where( - and( - eq(PartTable.id, part.id), - eq(PartTable.message_id, input.assistantMessageID), - eq(PartTable.session_id, SessionID.make(receipt.session_id)), - ), - ) - .run() - }, - { discard: true }, - ) - if (state !== "progress") { - const activityState = - state === "final" ? "settled" : state === "interrupted" ? "interrupted" : "recovery_required" - const terminal = yield* tx - .update(SessionLegacyActivityTable) - .set({ - state: activityState, - terminal_reason: assistantData.finish ?? receipt.request_error_code ?? state, - settled_at: now, - }) - .where( - and( - eq(SessionLegacyActivityTable.activity_id, input.activityID), - eq(SessionLegacyActivityTable.state, "active"), - ), - ) - .returning({ activityID: SessionLegacyActivityTable.activity_id }) - .get() - if (!terminal) - return yield* Effect.die(new Error(`legacy activity settlement CAS lost: ${input.activityID}`)) - } - return progress(updated) + if (!terminalActivity) + return yield* Effect.fail( + new Conflict({ intentID: run.runID, reason: "activity cancellation terminal CAS lost" }), + ) + yield* tx + .insert(SessionLegacyActivityTerminalTable) + .values({ + activity_id: run.activityID, + session_id: run.sessionID, + mutation_epoch: run.mutationEpoch, + state: "interrupted", + reason_code: "user_cancelled", + source: "cancel", + operation_id: `${run.runID}:terminal`, + run_id: run.runID, + membership_ordinal: membershipOrdinal, + owner_token: run.ownerToken, + created_at: now, + }) + .run() + return { + kind: "terminal_committed" as const, + invalidation: { activityID: run.activityID, sessionID: run.sessionID }, + } satisfies FinalizeResult }), { behavior: "immediate" }, ) - .pipe(Effect.orDie) + .pipe(Effect.catchTag("EffectDrizzleQueryError", Effect.die)) + .pipe(Effect.catchTag("SqlError", Effect.die)) }) +const cancelPendingActivitySteers = Effect.fn("SessionPromptIntent.cancelPendingActivitySteers")(function* ( + tx: Transaction, + run: RunIdentity, + now: number, +) { + const rows = yield* tx + .select({ id: SessionSteerTable.id }) + .from(SessionSteerTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.admitted_message_id, SessionSteerTable.id), + ) + .innerJoin( + SessionLegacyActivityAdmissionTable, + eq(SessionLegacyActivityAdmissionTable.admission_id, SessionActivityAdmissionTable.admission_id), + ) + .where( + and( + eq(SessionSteerTable.session_id, run.sessionID), + eq(SessionSteerTable.mutation_epoch, run.mutationEpoch), + eq(SessionSteerTable.delivery, "steer"), + eq(SessionLegacyActivityAdmissionTable.activity_id, run.activityID), + isNull(SessionSteerTable.consumed_seq), + isNull(SessionSteerTable.superseded_at), + ), + ) + .all() + if (rows.length === 0) return + const ids = rows.map((row) => row.id) + yield* tx + .update(SessionIntentTable) + .set({ execution_state: "canceled", execution_claim_id: null, execution_claimed_at: null }) + .where( + and( + eq(SessionIntentTable.session_id, run.sessionID), + eq(SessionIntentTable.mutation_epoch, run.mutationEpoch), + eq(SessionIntentTable.execution_mode, "run_now"), + eq(SessionIntentTable.execution_state, "pending"), + inArray(SessionIntentTable.admitted_message_id, ids), + ), + ) + .run() + yield* tx + .update(SessionSteerTable) + .set({ superseded_at: now }) + .where( + and( + eq(SessionSteerTable.session_id, run.sessionID), + eq(SessionSteerTable.mutation_epoch, run.mutationEpoch), + isNull(SessionSteerTable.consumed_seq), + isNull(SessionSteerTable.superseded_at), + inArray(SessionSteerTable.id, ids), + ), + ) + .run() +}) + +function terminalRunState(state: ActivityTerminalState) { + return state === "settled" + ? ("completed" as const) + : state === "failed" + ? ("failed" as const) + : state === "interrupted" + ? ("interrupted" as const) + : ("recovery_required" as const) +} + +function terminalDecisionForProgress( + decision: ActivityTerminalDecision, + progressState: typeof SessionActivityProgressTable.$inferSelect.state, +): ActivityTerminalDecision { + if (progressState === "recovery_required" && decision.state !== "recovery_required") + return { ...decision, state: "recovery_required", reasonCode: "provider_outcome_indeterminate" } + if (progressState === "interrupted" && decision.state !== "recovery_required" && decision.state !== "interrupted") + return { ...decision, state: "interrupted", reasonCode: "provider_aborted" } + return decision +} + export const recoverActiveActivities = Effect.fn("SessionPromptIntent.recoverActiveActivities")(function* ( ownerToken = SessionActivityOwner.processOwnerToken, + options?: { + readonly includeCurrentOwner?: boolean + readonly sessionID?: SessionID + readonly source?: "restart_recovery" | "same_process_recovery" + }, ) { const { db } = yield* Database.Service + const source = options?.source ?? "restart_recovery" const active = yield* db .select({ activityID: SessionLegacyActivityTable.activity_id, @@ -949,93 +2146,239 @@ export const recoverActiveActivities = Effect.fn("SessionPromptIntent.recoverAct .where( and( eq(SessionLegacyActivityTable.state, "active"), - sql`${SessionLegacyActivityTable.owner_token} != ${ownerToken}`, + options?.includeCurrentOwner ? undefined : sql`${SessionLegacyActivityTable.owner_token} != ${ownerToken}`, + options?.sessionID ? eq(SessionLegacyActivityTable.session_id, options.sessionID) : undefined, ), ) .all() .pipe(Effect.orDie) - return yield* Effect.forEach( - active, - (activity) => - Effect.gen(function* () { - const latest = yield* db - .select() - .from(SessionActivityProgressTable) - .where(eq(SessionActivityProgressTable.activity_id, activity.activityID)) - .orderBy(sql`${SessionActivityProgressTable.revision} DESC`) - .get() - .pipe(Effect.orDie) - const receipt = latest - ? yield* db - .select({ state: SessionToolRequestReceiptTable.provider_state }) - .from(SessionToolRequestReceiptTable) - .where(eq(SessionToolRequestReceiptTable.receipt_id, latest.provider_receipt_id)) - .get() - .pipe(Effect.orDie) - : undefined - const settled = - latest?.state === "provisional" && - receipt && - ["settled", "failed", "indeterminate_after_crash"].includes(receipt.state) - ? yield* settleProgress({ - activityID: activity.activityID, - assistantMessageID: MessageID.make(latest.assistant_message_id), + return yield* Effect.forEach(active, (activity) => + db + .transaction( + (tx) => + Effect.gen(function* () { + const current = yield* tx + .select({ + activityID: SessionLegacyActivityTable.activity_id, + sessionID: SessionLegacyActivityTable.session_id, + ownerToken: SessionLegacyActivityTable.owner_token, + mutationEpoch: SessionIntentTable.mutation_epoch, + executionMode: SessionIntentTable.execution_mode, + executionState: SessionIntentTable.execution_state, }) - : undefined - if (settled && settled.state !== "progress") - return { - activityID: activity.activityID, - sessionID: SessionID.make(activity.sessionID), - assistantMessageID: settled.assistantMessageID, - } satisfies ProjectionInvalidation - const recoveryProgressState = settled?.state ?? latest?.state - const recoveryReason = settled - ? "process restarted after settled activity progress" - : recoveryProgressState - ? `process restarted after activity progress ${recoveryProgressState}` - : "process restarted before provider progress admission" - yield* db - .transaction( - (tx) => - Effect.gen(function* () { - const now = Date.now() - yield* tx - .update(SessionLegacyActivityTable) - .set({ - state: "recovery_required", - terminal_reason: recoveryReason, - settled_at: now, + .from(SessionLegacyActivityTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.admission_id, SessionLegacyActivityTable.trigger_admission_id), + ) + .innerJoin(SessionIntentTable, eq(SessionIntentTable.intent_id, SessionActivityAdmissionTable.legacy_intent_id)) + .where( + and( + eq(SessionLegacyActivityTable.activity_id, activity.activityID), + eq(SessionLegacyActivityTable.state, "active"), + ), + ) + .get() + if (!current) return undefined + const run = yield* tx + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.activity_id, activity.activityID)) + .orderBy(sql`${SessionLegacyActivityRunTable.generation} DESC`) + .get() + if (!run && current.executionMode === "run_now" && current.executionState === "pending") return undefined + const latest = yield* tx + .select() + .from(SessionActivityProgressTable) + .where(eq(SessionActivityProgressTable.activity_id, activity.activityID)) + .orderBy(sql`${SessionActivityProgressTable.revision} DESC`) + .get() + const receipt = latest + ? yield* tx + .select() + .from(SessionToolRequestReceiptTable) + .where(eq(SessionToolRequestReceiptTable.receipt_id, latest.provider_receipt_id)) + .get() + : yield* tx + .select({ + receiptID: SessionToolRequestReceiptTable.receipt_id, + providerState: SessionToolRequestReceiptTable.provider_state, + requestErrorCode: SessionToolRequestReceiptTable.request_error_code, }) - .where( - and( - eq(SessionLegacyActivityTable.activity_id, activity.activityID), - eq(SessionLegacyActivityTable.state, "active"), - ), + .from(SessionToolRequestReceiptTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.admitted_message_id, SessionToolRequestReceiptTable.user_message_id), ) - .run() - if (!settled && latest?.state === "provisional") - yield* tx - .update(SessionActivityProgressTable) - .set({ state: "recovery_required", finish_observed: "process_restart", settled_at: now }) - .where( - and( - eq(SessionActivityProgressTable.activity_id, activity.activityID), - eq(SessionActivityProgressTable.revision, latest.revision), - eq(SessionActivityProgressTable.state, "provisional"), - ), - ) - .run() - }), - { behavior: "immediate" }, - ) - .pipe(Effect.orDie) - return { - activityID: activity.activityID, - sessionID: SessionID.make(activity.sessionID), - ...(latest ? { assistantMessageID: MessageID.make(latest.assistant_message_id) } : {}), - } satisfies ProjectionInvalidation - }), + .innerJoin( + SessionLegacyActivityAdmissionTable, + eq(SessionLegacyActivityAdmissionTable.admission_id, SessionActivityAdmissionTable.admission_id), + ) + .where(eq(SessionLegacyActivityAdmissionTable.activity_id, activity.activityID)) + .orderBy(sql`${SessionToolRequestReceiptTable.request_ordinal} DESC`) + .get() + const receiptState = receipt + ? "provider_state" in receipt + ? receipt.provider_state + : receipt.providerState + : undefined + const receiptID = receipt + ? "receipt_id" in receipt + ? receipt.receipt_id + : receipt.receiptID + : undefined + const requestErrorCode = receipt + ? "request_error_code" in receipt + ? receipt.request_error_code + : receipt.requestErrorCode + : undefined + const now = Date.now() + if (run) + yield* cancelPendingActivitySteers( + tx, + { + runID: run.run_id, + activityID: run.activity_id, + sessionID: SessionID.make(run.session_id), + mutationEpoch: run.mutation_epoch, + generation: run.generation, + ownerToken: run.owner_token, + }, + now, + ) + if (receiptID && ["preparing", "prepared"].includes(receiptState ?? "")) + yield* tx + .update(SessionToolRequestReceiptTable) + .set({ provider_state: "failed", terminal_at: now, request_error_code: "pre_dispatch_owner_lost" }) + .where( + and( + eq(SessionToolRequestReceiptTable.receipt_id, receiptID), + inArray(SessionToolRequestReceiptTable.provider_state, ["preparing", "prepared"] as const), + ), + ) + .run() + if (receiptID && ["dispatching", "streaming"].includes(receiptState ?? "")) + yield* tx + .update(SessionToolRequestReceiptTable) + .set({ + provider_state: "indeterminate_after_crash", + terminal_at: now, + request_error_code: "provider_outcome_indeterminate", + }) + .where( + and( + eq(SessionToolRequestReceiptTable.receipt_id, receiptID), + inArray(SessionToolRequestReceiptTable.provider_state, ["dispatching", "streaming"] as const), + ), + ) + .run() + if (latest?.state === "provisional" && !receiptID) + yield* tx + .update(SessionActivityProgressTable) + .set({ + state: "recovery_required", + finish_observed: "provider_receipt_missing", + settled_at: now, + }) + .where( + and( + eq(SessionActivityProgressTable.activity_id, activity.activityID), + eq(SessionActivityProgressTable.revision, latest.revision), + eq(SessionActivityProgressTable.state, "provisional"), + ), + ) + .run() + const settled = + latest && receiptID + ? yield* settleProgressInTransaction( + tx, + { + activityID: activity.activityID, + assistantMessageID: MessageID.make(latest.assistant_message_id), + }, + false, + ) + : undefined + const preDispatch = + !receiptID || + ["preparing", "prepared"].includes(receiptState ?? "") || + requestErrorCode === "pre_dispatch_owner_lost" || + requestErrorCode === "provider_not_dispatched_before_process_restart" + const terminalState = !run + ? ("recovery_required" as const) + : preDispatch + ? ("failed" as const) + : ("recovery_required" as const) + const reasonCode = !run + ? "legacy_run_identity_missing" + : preDispatch + ? "pre_dispatch_owner_lost" + : ["dispatching", "streaming", "indeterminate_after_crash"].includes(receiptState ?? "") + ? "provider_outcome_indeterminate" + : "host_terminal_decision_missing" + const terminalRun = run && ["running", "finalizing"].includes(run.state) + if (terminalRun) + yield* tx + .update(SessionLegacyActivityRunTable) + .set({ state: terminalRunState(terminalState), terminal_at: now, terminal_reason: reasonCode }) + .where( + and( + eq(SessionLegacyActivityRunTable.run_id, run.run_id), + inArray(SessionLegacyActivityRunTable.state, ["running", "finalizing"] as const), + ), + ) + .run() + const updated = yield* tx + .update(SessionLegacyActivityTable) + .set({ state: terminalState, terminal_reason: reasonCode, settled_at: now }) + .where( + and( + eq(SessionLegacyActivityTable.activity_id, activity.activityID), + eq(SessionLegacyActivityTable.owner_token, current.ownerToken), + eq(SessionLegacyActivityTable.state, "active"), + ), + ) + .returning({ activityID: SessionLegacyActivityTable.activity_id }) + .get() + if (!updated) return yield* Effect.die(new Error(`legacy activity recovery CAS lost: ${activity.activityID}`)) + const membership = yield* tx + .select({ ordinal: max(SessionLegacyActivityAdmissionTable.ordinal) }) + .from(SessionLegacyActivityAdmissionTable) + .where(eq(SessionLegacyActivityAdmissionTable.activity_id, activity.activityID)) + .get() + const terminalProgress = settled?.row ?? latest + const terminalRunID = run && terminalRunState(terminalState) === run.state ? run.run_id : terminalRun ? run.run_id : null + yield* tx + .insert(SessionLegacyActivityTerminalTable) + .values({ + activity_id: activity.activityID, + session_id: current.sessionID, + mutation_epoch: current.mutationEpoch, + state: terminalState, + reason_code: reasonCode, + source, + operation_id: `legacy-activity-recovery:v1:${source}:${activity.activityID}`, + run_id: terminalRunID, + assistant_message_id: terminalProgress?.assistant_message_id ?? null, + progress_revision: terminalProgress?.revision ?? null, + membership_ordinal: terminalProgress?.input_membership_ordinal ?? membership?.ordinal ?? 0, + owner_token: current.ownerToken, + created_at: now, + }) + .run() + return { + activityID: activity.activityID, + sessionID: SessionID.make(current.sessionID), + ...(terminalProgress + ? { assistantMessageID: MessageID.make(terminalProgress.assistant_message_id) } + : {}), + } satisfies ProjectionInvalidation + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie), ) + .pipe(Effect.map((activities) => activities.filter((activity) => activity !== undefined))) }) export const interruptActivity = Effect.fn("SessionPromptIntent.interruptActivity")(function* (activityID: string) { @@ -1046,9 +2389,16 @@ export const interruptActivity = Effect.fn("SessionPromptIntent.interruptActivit Effect.gen(function* () { const updated = yield* tx .update(SessionLegacyActivityTable) - .set({ state: "interrupted", terminal_reason: "aborted_before_provider_settlement", settled_at: Date.now() }) + .set({ + state: "interrupted", + terminal_reason: "aborted_before_provider_settlement", + settled_at: Date.now(), + }) .where( - and(eq(SessionLegacyActivityTable.activity_id, activityID), eq(SessionLegacyActivityTable.state, "active")), + and( + eq(SessionLegacyActivityTable.activity_id, activityID), + eq(SessionLegacyActivityTable.state, "active"), + ), ) .returning({ sessionID: SessionLegacyActivityTable.session_id }) .get() diff --git a/packages/deepagent-code/src/session/prompt.ts b/packages/deepagent-code/src/session/prompt.ts index 51a18a82b..e4e91dcc0 100644 --- a/packages/deepagent-code/src/session/prompt.ts +++ b/packages/deepagent-code/src/session/prompt.ts @@ -90,6 +90,7 @@ import { projectDurableSettledRun, projectRecoveredSubagentRun, TaskTool, type T import { SessionRunState } from "./run-state" import { SessionSteer } from "./steer" import { SessionPromptIntent } from "./prompt-intent" +import { SessionActivityOwner } from "./activity-owner" import { pause as pauseAtActivityCrashPoint } from "./activity-crash-test" import { writeGovernanceAudit } from "./goal-governance-audit" import { RuntimeFlags } from "@/effect/runtime-flags" @@ -561,10 +562,12 @@ const promptInputToPrompt = (parts: PromptInput["parts"]): Effect.Effect Effect.Effect - readonly prompt: (input: PromptInput) => Effect.Effect + readonly prompt: ( + input: PromptInput, + ) => Effect.Effect readonly promptAsync: ( input: PromptInput, - ) => Effect.Effect + ) => Effect.Effect readonly prepareTaskInput: ( input: PromptInput, timeCreated: number, @@ -587,10 +590,12 @@ export interface Interface { // own busy semantics), preserving pre-steering behavior exactly. readonly promptOrSteer: ( input: PromptInput, - ) => Effect.Effect + ) => Effect.Effect readonly loop: (input: LoopInput, onRunning?: Effect.Effect) => Effect.Effect readonly shell: (input: ShellInput) => Effect.Effect - readonly command: (input: CommandInput) => Effect.Effect + readonly command: ( + input: CommandInput, + ) => Effect.Effect readonly resolvePromptParts: (template: string) => Effect.Effect readonly refineIntelligenceDraft: (input: { sessionID: SessionID @@ -633,12 +638,12 @@ type PromptLifecycle = { type ExecutePrompt = ( input: PromptInput, lifecycle?: PromptLifecycle, -) => Effect.Effect +) => Effect.Effect type ExecutePromptOrSteer = ( input: PromptInput, lifecycle?: PromptLifecycle, -) => Effect.Effect +) => Effect.Effect export const layer = Layer.effect( Service, @@ -698,6 +703,16 @@ export const layer = Layer.effect( messageID: input.assistantMessageID, }) : Effect.void + const reconcileInactiveActivity = Effect.fn("SessionPrompt.reconcileInactiveActivity")(function* ( + sessionID: SessionID, + ) { + const activities = yield* SessionPromptIntent.recoverActiveActivities(SessionActivityOwner.processOwnerToken, { + includeCurrentOwner: true, + sessionID, + source: "same_process_recovery", + }).pipe(Effect.provideService(Database.Service, database)) + yield* Effect.forEach(activities, publishActivityProjection, { discard: true }) + }) yield* recoverProviderReceiptsOnStartup() yield* SessionPromptIntent.recoverActiveActivities().pipe( Effect.provideService(Database.Service, database), @@ -1641,6 +1656,8 @@ export const layer = Layer.effect( readonly persist?: boolean readonly timeCreated?: number readonly intent?: PromptLifecycle["intent"] + readonly executionMode?: SessionPromptIntent.ExecutionMode + readonly onMaterialized?: (turn: SessionPromptIntent.MaterializedTurn) => Effect.Effect }, ) { const persist = options?.persist !== false @@ -2045,8 +2062,14 @@ export const layer = Layer.effect( if (!persist) return { info, parts } if (options?.intent) { - yield* SessionPromptIntent.materializeTurn({ receipt: options.intent, message: { info, parts } }).pipe( + yield* SessionPromptIntent.materializeTurn({ + receipt: options.intent, + message: { info, parts }, + executionMode: options.executionMode, + }).pipe( Effect.provideService(Database.Service, database), + Effect.tap((materialized) => options.onMaterialized?.(materialized) ?? Effect.void), + Effect.uninterruptible, ) } if (current?.agent !== info.agent) { @@ -2242,6 +2265,7 @@ export const layer = Layer.effect( variant: input.intentVariant ?? "original", payloadHash: promptIntentPayloadHash(input), messageID, + executionMode: input.noReply === true ? "deferred" : "run_now", }).pipe(Effect.provideService(Database.Service, database)) if (claimed.kind === "admitted") { if (claimed.receipt.delivery !== "turn") @@ -2285,42 +2309,92 @@ export const layer = Layer.effect( ), ) } - const mutationEpoch = - lifecycle?.intent?.mutationEpoch ?? (yield* sessions.mutationEpoch(session.id).pipe(Effect.orDie)) - yield* revert.cleanup(session, mutationEpoch) - const pipeline = yield* buildPromptPipelineSubmission(input) - const message = yield* createUserMessage( - { - ...input, - parts: pipeline.parts, - metadata: { - ...(input.metadata ?? {}), - deepagent: { - ...(isRecord(input.metadata?.deepagent) ? input.metadata.deepagent : {}), - prompt_pipeline: pipeline.metadata, + const ownedRun: { value?: SessionPromptIntent.RunIdentity } = {} + const admit = Effect.gen(function* () { + const mutationEpoch = + lifecycle?.intent?.mutationEpoch ?? (yield* sessions.mutationEpoch(session.id).pipe(Effect.orDie)) + yield* revert.cleanup(session, mutationEpoch) + const pipeline = yield* buildPromptPipelineSubmission(input) + const materialized = yield* Ref.make(undefined) + const message = yield* createUserMessage( + { + ...input, + parts: pipeline.parts, + metadata: { + ...(input.metadata ?? {}), + deepagent: { + ...(isRecord(input.metadata?.deepagent) ? input.metadata.deepagent : {}), + prompt_pipeline: pipeline.metadata, + }, }, }, - }, - lifecycle?.intent ? { intent: lifecycle.intent } : undefined, - ) - yield* sessions.touch(input.sessionID) + lifecycle?.intent + ? { + intent: lifecycle.intent, + executionMode: input.noReply === true ? "deferred" : "run_now", + onMaterialized: (turn) => + Ref.set(materialized, turn).pipe( + Effect.tap(() => + Effect.sync(() => { + ownedRun.value = turn.run + }), + ), + ), + } + : undefined, + ) + yield* sessions.touch(input.sessionID) - const permissions: PermissionV1.Rule[] = [] - for (const [t, enabled] of Object.entries(input.tools ?? {})) { - permissions.push({ permission: t, action: enabled ? "allow" : "deny", pattern: "*" }) - } - if (permissions.length > 0) { - session.permission = permissions - yield* sessions.setPermission({ sessionID: session.id, permission: permissions }) - } + const permissions: PermissionV1.Rule[] = [] + for (const [t, enabled] of Object.entries(input.tools ?? {})) { + permissions.push({ permission: t, action: enabled ? "allow" : "deny", pattern: "*" }) + } + if (permissions.length > 0) { + session.permission = permissions + yield* sessions.setPermission({ sessionID: session.id, permission: permissions }) + } + return { message, materialized: yield* Ref.get(materialized) } + }) if (input.noReply === true) { - if (lifecycle) yield* lifecycle.ready({ messageID: message.info.id, delivery: "turn" }) - return message + const admitted = yield* admit + if (admitted.materialized && "run" in admitted.materialized) + return yield* Effect.die(new Error(`deferred prompt created an activity run: ${input.sessionID}`)) + if (lifecycle) yield* lifecycle.ready({ messageID: admitted.message.info.id, delivery: "turn" }) + return admitted.message } - const first = yield* loop( - { sessionID: input.sessionID }, - lifecycle?.ready({ messageID: message.info.id, delivery: "turn" }), + const first = yield* state.startRunning( + input.sessionID, + lastAssistant(input.sessionID), + Effect.gen(function* () { + yield* reconcileInactiveActivity(input.sessionID) + const admitted = yield* admit + if (!admitted.materialized || !("run" in admitted.materialized)) + return yield* Effect.die(new Error(`run-now prompt did not create an activity run: ${input.sessionID}`)) + const run = admitted.materialized.run + if (!run) return yield* Effect.die(new Error(`run-now prompt created an empty activity run: ${input.sessionID}`)) + yield* pauseAtActivityCrashPoint("after_admit_and_bind") + if (lifecycle) yield* lifecycle.ready({ messageID: admitted.message.info.id, delivery: "turn" }) + return yield* runLoop(input.sessionID, false, run) + }).pipe( + Effect.onInterrupt(() => { + const run = ownedRun.value + return Effect.all( + [ + settleFederatedActivity(input.sessionID, "interrupted"), + run + ? SessionPromptIntent.finalizeCancellationBeforeProgress(run).pipe( + Effect.provideService(Database.Service, database), + Effect.flatMap((result) => + result ? publishActivityProjection(result.invalidation) : Effect.void, + ), + ) + : Effect.void, + ], + { discard: true }, + ) + }), + ), ) if (isStructuredFinalizer(input.metadata)) return first // V3 Plan A: mode-driven multi-round autonomous loop for high/max/ultra. It remains @@ -2764,14 +2838,22 @@ export const layer = Layer.effect( "", ].join("\n") - const runLoop: (sessionID: SessionID, drainFirst?: boolean) => Effect.Effect = Effect.fn( - "SessionPrompt.run", - )( + const runLoop: ( + sessionID: SessionID, + drainFirst?: boolean, + activityRun?: SessionPromptIntent.RunIdentity, + onActivityRun?: (run: SessionPromptIntent.RunIdentity) => void, + ) => Effect.Effect = Effect.fn("SessionPrompt.run")( // §S1.2: `drainFirst` — a PURE-DRAIN turn (started to absorb a steer that landed in the isBusy→admit // race, with no initiating user message of its own) must drain on step 0 too; otherwise the step-0 // skip + the immediate finish check would break before the steer is ever consumed. A normal turn // leaves it false so the initiating message samples first (S1.1). - function* (sessionID: SessionID, drainFirst = false) { + function* ( + sessionID: SessionID, + drainFirst = false, + activityRun?: SessionPromptIntent.RunIdentity, + onActivityRun?: (run: SessionPromptIntent.RunIdentity) => void, + ) { const ctx = yield* InstanceState.context const slog = elog.with({ sessionID }) let structured: unknown @@ -2828,12 +2910,62 @@ export const layer = Layer.effect( ), ) const initialUser = MessageV2.latest(initialMessages).user - let legacyActivity = initialUser - ? yield* SessionPromptIntent.activityForMessage({ sessionID, messageID: initialUser.id }).pipe( - Effect.provideService(Database.Service, database), - Effect.map((activity) => (activity?.state === "active" ? activity : undefined)), - ) - : undefined + let currentActivityRun = activityRun + let legacyActivity = currentActivityRun + ? { activityID: currentActivityRun.activityID, state: "active" as const } + : initialUser + ? yield* SessionPromptIntent.activityForMessage({ sessionID, messageID: initialUser.id }).pipe( + Effect.provideService(Database.Service, database), + Effect.map((activity) => (activity?.state === "active" ? activity : undefined)), + ) + : undefined + let providerBoundary: SessionPromptIntent.ProviderInputBoundary | undefined + let activityTerminalCommitted = false + const terminalDecision = ( + state: SessionPromptIntent.ActivityTerminalState, + reasonCode: string, + source: "provider_final" | "host_stop" | "cancel" = "host_stop", + ): SessionPromptIntent.ActivityTerminalDecision | undefined => + currentActivityRun + ? { + state, + reasonCode, + source, + operationID: `${currentActivityRun.runID}:terminal`, + ownerToken: currentActivityRun.ownerToken, + } + : undefined + const finalizeHostExit = ( + state: SessionPromptIntent.ActivityTerminalState, + reasonCode: string, + assistantMessageID?: MessageID, + source: "provider_final" | "host_stop" | "compaction" = "host_stop", + ) => { + const decision = terminalDecision(state, reasonCode, source === "provider_final" ? source : "host_stop") + if (!currentActivityRun || !providerBoundary || !decision) return Effect.succeed(undefined) + const finalized = assistantMessageID + ? SessionPromptIntent.finalizeActivityWithRevision({ + run: currentActivityRun, + assistantMessageID, + decision, + }) + : SessionPromptIntent.finalizeActivityWithoutRevision({ + run: currentActivityRun, + membershipOrdinal: providerBoundary.membershipOrdinal, + decision: { ...decision, source }, + }) + return finalized.pipe( + Effect.provideService(Database.Service, database), + Effect.tap((result) => + Effect.sync(() => { + activityTerminalCommitted = result.kind !== "follow_up_required" + }), + ), + Effect.tap((result) => + result.kind === "follow_up_required" ? Effect.void : publishActivityProjection(result.invalidation), + ), + ) + } const publishActivityProgress = (progress: SessionPromptIntent.Progress) => Effect.gen(function* () { yield* sessions.publishMessageProjection({ @@ -2900,6 +3032,17 @@ export const layer = Layer.effect( // arrived after the model said "done" is naturally absorbed on this next pass. if (step > 0 || drainFirst) { if (!(yield* compaction.hasPending(sessionID))) { + if (!currentActivityRun && flags.v4Steering && (yield* steerBuffer.hasPending(sessionID))) { + currentActivityRun = yield* SessionPromptIntent.claimActiveActivityRun({ sessionID }).pipe( + Effect.provideService(Database.Service, database), + Effect.tap((run) => Effect.sync(() => run && onActivityRun?.(run))), + Effect.uninterruptible, + ) + if (currentActivityRun) { + yield* pauseAtActivityCrashPoint("after_admit_and_bind") + legacyActivity = { activityID: currentActivityRun.activityID, state: "active" as const } + } + } const absorbed = yield* drainSteers(sessionID, drainFirst && step === 0) pendingContextInputIds = [...pendingContextInputIds, ...absorbed] } @@ -2923,6 +3066,17 @@ export const layer = Layer.effect( messageID: lastUser.id, }).pipe(Effect.provideService(Database.Service, database)) if (currentLegacyActivity?.state === "active") legacyActivity = currentLegacyActivity + if (currentActivityRun) { + const frozen = yield* SessionPromptIntent.freezeProviderInputBoundary(currentActivityRun, { + includePendingSteers: flags.v4Steering, + }).pipe(Effect.provideService(Database.Service, database)) + if (frozen.kind === "pending_steer") { + const absorbed = yield* drainSteers(sessionID) + pendingContextInputIds = [...pendingContextInputIds, ...absorbed] + continue + } + providerBoundary = frozen.boundary + } const finalizerMode = isStructuredFinalizer(lastUser.metadata) const finalizerAllowsText = structuredFinalizerAllowsText(lastUser.metadata) @@ -2932,6 +3086,12 @@ export const layer = Layer.effect( const elapsed = taskActivity?.startedAt ? Math.max(0, Date.now() - taskActivity.startedAt) : 0 if (lastAssistant && taskActivity?.maxWallMs && elapsed >= taskActivity.maxWallMs) { yield* failTaskBudget(lastAssistant, "wall_time", taskActivity.maxWallMs, elapsed) + const finalized = yield* finalizeHostExit( + "failed", + "task_wall_budget_exhausted", + lastAssistant.id, + ) + if (finalized?.kind === "follow_up_required") continue break } // Some providers return "stop" even when the assistant message contains @@ -2981,6 +3141,8 @@ export const layer = Layer.effect( yield* slog.warn("output soft-landing: continuation cap reached, ending turn", { max: outputContinuationMax(), }) + const finalized = yield* finalizeHostExit("failed", "output_continuation_exhausted", lastAssistant.id) + if (finalized?.kind === "follow_up_required") continue } if (orphan) { yield* slog.warn("loop exit with orphaned interrupted tool", { @@ -3007,6 +3169,8 @@ export const layer = Layer.effect( if (step === 1 && finalizerMode) LLMRequestPrep.resetCacheHitOutcome(sessionID) if (lastAssistant && taskActivity?.maxSteps && step > taskActivity.maxSteps) { yield* failTaskBudget(lastAssistant, "steps", taskActivity.maxSteps, step - 1) + const finalized = yield* finalizeHostExit("failed", "task_step_budget_exhausted", lastAssistant.id) + if (finalized?.kind === "follow_up_required") continue break } if (step === 1 && !finalizerMode) { @@ -3036,7 +3200,11 @@ export const layer = Layer.effect( auto: task.auto, overflow: task.overflow, }) - if (result === "stop") break + if (result === "stop") { + const finalized = yield* finalizeHostExit("recovery_required", "compaction_stop", undefined, "compaction") + if (finalized?.kind === "follow_up_required") continue + break + } continue } @@ -3223,6 +3391,11 @@ export const layer = Layer.effect( reason: finalizerDecision.reason, protocol: finalizerDecision.protocol, }) + const finalized = yield* finalizeHostExit( + "failed", + `structured_finalizer_unsupported:${finalizerDecision.reason}`, + ) + if (finalized?.kind === "follow_up_required") continue break } @@ -3241,13 +3414,40 @@ export const layer = Layer.effect( } = {} const receiptFinalizer: { value?: () => Effect.Effect } = {} const activityProgressFinalizer: { value?: () => Effect.Effect } = {} + let progressAssistantMessageID: MessageID | undefined const finalizeInterruptedTurn = Effect.uninterruptible( Effect.gen(function* () { yield* finalizeInterruptedAssistant receiptTerminal.value ??= { state: "failed", errorCode: "AbortError" } if (receiptFinalizer.value) yield* receiptFinalizer.value() - if (activityProgressFinalizer.value) yield* activityProgressFinalizer.value() - if (legacyActivity && !activityProgressFinalizer.value) + const decision = terminalDecision("interrupted", "user_cancelled", "cancel") + const finalized = + currentActivityRun && providerBoundary && decision + ? yield* (progressAssistantMessageID + ? SessionPromptIntent.finalizeActivityWithRevision({ + run: currentActivityRun, + assistantMessageID: progressAssistantMessageID, + decision, + }) + : SessionPromptIntent.finalizeActivityWithoutRevision({ + run: currentActivityRun, + membershipOrdinal: providerBoundary.membershipOrdinal, + decision, + })) + .pipe( + Effect.provideService(Database.Service, database), + Effect.tap((result) => + Effect.sync(() => { + activityTerminalCommitted = result.kind !== "follow_up_required" + }), + ), + Effect.tap((result) => + result.kind === "follow_up_required" ? Effect.void : publishActivityProjection(result.invalidation), + ), + ) + : undefined + if (!finalized && activityProgressFinalizer.value) yield* activityProgressFinalizer.value() + if (legacyActivity && !activityProgressFinalizer.value && !finalized) yield* SessionPromptIntent.interruptActivity(legacyActivity.activityID).pipe( Effect.provideService(Database.Service, database), Effect.flatMap((activity) => (activity ? publishActivityProjection(activity) : Effect.void)), @@ -3583,13 +3783,12 @@ export const layer = Layer.effect( activityID: progressActivity.activityID, assistantMessageID: handle.message.id, providerReceiptID: receiptID, - }).pipe( - Effect.provideService(Database.Service, database), - Effect.tap(publishActivityProgress), - ) + membershipOrdinal: providerBoundary?.membershipOrdinal, + }).pipe(Effect.provideService(Database.Service, database), Effect.tap(publishActivityProgress)) + if (progressActivity) progressAssistantMessageID = handle.message.id if (progressActivity) activityProgressFinalizer.value = () => - SessionPromptIntent.settleProgress({ + (currentActivityRun ? SessionPromptIntent.settleProgressOnly : SessionPromptIntent.settleProgress)({ activityID: progressActivity.activityID, assistantMessageID: handle.message.id, }).pipe( @@ -3789,18 +3988,35 @@ export const layer = Layer.effect( }) receiptFinalizer.value = finalizeReceipt const turnSettled = { value: false } - const settleProviderTurn = () => + const settleProviderTurn = (decision?: SessionPromptIntent.ActivityTerminalDecision) => Effect.gen(function* () { - if (turnSettled.value) return + if (turnSettled.value) return undefined yield* finalizeReceipt() yield* pauseAtActivityCrashPoint("after_provider_receipt_terminal") - if (activityProgressFinalizer.value) yield* activityProgressFinalizer.value() + const finalized = + decision && currentActivityRun && progressActivity + ? yield* SessionPromptIntent.finalizeActivityWithRevision({ + run: currentActivityRun, + assistantMessageID: handle.message.id, + decision, + }).pipe( + Effect.provideService(Database.Service, database), + Effect.tap((result) => + result.kind === "follow_up_required" + ? Effect.void + : publishActivityProjection(result.invalidation), + ), + ) + : undefined + if (!finalized && activityProgressFinalizer.value) yield* activityProgressFinalizer.value() + activityProgressFinalizer.value = undefined yield* pauseAtActivityCrashPoint("after_progress_settled") turnSettled.value = true // Summary diffs mutate user-message metadata. Run them only after the Provider // receipt is terminal so cancellation cannot strand an admitted request. if (step === 1 && !finalizerMode) yield* summary.summarize({ sessionID, messageID: lastUser.id }).pipe(Effect.ignore) + return finalized }) const prepareAdapterReceipt = (input: { finalRequestHash: string @@ -3980,7 +4196,7 @@ export const layer = Layer.effect( handle.message.structured = structured handle.message.finish = handle.message.finish ?? "stop" yield* sessions.updateMessage(handle.message) - yield* settleProviderTurn() + yield* settleProviderTurn(terminalDecision("settled", "structured_output_completed", "provider_final")) return "break" as const } @@ -3991,7 +4207,9 @@ export const layer = Layer.effect( (part) => part.type === "text" && !part.synthetic && !part.ignored && part.text.trim() !== "", ) if (hasTextFallback) { - yield* settleProviderTurn() + yield* settleProviderTurn( + terminalDecision("settled", "structured_output_completed", "provider_final"), + ) return "break" as const } if (!handle.message.error) { @@ -4001,7 +4219,7 @@ export const layer = Layer.effect( }).toObject() yield* sessions.updateMessage(handle.message) } - yield* settleProviderTurn() + yield* settleProviderTurn(terminalDecision("failed", "structured_finalizer_invalid")) return "break" as const } @@ -4013,7 +4231,7 @@ export const layer = Layer.effect( retries: 0, }).toObject() yield* sessions.updateMessage(handle.message) - yield* settleProviderTurn() + yield* settleProviderTurn(terminalDecision("failed", "structured_output_missing")) return "break" as const } } @@ -4053,7 +4271,7 @@ export const layer = Layer.effect( retries: structuredFailedAttempts, }).toObject() yield* sessions.updateMessage(handle.message) - yield* settleProviderTurn() + yield* settleProviderTurn(terminalDecision("failed", "structured_output_retry_exhausted")) yield* slog.warn("structured-output retry cap reached", { attempts: structuredFailedAttempts, retryMax, @@ -4077,9 +4295,30 @@ export const layer = Layer.effect( } } + if (result.action === "stop") { + const rejected = + result.reason.code === "user_rejected_question" || result.reason.code === "user_rejected_permission" + const finalized = yield* settleProviderTurn( + terminalDecision( + rejected ? "interrupted" : "failed", + result.reason.code === "assistant_error" + ? `${result.reason.code}:${result.reason.errorName}` + : result.reason.code, + ), + ) + return finalized?.kind === "follow_up_required" ? ("continue" as const) : ("break" as const) + } + const hasToolCalls = response.parts.some( + (part) => part.type === "tool" && !part.metadata?.providerExecuted && !isOrphanedInterruptedTool(part), + ) + if (result.action === "continue" && handle.message.finish === "stop" && !hasToolCalls) { + const finalized = yield* settleProviderTurn( + terminalDecision("settled", "assistant_completed", "provider_final"), + ) + return finalized?.kind === "follow_up_required" ? ("continue" as const) : ("break" as const) + } yield* settleProviderTurn() - if (result === "stop") return "break" as const - if (result === "compact") { + if (result.action === "compact") { // V4.0.1 P0 — a turn-internal hard compaction (the provider signalled overflow mid-stream). // This IS a hard rollover, so bump the soft-landing generation + reset its flags so the next // window can warn + flush again. Gated by softLandingCompaction; OFF ⇒ unchanged behavior. @@ -4109,6 +4348,7 @@ export const layer = Layer.effect( // it (codex: needsFollowUp = modelSaidContinue || pendingInput-nonempty). A non-consuming peek; // the actual drain (consume-once) happens at the next iteration's top. Gated by the flag. if (outcome === "break") { + if (activityTerminalCommitted) break if (!finalizerMode && flags.v4Steering && (yield* steerBuffer.hasPending(sessionID))) { yield* slog.info("steer pending at model boundary, continuing to absorb") continue @@ -4199,6 +4439,10 @@ export const layer = Layer.effect( input: PromptInput, lifecycle?: PromptLifecycle, ) { + if (input.noReply === true) { + const message = yield* prompt(input, lifecycle) + return { kind: "turn" as const, message } + } if (!flags.v4Steering) { const message = yield* prompt(input, lifecycle) return { kind: "turn" as const, message } @@ -4259,7 +4503,7 @@ export const layer = Layer.effect( const promptAsync: ( input: PromptInput, - ) => Effect.Effect = Effect.fn( + ) => Effect.Effect = Effect.fn( "SessionPrompt.promptAsync", )(function* (input: PromptInput) { const messageID = input.messageID ?? MessageID.ascending() @@ -4273,6 +4517,7 @@ export const layer = Layer.effect( (promptPipelineRequest(input.metadata).confirmedDraftID ? "rewritten" : "original"), payloadHash: promptIntentPayloadHash(input), messageID, + executionMode: input.noReply === true ? "deferred" : "run_now", }).pipe(Effect.provideService(Database.Service, database)) : undefined if (claim?.kind === "admitted") @@ -4288,7 +4533,10 @@ export const layer = Layer.effect( parts: stableIntentParts(input.parts, claimed.intentID), } : { ...input, messageID } - const admission = yield* Deferred.make() + const admission = yield* Deferred.make< + PromptAdmissionReceipt, + Image.Error | SessionPromptIntent.Error | Session.BusyError + >() if (claimed) { yield* Effect.suspend(() => SessionPromptIntent.renew({ @@ -4369,11 +4617,56 @@ export const layer = Layer.effect( if (FSUtil.resolve(session.directory) !== FSUtil.resolve(current.directory)) { return yield* instances.provide({ directory: session.directory }, loop(input, onRunning)) } + const ownedRun: { value?: SessionPromptIntent.RunIdentity } = {} return yield* state.ensureRunning( input.sessionID, lastAssistant(input.sessionID), - runLoop(input.sessionID, input.drainFirst ?? false).pipe( - Effect.onInterrupt(() => settleFederatedActivity(input.sessionID, "interrupted")), + Effect.gen(function* () { + yield* reconcileInactiveActivity(input.sessionID) + const deferredRun = yield* SessionPromptIntent.claimDeferredActivity({ sessionID: input.sessionID }).pipe( + Effect.provideService(Database.Service, database), + Effect.tap((run) => + Effect.sync(() => { + ownedRun.value = run + }), + ), + Effect.uninterruptible, + ) + const activityRun = + deferredRun ?? + (input.drainFirst + ? yield* SessionPromptIntent.claimActiveActivityRun({ sessionID: input.sessionID }).pipe( + Effect.provideService(Database.Service, database), + Effect.tap((run) => + Effect.sync(() => { + ownedRun.value = run + }), + ), + Effect.uninterruptible, + ) + : undefined) + if (activityRun) yield* pauseAtActivityCrashPoint("after_admit_and_bind") + return yield* runLoop(input.sessionID, input.drainFirst ?? false, activityRun, (run) => { + ownedRun.value = run + }) + }).pipe( + Effect.onInterrupt(() => { + const run = ownedRun.value + return Effect.all( + [ + settleFederatedActivity(input.sessionID, "interrupted"), + run + ? SessionPromptIntent.finalizeCancellationBeforeProgress(run).pipe( + Effect.provideService(Database.Service, database), + Effect.flatMap((result) => + result ? publishActivityProjection(result.invalidation) : Effect.void, + ), + ) + : Effect.void, + ], + { discard: true }, + ) + }), ), onRunning, ) @@ -4732,8 +5025,24 @@ export const layer = Layer.effect( TaskDelivery.deliverOne({ item, ownerToken: deliveryOwner, - driveParentLoop: () => - runLoop(item.parentSessionID).pipe(Effect.provideService(InstanceRef, ctx)), + driveParentLoop: () => { + const ownedRun: { value?: SessionPromptIntent.RunIdentity } = {} + return runLoop(item.parentSessionID, false, undefined, (run) => { + ownedRun.value = run + }).pipe( + Effect.provideService(InstanceRef, ctx), + Effect.onInterrupt(() => { + const run = ownedRun.value + if (!run) return Effect.void + return SessionPromptIntent.finalizeCancellationBeforeProgress(run).pipe( + Effect.provideService(Database.Service, database), + Effect.flatMap((result) => + result ? publishActivityProjection(result.invalidation) : Effect.void, + ), + ) + }), + ) + }, }).pipe( Effect.provideService(Database.Service, database), Effect.tap((result) => Ref.set(delivered, result)), @@ -5058,7 +5367,7 @@ const promptIntentPayloadHash = (input: PromptInput) => sessionID: input.sessionID, model: input.model, agent: input.agent, - noReply: input.noReply, + executionMode: input.noReply === true ? "deferred" : "run_now", tools: input.tools, format: input.format, system: input.system, diff --git a/packages/deepagent-code/src/session/run-state.ts b/packages/deepagent-code/src/session/run-state.ts index 89b8899ea..bb7d23377 100644 --- a/packages/deepagent-code/src/session/run-state.ts +++ b/packages/deepagent-code/src/session/run-state.ts @@ -6,6 +6,10 @@ import { Effect, Latch, Layer, Scope, Context } from "effect" import { Session } from "./session" import { SessionID } from "./schema" import { SessionStatus } from "./status" +import type { Image } from "@/image/image" +import type { SessionPromptIntent } from "./prompt-intent" + +type RunError = Image.Error | SessionPromptIntent.Error export interface Interface { readonly assertNotBusy: (sessionID: SessionID) => Effect.Effect @@ -18,9 +22,14 @@ export interface Interface { readonly ensureRunning: ( sessionID: SessionID, onInterrupt: Effect.Effect, - work: Effect.Effect, + work: Effect.Effect, onRunning?: Effect.Effect, ) => Effect.Effect + readonly startRunning: ( + sessionID: SessionID, + onInterrupt: Effect.Effect, + work: Effect.Effect, + ) => Effect.Effect readonly startShell: ( sessionID: SessionID, onInterrupt: Effect.Effect, @@ -40,7 +49,7 @@ export const layer = Layer.effect( const state = yield* InstanceState.make( Effect.fn("SessionRunState.state")(function* () { const scope = yield* Scope.Scope - const runners = new Map>() + const runners = new Map>() yield* Effect.addFinalizer( Effect.fnUntraced(function* () { yield* Effect.forEach(runners.values(), (runner) => runner.cancel, { @@ -61,7 +70,7 @@ export const layer = Layer.effect( const data = yield* InstanceState.get(state) const existing = data.runners.get(sessionID) if (existing) return existing - const next = Runner.make(data.scope, { + const next = Runner.make(data.scope, { onIdle: Effect.gen(function* () { data.runners.delete(sessionID) yield* status.set(sessionID, { type: "idle" }) @@ -98,10 +107,12 @@ export const layer = Layer.effect( const ensureRunning = Effect.fn("SessionRunState.ensureRunning")(function* ( sessionID: SessionID, onInterrupt: Effect.Effect, - work: Effect.Effect, + work: Effect.Effect, onRunning?: Effect.Effect, ) { - return yield* (yield* runner(sessionID, onInterrupt)).ensureRunning(work, onRunning) + return yield* (yield* runner(sessionID, onInterrupt)) + .ensureRunning(work, onRunning) + .pipe(Effect.catch(Effect.die)) }) const startShell = Effect.fn("SessionRunState.startShell")(function* ( @@ -112,10 +123,24 @@ export const layer = Layer.effect( ) { return yield* (yield* runner(sessionID, onInterrupt)) .startShell(work, ready) + .pipe( + Effect.catch((error) => + error instanceof Runner.Busy ? Effect.fail(busyError(sessionID)) : Effect.die(error), + ), + ) + }) + + const startRunning = Effect.fn("SessionRunState.startRunning")(function* ( + sessionID: SessionID, + onInterrupt: Effect.Effect, + work: Effect.Effect, + ) { + return yield* (yield* runner(sessionID, onInterrupt)) + .startRunning(work) .pipe(Effect.catchTag("RunnerBusy", () => Effect.fail(busyError(sessionID)))) }) - return Service.of({ assertNotBusy, isBusy, cancel, ensureRunning, startShell }) + return Service.of({ assertNotBusy, isBusy, cancel, ensureRunning, startRunning, startShell }) }), ) diff --git a/packages/deepagent-code/src/session/steer.ts b/packages/deepagent-code/src/session/steer.ts index 8e8e491f6..459a3654d 100644 --- a/packages/deepagent-code/src/session/steer.ts +++ b/packages/deepagent-code/src/session/steer.ts @@ -20,6 +20,7 @@ import { SessionPromptEpochTable } from "./prompt-epoch.sql" import { SessionActivityAdmissionTable, SessionLegacyActivityAdmissionTable, + SessionLegacyActivityRunTable, SessionLegacyActivityTable, } from "./activity-sql" import { SessionActivityOwner } from "./activity-owner" @@ -242,6 +243,8 @@ export const layer = Layer.effect( admitted_message_id: admitted.id, correlation_id: input.correlationID ?? admitted.id, mutation_epoch: session.mutationEpoch, + execution_mode: delivery === "steer" ? "run_now" : "legacy", + execution_state: delivery === "steer" ? "pending" : "legacy", version: 1, time_created: admitted.timeCreated, time_selected: admitted.timeCreated, @@ -281,6 +284,7 @@ export const layer = Layer.effect( delivery, payload_fingerprint_kind: "payload_hash", payload_fingerprint: payloadHash, + execution_mode: storedIntent.execution_mode, created_at: storedIntent.time_created, }) .onConflictDoNothing() @@ -574,6 +578,71 @@ export const layer = Layer.effect( .get() .pipe(Effect.orDie) if (!row) return false + const execution = yield* tx + .select({ + intentID: SessionIntentTable.intent_id, + executionMode: SessionIntentTable.execution_mode, + executionState: SessionIntentTable.execution_state, + executionClaimID: SessionIntentTable.execution_claim_id, + activityID: SessionLegacyActivityAdmissionTable.activity_id, + }) + .from(SessionIntentTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.legacy_intent_id, SessionIntentTable.intent_id), + ) + .innerJoin( + SessionLegacyActivityAdmissionTable, + eq(SessionLegacyActivityAdmissionTable.admission_id, SessionActivityAdmissionTable.admission_id), + ) + .innerJoin( + SessionLegacyActivityTable, + eq(SessionLegacyActivityTable.activity_id, SessionLegacyActivityAdmissionTable.activity_id), + ) + .where( + and( + eq(SessionActivityAdmissionTable.admitted_message_id, input.admitted.id), + eq(SessionLegacyActivityTable.state, "active"), + ), + ) + .get() + .pipe(Effect.orDie) + if (execution?.executionMode === "run_now") { + const run = yield* tx + .select({ runID: SessionLegacyActivityRunTable.run_id }) + .from(SessionLegacyActivityRunTable) + .where( + and( + eq(SessionLegacyActivityRunTable.activity_id, execution.activityID), + eq(SessionLegacyActivityRunTable.session_id, input.admitted.sessionID), + eq(SessionLegacyActivityRunTable.mutation_epoch, session.mutationEpoch), + inArray(SessionLegacyActivityRunTable.state, ["running", "finalizing"]), + ), + ) + .get() + .pipe(Effect.orDie) + if (!run) return yield* Effect.die("SessionSteer.materialize: active steer has no live run") + if (execution.executionState === "pending") + yield* tx + .update(SessionIntentTable) + .set({ + execution_state: "claimed", + execution_claim_id: run.runID, + execution_claimed_at: DateTime.toEpochMillis(yield* DateTime.now), + }) + .where( + and( + eq(SessionIntentTable.intent_id, execution.intentID), + eq(SessionIntentTable.execution_mode, "run_now"), + eq(SessionIntentTable.execution_state, "pending"), + isNull(SessionIntentTable.execution_claim_id), + ), + ) + .run() + .pipe(Effect.orDie) + if (execution.executionState === "claimed" && execution.executionClaimID !== run.runID) + return yield* Effect.die("SessionSteer.materialize: steer run claim conflicts") + } const { id: _, sessionID: __, ...message } = input.info const data = message as Types.DeepMutable const storedMessage = yield* tx diff --git a/packages/deepagent-code/test/effect/runner.test.ts b/packages/deepagent-code/test/effect/runner.test.ts index 410b79ced..dfc504a7e 100644 --- a/packages/deepagent-code/test/effect/runner.test.ts +++ b/packages/deepagent-code/test/effect/runner.test.ts @@ -11,6 +11,29 @@ const waitForState = (runner: Runner.Runner, tag: Runner.State describe("Runner", () => { // --- ensureRunning semantics --- + it.live( + "startRunning reserves idle atomically and rejects a concurrent start", + Effect.gen(function* () { + const s = yield* Scope.Scope + const runner = Runner.make(s) + const started = yield* Deferred.make() + const release = yield* Deferred.make() + const first = yield* runner + .startRunning( + Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(release)), Effect.as("first")), + ) + .pipe(Effect.forkChild) + + yield* Deferred.await(started) + const second = yield* runner.startRunning(Effect.succeed("second")).pipe(Effect.exit) + + expect(Exit.isFailure(second)).toBe(true) + expect(runner.state._tag).toBe("Running") + yield* Deferred.succeed(release, undefined) + expect(yield* Fiber.join(first)).toBe("first") + }), + ) + it.live( "ensureRunning starts work and returns result", Effect.gen(function* () { diff --git a/packages/deepagent-code/test/session/compaction.test.ts b/packages/deepagent-code/test/session/compaction.test.ts index d43832872..1a23c2e70 100644 --- a/packages/deepagent-code/test/session/compaction.test.ts +++ b/packages/deepagent-code/test/session/compaction.test.ts @@ -209,6 +209,7 @@ function fake( result: "continue" | "compact", session: SessionNs.Interface, ) { + const decision = { action: result } as const const msg = input.assistantMessage const processSummary: SessionProcessorModule.SessionProcessor.Handle["processSummary"] = Effect.fn( "TestSessionProcessor.processSummary", @@ -230,7 +231,7 @@ function fake( }) } yield* attempt.settled - return result + return decision }) return { get message() { @@ -238,7 +239,7 @@ function fake( }, updateToolCall: Effect.fn("TestSessionProcessor.updateToolCall")(() => Effect.succeed(undefined)), completeToolCall: Effect.fn("TestSessionProcessor.completeToolCall")(() => Effect.void), - process: Effect.fn("TestSessionProcessor.process")(() => Effect.succeed(result)), + process: Effect.fn("TestSessionProcessor.process")(() => Effect.succeed(decision)), processSummary, } satisfies SessionProcessorModule.SessionProcessor.Handle } diff --git a/packages/deepagent-code/test/session/processor-effect.test.ts b/packages/deepagent-code/test/session/processor-effect.test.ts index 1dadfb852..8b23e2eca 100644 --- a/packages/deepagent-code/test/session/processor-effect.test.ts +++ b/packages/deepagent-code/test/session/processor-effect.test.ts @@ -1,5 +1,6 @@ import { NodeFileSystem } from "@effect/platform-node" import { SessionV1 } from "@deepagent-code/core/v1/session" +import { PermissionV1 } from "@deepagent-code/core/v1/permission" import { Database } from "@deepagent-code/core/database/database" import { EventV2Bridge } from "@/event-v2-bridge" import { expect } from "bun:test" @@ -12,6 +13,7 @@ import { Agent as AgentSvc } from "../../src/agent/agent" import { Config } from "@/config/config" import { Image } from "@/image/image" import { Permission } from "../../src/permission" +import { Question } from "../../src/question" import { Plugin } from "../../src/plugin" import { Provider } from "@/provider/provider" @@ -237,6 +239,49 @@ const providerErrorEnv = SessionProcessor.layer.pipe( ) const itProviderError = testEffect(providerErrorEnv) +function typedToolFailureEnv(failure: { id: string; name: string; error: Error }) { + return SessionProcessor.layer.pipe( + Layer.provide(summary), + Layer.provide(Image.defaultLayer), + Layer.provide(RuntimeFlags.layer({ experimentalEventSystem: true })), + Layer.provide( + Layer.succeed( + LLM.Service, + LLM.Service.of({ + stream: () => + Stream.make( + LLMEvent.stepStart({ index: 0 }), + LLMEvent.toolInputStart({ id: failure.id, name: failure.name }), + LLMEvent.toolInputEnd({ id: failure.id, name: failure.name }), + LLMEvent.toolCall({ id: failure.id, name: failure.name, input: {} }), + LLMEvent.toolResult({ + id: failure.id, + name: failure.name, + result: { type: "error", value: failure.error }, + }), + LLMEvent.stepFinish({ index: 0, reason: "tool-calls" }), + LLMEvent.finish({ reason: "tool-calls" }), + ), + }), + ), + ), + Layer.provideMerge(deps), + ) +} +const itQuestionRejected = testEffect( + typedToolFailureEnv({ id: "question-call", name: "question", error: new Question.RejectedError() }), +) +const itPermissionRejected = testEffect( + typedToolFailureEnv({ id: "permission-call", name: "bash", error: new PermissionV1.RejectedError() }), +) +const itPermissionCorrected = testEffect( + typedToolFailureEnv({ + id: "permission-corrected-call", + name: "bash", + error: new PermissionV1.CorrectedError({ feedback: "Use the approved directory" }), + }), +) + const fragmentFailureLLM = Layer.succeed( LLM.Service, LLM.Service.of({ @@ -451,10 +496,92 @@ const boot = Effect.fn("test.boot")(function* () { return { processors, session, provider } }) +const processTypedToolFailure = Effect.fn("test.processTypedToolFailure")(function* (dir: string) { + const { processors, session, provider } = yield* boot() + const chat = yield* session.create({}) + const parent = yield* user(chat.id, "run the tool") + const msg = yield* assistant(chat.id, parent.id, path.resolve(dir)) + const mdl = yield* provider.getModel(ref.providerID, ref.modelID) + const handle = yield* processors.create({ assistantMessage: msg, sessionID: chat.id, model: mdl }) + const decision = yield* handle.process({ + user: { + id: parent.id, + sessionID: chat.id, + role: "user", + time: parent.time, + agent: parent.agent, + model: { providerID: ref.providerID, modelID: ref.modelID }, + } satisfies SessionV1.User, + sessionID: chat.id, + model: mdl, + agent: agent(), + system: [], + messages: [{ role: "user", content: "run the tool" }], + tools: {}, + }) + const part = (yield* MessageV2.parts(msg.id)).find( + (candidate): candidate is SessionV1.ToolPart => candidate.type === "tool", + ) + return { decision, part } +}) + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- +itQuestionRejected.live("question rejection returns a typed terminal decision and durable failure code", () => + provideTmpdirInstance( + (dir) => + Effect.gen(function* () { + const result = yield* processTypedToolFailure(dir) + + expect(result.decision).toEqual({ + action: "stop", + reason: { code: "user_rejected_question", callID: "question-call" }, + }) + expect(result.part?.state.status).toBe("error") + if (result.part?.state.status === "error") + expect(result.part.state.metadata?.failureCode).toBe("user_rejected_question") + }), + { config: cfg }, + ), +) + +itPermissionRejected.live("plain permission rejection returns a typed terminal decision and durable failure code", () => + provideTmpdirInstance( + (dir) => + Effect.gen(function* () { + const result = yield* processTypedToolFailure(dir) + + expect(result.decision).toEqual({ + action: "stop", + reason: { code: "user_rejected_permission", callID: "permission-call" }, + }) + expect(result.part?.state.status).toBe("error") + if (result.part?.state.status === "error") + expect(result.part.state.metadata?.failureCode).toBe("user_rejected_permission") + }), + { config: cfg }, + ), +) + +itPermissionCorrected.live("permission correction remains a non-terminal tool failure", () => + provideTmpdirInstance( + (dir) => + Effect.gen(function* () { + const result = yield* processTypedToolFailure(dir) + + expect(result.decision).toEqual({ action: "continue" }) + expect(result.part?.state.status).toBe("error") + if (result.part?.state.status === "error") { + expect(result.part.state.error).toContain("Use the approved directory") + expect(result.part.state.metadata?.failureCode).toBeUndefined() + } + }), + { config: cfg }, + ), +) + itPlanProtocol.live("session.processor persists activity-scoped plan violations and stops without retry", () => provideTmpdirInstance( (dir) => @@ -500,12 +627,12 @@ itPlanProtocol.live("session.processor persists activity-scoped plan violations (part): part is SessionV1.ToolPart => part.type === "tool" && part.tool === "plan", ) - expect(first.result).toBe("continue") + expect(first.result).toEqual({ action: "continue" }) expect(first.handle.message.error).toBeUndefined() expect(firstPlan?.state.status).toBe("completed") if (firstPlan?.state.status === "completed") expect(firstPlan.state.metadata.plan_attempt_ordinal).toBe(1) - expect(second.result).toBe("stop") + expect(second.result.action).toBe("stop") expect(second.handle.message.finish).toBe("error") expect(second.handle.message.error).toMatchObject({ name: "PlanProtocolViolation", @@ -567,9 +694,9 @@ itOrphanPlanProtocol.live("schema failure before durable tool-call consumes the tracker = new PlanProtocolTracker(restorePlanProtocolFailures(yield* MessageV2.stream(chat.id))) expect(restorePlanProtocolFailures(yield* MessageV2.stream(chat.id))).toBe(1) const second = yield* run() - expect(first.result).toBe("continue") + expect(first.result).toEqual({ action: "continue" }) expect(first.handle.message.error).toBeUndefined() - expect(second.result).toBe("stop") + expect(second.result.action).toBe("stop") expect(second.handle.message.finish).toBe("error") expect(second.handle.message.error).toMatchObject({ name: "PlanProtocolViolation", @@ -636,7 +763,7 @@ it.live("session.processor effect tests capture llm input cleanly", () => const parts = yield* MessageV2.parts(msg.id) const calls = yield* llm.calls - expect(value).toBe("continue") + expect(value).toEqual({ action: "continue" }) expect(calls).toBe(1) expect(transitions).toEqual(["dispatching", "streaming", "settled"]) expect(parts.some((part) => part.type === "text" && part.text === "hello")).toBe(true) @@ -769,7 +896,7 @@ it.live("session.processor effect tests stop after token overflow requests compa }) const parts = yield* MessageV2.parts(msg.id) - expect(value).toBe("compact") + expect(value).toEqual({ action: "compact" }) expect(parts.some((part) => part.type === "text" && part.text === "after")).toBe(true) expect(parts.some((part) => part.type === "step-finish")).toBe(true) }), @@ -817,7 +944,7 @@ it.live("session.processor effect tests capture reasoning from http mock", () => const reasoning = parts.find((part): part is SessionV1.ReasoningPart => part.type === "reasoning") const text = parts.find((part): part is SessionV1.TextPart => part.type === "text") - expect(value).toBe("continue") + expect(value).toEqual({ action: "continue" }) expect(yield* llm.calls).toBe(1) expect(reasoning?.text).toBe("think") expect(text?.text).toBe("done") @@ -864,7 +991,7 @@ it.live("session.processor effect tests reset reasoning state across retries", ( const parts = yield* MessageV2.parts(msg.id) const reasoning = parts.filter((part): part is SessionV1.ReasoningPart => part.type === "reasoning") - expect(value).toBe("continue") + expect(value).toEqual({ action: "continue" }) expect(yield* llm.calls).toBe(2) expect(reasoning.some((part) => part.text === "two")).toBe(true) expect(reasoning.some((part) => part.text === "onetwo")).toBe(false) @@ -928,7 +1055,7 @@ it.live("session.processor effect tests never retry a durable provider attempt", }, ) - expect(value).toBe("stop") + expect(value.action).toBe("stop") expect(yield* llm.calls).toBe(1) expect(transitions).toEqual(["dispatching", "failed"]) }), @@ -971,7 +1098,7 @@ it.live("session.processor effect tests do not retry unknown json errors", () => tools: {}, }) - expect(value).toBe("stop") + expect(value.action).toBe("stop") expect(yield* llm.calls).toBe(1) expect(handle.message.error?.name).toBe("APIError") }), @@ -1017,7 +1144,7 @@ it.live("session.processor effect tests retry recognized structured json errors" const parts = yield* MessageV2.parts(msg.id) - expect(value).toBe("continue") + expect(value).toEqual({ action: "continue" }) expect(yield* llm.calls).toBe(2) expect(parts.some((part) => part.type === "text" && part.text === "after")).toBe(true) expect(handle.message.error).toBeUndefined() @@ -1072,7 +1199,7 @@ it.live("session.processor effect tests publish retry status updates", () => yield* off - expect(value).toBe("continue") + expect(value).toEqual({ action: "continue" }) expect(yield* llm.calls).toBe(2) expect(states).toStrictEqual([1]) }), @@ -1115,7 +1242,7 @@ it.live("session.processor effect tests compact on structured context overflow", tools: {}, }) - expect(value).toBe("compact") + expect(value).toEqual({ action: "compact" }) expect(yield* llm.calls).toBe(1) expect(handle.message.error).toBeUndefined() }), @@ -1171,7 +1298,7 @@ it.live("session.processor effect tests complete AI SDK tool calls when native f const parts = yield* MessageV2.parts(msg.id) const call = parts.find((part): part is SessionV1.ToolPart => part.type === "tool") - expect(value).toBe("continue") + expect(value).toEqual({ action: "continue" }) expect(yield* llm.calls).toBe(1) expect(call?.callID).toBe("call_1") expect(call?.tool).toBe("lookup") @@ -1482,7 +1609,7 @@ itFragmentFailure.live("session.processor effect tests flush partial v2 fragment messages: [{ role: "user", content: "provider failure" }], tools: {}, }), - ).toBe("stop") + ).toMatchObject({ action: "stop", reason: { code: "assistant_error" } }) yield* off const failed = seen.indexOf(SessionEvent.Step.Failed.type) @@ -1544,7 +1671,7 @@ itIncidentOriginal.live("§9.3 BUG-010: original incident payloads (no operation const first = yield* run() // Dispatch 1: first malformed plan → correctable error, session continues - expect(first.result).toBe("continue") + expect(first.result).toEqual({ action: "continue" }) expect(first.handle.message.error).toBeUndefined() const firstPlan = first.parts.find( (part): part is SessionV1.ToolPart => part.type === "tool" && part.tool === "plan", @@ -1558,7 +1685,7 @@ itIncidentOriginal.live("§9.3 BUG-010: original incident payloads (no operation const second = yield* run() // Dispatch 2: second consecutive malformed plan → PlanProtocolViolation, stops - expect(second.result).toBe("stop") + expect(second.result.action).toBe("stop") expect(second.handle.message.finish).toBe("error") expect(second.handle.message.error).toMatchObject({ name: "PlanProtocolViolation", @@ -1614,11 +1741,11 @@ itIncidentForwardCompat.live("§9.3 BUG-010: forward-compatible incident payload }) const first = yield* run() - expect(first.result).toBe("continue") + expect(first.result).toEqual({ action: "continue" }) expect(first.handle.message.error).toBeUndefined() const second = yield* run() - expect(second.result).toBe("stop") + expect(second.result.action).toBe("stop") expect(second.handle.message.finish).toBe("error") expect(second.handle.message.error).toMatchObject({ name: "PlanProtocolViolation" }) diff --git a/packages/deepagent-code/test/session/prompt-intent.test.ts b/packages/deepagent-code/test/session/prompt-intent.test.ts index 13c9e3e5c..982fdb276 100644 --- a/packages/deepagent-code/test/session/prompt-intent.test.ts +++ b/packages/deepagent-code/test/session/prompt-intent.test.ts @@ -10,11 +10,12 @@ import { PartTable, SessionInputTable, SessionIntentTable, + SessionSteerTable, SessionTable, } from "@deepagent-code/core/session/sql" import { SessionV1 } from "@deepagent-code/core/v1/session" import { eq } from "drizzle-orm" -import { Effect } from "effect" +import { Effect, Exit } from "effect" import { MessageV2 } from "../../src/session/message-v2" import { SessionMutationEpoch } from "../../src/session/mutation-epoch" import { SessionPromptIntent } from "../../src/session/prompt-intent" @@ -22,7 +23,9 @@ import { SessionActivityAdmissionTable, SessionActivityProgressTable, SessionLegacyActivityAdmissionTable, + SessionLegacyActivityRunTable, SessionLegacyActivityTable, + SessionLegacyActivityTerminalTable, } from "../../src/session/activity-sql" import { MessageID, PartID, SessionID } from "../../src/session/schema" import { SessionToolRequestReceiptTable } from "../../src/session/tool-request-receipt.sql" @@ -61,6 +64,7 @@ const claim = (input: { variant?: SessionPromptIntent.Variant payloadHash?: string source?: SessionPromptIntent.Source + executionMode?: SessionPromptIntent.ExecutionMode }) => SessionPromptIntent.claim({ intentID: input.intentID, @@ -69,6 +73,7 @@ const claim = (input: { variant: input.variant ?? "original", payloadHash: input.payloadHash ?? "payload-a", messageID: input.messageID, + executionMode: input.executionMode, }) const message = (messageID: MessageID): { info: SessionV1.User; parts: SessionV1.Part[] } => ({ @@ -207,6 +212,358 @@ describe("SessionPromptIntent", () => { }), ) + it.effect("deferred admission creates no Activity until an explicit resume claims exactly one run", () => + Effect.gen(function* () { + yield* setup + const deferred = yield* claim({ + intentID: "intent_deferred_resume", + messageID: MessageID.make("msg_deferred_resume"), + executionMode: "deferred", + }) + expect(deferred.kind).toBe("claimed") + if (deferred.kind !== "claimed") return + const materialized = yield* SessionPromptIntent.materializeTurn({ + receipt: deferred.receipt, + message: message(deferred.receipt.messageID), + executionMode: "deferred", + }) + expect(materialized.executionMode).toBe("deferred") + expect("run" in materialized).toBeFalse() + + const { db } = yield* Database.Service + expect(yield* db.select().from(SessionLegacyActivityTable).all().pipe(Effect.orDie)).toHaveLength(0) + expect(yield* db.select().from(SessionLegacyActivityRunTable).all().pipe(Effect.orDie)).toHaveLength(0) + expect( + yield* db + .select() + .from(SessionIntentTable) + .where(eq(SessionIntentTable.intent_id, deferred.receipt.intentID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ execution_mode: "deferred", execution_state: "pending", execution_claim_id: null }) + + const run = yield* SessionPromptIntent.claimDeferredActivity({ + sessionID, + messageID: deferred.receipt.messageID, + }) + expect(run).toBeDefined() + if (!run) return + expect(yield* db.select().from(SessionLegacyActivityTable).all().pipe(Effect.orDie)).toHaveLength(1) + expect(yield* db.select().from(SessionLegacyActivityRunTable).all().pipe(Effect.orDie)).toMatchObject([ + { run_id: run.runID, activity_id: run.activityID, state: "running" }, + ]) + expect(yield* db.select().from(SessionLegacyActivityAdmissionTable).all().pipe(Effect.orDie)).toMatchObject([ + { activity_id: run.activityID, ordinal: 0, role: "trigger" }, + ]) + expect( + yield* db + .select() + .from(SessionIntentTable) + .where(eq(SessionIntentTable.intent_id, deferred.receipt.intentID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ execution_state: "claimed", execution_claim_id: run.runID }) + expect( + yield* SessionPromptIntent.claimDeferredActivity({ sessionID, messageID: deferred.receipt.messageID }), + ).toBeUndefined() + }), + ) + + it.effect("a normal run absorbs deferred context and freezes its membership ordinal into progress", () => + Effect.gen(function* () { + yield* setup + const deferred = yield* claim({ + intentID: "intent_deferred_absorbed", + messageID: MessageID.make("msg_deferred_absorbed"), + executionMode: "deferred", + }) + if (deferred.kind !== "claimed") return + yield* SessionPromptIntent.materializeTurn({ + receipt: deferred.receipt, + message: message(deferred.receipt.messageID), + executionMode: "deferred", + }) + + const trigger = yield* claim({ + intentID: "intent_run_absorbs_deferred", + messageID: MessageID.make("msg_run_absorbs_deferred"), + executionMode: "run_now", + }) + if (trigger.kind !== "claimed") return + const materialized = yield* SessionPromptIntent.materializeTurn({ + receipt: trigger.receipt, + message: message(trigger.receipt.messageID), + executionMode: "run_now", + }) + expect("run" in materialized).toBeTrue() + if (!("run" in materialized)) return + + const { db } = yield* Database.Service + expect(yield* db.select().from(SessionLegacyActivityAdmissionTable).all().pipe(Effect.orDie)).toMatchObject([ + { activity_id: materialized.run.activityID, ordinal: 0, role: "trigger" }, + { activity_id: materialized.run.activityID, ordinal: 1, role: "deferred_context" }, + ]) + expect( + yield* db + .select() + .from(SessionIntentTable) + .where(eq(SessionIntentTable.intent_id, deferred.receipt.intentID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ execution_state: "absorbed", execution_claim_id: materialized.run.runID }) + + const boundary = yield* SessionPromptIntent.freezeProviderInputBoundary(materialized.run) + expect(boundary).toMatchObject({ kind: "ready", boundary: { membershipOrdinal: 1 } }) + if (boundary.kind !== "ready") return + const assistantID = MessageID.make("msg_absorbed_progress_assistant") + yield* db + .insert(MessageTable) + .values({ + id: assistantID, + session_id: sessionID, + time_created: 2, + data: { + role: "assistant", + parentID: trigger.receipt.messageID, + mode: "build", + agent: "build", + path: { cwd: "/project", root: "/project" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + modelID: ModelV2.ID.make("test"), + providerID: ProviderV2.ID.make("test"), + time: { created: 2 }, + } as typeof MessageTable.$inferInsert.data, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: "receipt-absorbed-boundary", + request_ordinal: 1, + session_id: sessionID, + user_message_id: trigger.receipt.messageID, + assistant_message_id: assistantID, + provider_id: "test", + model_id: "test", + registry_tool_ids: [], + permission_filtered_tool_ids: [], + final_offered_tool_ids: [], + call_ids: [], + provider_state: "preparing", + request_state: "prepared", + created_at: 2, + }) + .run() + .pipe(Effect.orDie) + yield* SessionPromptIntent.beginProgress({ + activityID: materialized.run.activityID, + assistantMessageID: assistantID, + providerReceiptID: "receipt-absorbed-boundary", + membershipOrdinal: boundary.boundary.membershipOrdinal, + }) + expect( + yield* db + .select() + .from(SessionActivityProgressTable) + .where(eq(SessionActivityProgressTable.assistant_message_id, assistantID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ input_membership_ordinal: 1, state: "provisional" }) + }), + ) + + it.effect("typed question rejection atomically terminalizes a progress revision and exact-replays", () => + Effect.gen(function* () { + yield* setup + const trigger = yield* claim({ + intentID: "intent_question_rejected", + messageID: MessageID.make("msg_question_rejected_user"), + }) + if (trigger.kind !== "claimed") return + const materialized = yield* SessionPromptIntent.materializeTurn({ + receipt: trigger.receipt, + message: message(trigger.receipt.messageID), + }) + if (!("run" in materialized)) return + const boundary = yield* SessionPromptIntent.freezeProviderInputBoundary(materialized.run) + if (boundary.kind !== "ready") return + + const { db } = yield* Database.Service + const assistantID = MessageID.make("msg_question_rejected_assistant") + yield* db + .insert(MessageTable) + .values({ + id: assistantID, + session_id: sessionID, + time_created: 2, + data: { + role: "assistant", + parentID: trigger.receipt.messageID, + mode: "build", + agent: "build", + path: { cwd: "/project", root: "/project" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + modelID: ModelV2.ID.make("test"), + providerID: ProviderV2.ID.make("test"), + time: { created: 2, completed: 3 }, + finish: "tool-calls", + } as typeof MessageTable.$inferInsert.data, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: "receipt-question-rejected", + request_ordinal: 1, + session_id: sessionID, + user_message_id: trigger.receipt.messageID, + assistant_message_id: assistantID, + provider_id: "test", + model_id: "test", + registry_tool_ids: ["question"], + permission_filtered_tool_ids: ["question"], + final_offered_tool_ids: ["question"], + call_ids: ["call-question-rejected"], + provider_state: "settled", + terminal_at: 3, + response_fingerprint: "response-question-rejected", + request_state: "dispatched", + created_at: 2, + }) + .run() + .pipe(Effect.orDie) + yield* SessionPromptIntent.beginProgress({ + activityID: materialized.run.activityID, + assistantMessageID: assistantID, + providerReceiptID: "receipt-question-rejected", + membershipOrdinal: boundary.boundary.membershipOrdinal, + }) + const decision = { + state: "interrupted" as const, + reasonCode: "user_rejected_question", + source: "host_stop" as const, + operationID: `${materialized.run.runID}:terminal`, + ownerToken: materialized.run.ownerToken, + } + + expect( + yield* SessionPromptIntent.finalizeActivityWithRevision({ + run: materialized.run, + assistantMessageID: assistantID, + decision, + }), + ).toMatchObject({ kind: "terminal_committed" }) + expect( + yield* db + .select() + .from(SessionActivityProgressTable) + .where(eq(SessionActivityProgressTable.assistant_message_id, assistantID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ state: "progress", input_membership_ordinal: 0 }) + expect( + yield* db + .select() + .from(SessionLegacyActivityTable) + .where(eq(SessionLegacyActivityTable.activity_id, materialized.run.activityID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ state: "interrupted", terminal_reason: "user_rejected_question" }) + expect( + yield* db + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, materialized.run.runID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ state: "interrupted", terminal_reason: "user_rejected_question" }) + expect(yield* db.select().from(SessionLegacyActivityTerminalTable).all().pipe(Effect.orDie)).toMatchObject([ + { + activity_id: materialized.run.activityID, + run_id: materialized.run.runID, + state: "interrupted", + reason_code: "user_rejected_question", + membership_ordinal: 0, + }, + ]) + expect( + yield* SessionPromptIntent.finalizeActivityWithRevision({ + run: materialized.run, + assistantMessageID: assistantID, + decision, + }), + ).toMatchObject({ kind: "exact_replay" }) + const divergent = yield* SessionPromptIntent.finalizeActivityWithRevision({ + run: materialized.run, + assistantMessageID: assistantID, + decision: { ...decision, reasonCode: "different_reason" }, + }).pipe(Effect.exit) + expect(Exit.isFailure(divergent)).toBeTrue() + }), + ) + + it.effect("host-only terminal exact-replays and rejects divergent ordinal or payload", () => + Effect.gen(function* () { + yield* setup + const trigger = yield* claim({ + intentID: "intent_host_only_terminal", + messageID: MessageID.make("msg_host_only_terminal"), + }) + if (trigger.kind !== "claimed") return + const materialized = yield* SessionPromptIntent.materializeTurn({ + receipt: trigger.receipt, + message: message(trigger.receipt.messageID), + }) + if (!("run" in materialized) || !materialized.run) return + const boundary = yield* SessionPromptIntent.freezeProviderInputBoundary(materialized.run) + if (boundary.kind !== "ready") return + const decision = { + state: "failed" as const, + reasonCode: "structured_finalizer_unsupported", + source: "host_stop" as const, + operationID: `${materialized.run.runID}:terminal`, + ownerToken: materialized.run.ownerToken, + } + + expect( + yield* SessionPromptIntent.finalizeActivityWithoutRevision({ + run: materialized.run, + membershipOrdinal: boundary.boundary.membershipOrdinal, + decision, + }), + ).toMatchObject({ kind: "terminal_committed" }) + expect( + yield* SessionPromptIntent.finalizeActivityWithoutRevision({ + run: materialized.run, + membershipOrdinal: boundary.boundary.membershipOrdinal, + decision, + }), + ).toMatchObject({ kind: "exact_replay" }) + expect( + Exit.isFailure( + yield* SessionPromptIntent.finalizeActivityWithoutRevision({ + run: materialized.run, + membershipOrdinal: boundary.boundary.membershipOrdinal + 1, + decision, + }).pipe(Effect.exit), + ), + ).toBeTrue() + expect( + Exit.isFailure( + yield* SessionPromptIntent.finalizeActivityWithoutRevision({ + run: materialized.run, + membershipOrdinal: boundary.boundary.membershipOrdinal, + decision: { ...decision, reasonCode: "different_reason" }, + }).pipe(Effect.exit), + ), + ).toBeTrue() + }), + ) + it.effect("terminal provider receipts settle provisional progress deterministically after restart", () => Effect.gen(function* () { yield* setup @@ -318,7 +675,22 @@ describe("SessionPromptIntent", () => { .where(eq(SessionLegacyActivityTable.activity_id, activity.activityID)) .get() .pipe(Effect.orDie), - ).toMatchObject({ state: "settled", terminal_reason: "stop" }) + ).toMatchObject({ state: "recovery_required", terminal_reason: "host_terminal_decision_missing" }) + expect(yield* db.select().from(SessionLegacyActivityRunTable).all().pipe(Effect.orDie)).toMatchObject([ + { state: "recovery_required", terminal_reason: "host_terminal_decision_missing" }, + ]) + expect(yield* db.select().from(SessionLegacyActivityTerminalTable).all().pipe(Effect.orDie)).toMatchObject([ + { + activity_id: activity.activityID, + state: "recovery_required", + reason_code: "host_terminal_decision_missing", + source: "restart_recovery", + assistant_message_id: assistantID, + progress_revision: 0, + membership_ordinal: 0, + }, + ]) + expect(yield* SessionPromptIntent.recoverActiveActivities("next-process-owner")).toEqual([]) expect( yield* db .select({ data: PartTable.data }) @@ -457,8 +829,22 @@ describe("SessionPromptIntent", () => { .pipe(Effect.orDie), ).toMatchObject({ state: "recovery_required", - terminal_reason: "process restarted after settled activity progress", + terminal_reason: "host_terminal_decision_missing", }) + expect(yield* db.select().from(SessionLegacyActivityRunTable).all().pipe(Effect.orDie)).toMatchObject([ + { state: "recovery_required", terminal_reason: "host_terminal_decision_missing" }, + ]) + expect(yield* db.select().from(SessionLegacyActivityTerminalTable).all().pipe(Effect.orDie)).toMatchObject([ + { + activity_id: activity.activityID, + state: "recovery_required", + reason_code: "host_terminal_decision_missing", + source: "restart_recovery", + assistant_message_id: assistantID, + progress_revision: 0, + membership_ordinal: 0, + }, + ]) }), ) diff --git a/packages/deepagent-code/test/session/prompt.test.ts b/packages/deepagent-code/test/session/prompt.test.ts index 2ff6f4258..6820a7376 100644 --- a/packages/deepagent-code/test/session/prompt.test.ts +++ b/packages/deepagent-code/test/session/prompt.test.ts @@ -47,11 +47,13 @@ import { providerResponseFingerprint, recoverProviderReceiptsOnStartup, SessionP import { SessionRevert } from "../../src/session/revert" import { SessionRunState } from "../../src/session/run-state" import { SessionSteer } from "../../src/session/steer" +import { SessionPromptIntent } from "../../src/session/prompt-intent" import { MessageID, PartID, SessionID } from "../../src/session/schema" import { SessionStatus } from "../../src/session/status" import { SessionV2 } from "@deepagent-code/core/session" import { SessionExecution } from "@deepagent-code/core/session/execution" import { SessionMessage } from "@deepagent-code/core/session/message" +import { Prompt } from "@deepagent-code/core/session/prompt" import { Skill } from "../../src/skill" import { SystemPrompt } from "../../src/session/system" import { Shell } from "../../src/shell/shell" @@ -90,7 +92,10 @@ import { CompactionArtifactTable, CompactionRunTable } from "@/session/compactio import { SessionActivityAdmissionTable, SessionActivityProgressTable, + SessionLegacyActivityAdmissionTable, + SessionLegacyActivityRunTable, SessionLegacyActivityTable, + SessionLegacyActivityTerminalTable, } from "@/session/activity-sql" void Log.init({ print: false }) @@ -917,6 +922,9 @@ it.instance("loop calls LLM and returns assistant message", () => noReply: true, parts: [{ type: "text", text: "hello" }], }) + const { db } = yield* Database.Service + expect(yield* db.select().from(SessionLegacyActivityTable).all().pipe(Effect.orDie)).toHaveLength(0) + expect(yield* db.select().from(SessionLegacyActivityRunTable).all().pipe(Effect.orDie)).toHaveLength(0) yield* llm.text("world") const result = yield* prompt.loop({ sessionID: chat.id }) @@ -924,6 +932,165 @@ it.instance("loop calls LLM and returns assistant message", () => const parts = result.parts.filter((p) => p.type === "text") expect(parts.some((p) => p.type === "text" && p.text === "world")).toBe(true) expect(yield* llm.hits).toHaveLength(1) + expect(yield* db.select().from(SessionLegacyActivityTable).all().pipe(Effect.orDie)).toHaveLength(1) + expect(yield* db.select().from(SessionLegacyActivityRunTable).all().pipe(Effect.orDie)).toHaveLength(1) + }), +) + +it.instance("a new prompt reconciles a same-process pre-dispatch owner loss before admission", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const steer = yield* SessionSteer.Service + const sessions = yield* Session.Service + const { db } = yield* Database.Service + const chat = yield* sessions.create({ + title: "Same-process owner loss", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + const deferred = yield* prompt.prompt({ + sessionID: chat.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "orphaned before provider dispatch" }], + }) + const orphanedRun = yield* SessionPromptIntent.claimDeferredActivity({ + sessionID: chat.id, + messageID: deferred.info.id, + }) + expect(orphanedRun).toBeDefined() + const attachedSteer = yield* awaitWithTimeout( + steer.admit({ + sessionID: chat.id, + prompt: new Prompt({ text: "attached to the orphaned run" }), + }), + "timed out attaching steer to orphaned run", + "2 seconds", + ) + expect(yield* llm.hits).toHaveLength(0) + + yield* awaitWithTimeout(llm.text("recovered next prompt"), "timed out queuing recovery response", "2 seconds") + const next = yield* awaitWithTimeout( + prompt.prompt({ + sessionID: chat.id, + agent: "build", + parts: [{ type: "text", text: "start after owner loss" }], + }), + "timed out reconciling same-process owner loss", + "3 seconds", + ) + expect(next.parts.some((part) => part.type === "text" && part.text === "recovered next prompt")).toBeTrue() + expect(yield* llm.hits).toHaveLength(1) + expect( + yield* db + .select() + .from(SessionSteerTable) + .where(eq(SessionSteerTable.id, attachedSteer.id)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ consumed_seq: null, superseded_at: expect.any(Number) }) + expect( + yield* db + .select() + .from(SessionIntentTable) + .where(eq(SessionIntentTable.admitted_message_id, attachedSteer.id)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ execution_mode: "run_now", execution_state: "canceled" }) + + expect( + yield* db + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.session_id, chat.id)) + .orderBy(SessionLegacyActivityRunTable.generation) + .all() + .pipe(Effect.orDie), + ).toMatchObject([ + { run_id: orphanedRun?.runID, state: "failed", terminal_reason: "pre_dispatch_owner_lost" }, + { state: "completed", terminal_reason: "assistant_completed" }, + ]) + expect( + yield* db + .select() + .from(SessionLegacyActivityTerminalTable) + .where(eq(SessionLegacyActivityTerminalTable.session_id, chat.id)) + .orderBy(SessionLegacyActivityTerminalTable.created_at) + .all() + .pipe(Effect.orDie), + ).toMatchObject([ + { + run_id: orphanedRun?.runID, + state: "failed", + reason_code: "pre_dispatch_owner_lost", + source: "same_process_recovery", + }, + { state: "settled", reason_code: "assistant_completed", source: "provider_final" }, + ]) + }), + 15_000, +) + +it.instance("question rejection terminalizes the live run and permits the next prompt", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const question = yield* Question.Service + const { db } = yield* Database.Service + const chat = yield* sessions.create({ + title: "Question rejection lifecycle", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + yield* llm.tool("question", { + questions: [ + { + question: "Continue with this approach?", + header: "Continue", + options: [{ label: "Yes", description: "Continue" }], + }, + ], + }) + yield* llm.text("next prompt succeeded") + + const first = yield* prompt + .prompt({ + sessionID: chat.id, + agent: "build", + parts: [{ type: "text", text: "ask before proceeding" }], + }) + .pipe(Effect.forkChild) + const request = yield* pollWithTimeout( + Effect.gen(function* () { + const pending = yield* question.list() + return pending.find((item) => item.sessionID === chat.id) + }), + "timed out waiting for question request", + ) + yield* question.reject(request.id) + const firstResult = yield* Fiber.join(first) + expect(firstResult.info.role).toBe("assistant") + + expect(yield* db.select().from(SessionActivityProgressTable).all().pipe(Effect.orDie)).toMatchObject([ + { state: "progress" }, + ]) + expect(yield* db.select().from(SessionLegacyActivityTable).all().pipe(Effect.orDie)).toMatchObject([ + { state: "interrupted", terminal_reason: "user_rejected_question" }, + ]) + expect(yield* db.select().from(SessionLegacyActivityRunTable).all().pipe(Effect.orDie)).toMatchObject([ + { state: "interrupted", terminal_reason: "user_rejected_question" }, + ]) + expect(yield* db.select().from(SessionLegacyActivityTerminalTable).all().pipe(Effect.orDie)).toMatchObject([ + { state: "interrupted", reason_code: "user_rejected_question", source: "host_stop" }, + ]) + + const next = yield* prompt.prompt({ + sessionID: chat.id, + agent: "build", + parts: [{ type: "text", text: "continue now" }], + }) + expect(next.parts.some((part) => part.type === "text" && part.text === "next prompt succeeded")).toBeTrue() + expect(yield* llm.hits).toHaveLength(2) }), ) @@ -1550,6 +1717,7 @@ worldStateCompaction.instance( const steerBuffer = yield* SessionSteer.Service const compaction = yield* SessionCompaction.Service const epochs = yield* PromptEpoch.Service + const { db } = yield* Database.Service const chat = yield* sessions.create({ title: "Compaction steer ordering" }) const seeded = yield* seed(chat.id, { finish: "stop" }) seeded.assistant.tokens.input = 95_000 @@ -1612,6 +1780,153 @@ worldStateCompaction.instance( return value.role === "user" && value.content === "CONCURRENT_STEER_EXACTLY_ONCE" }) expect(userMessages).toHaveLength(1) + const activity = yield* db + .select() + .from(SessionLegacyActivityTable) + .where(eq(SessionLegacyActivityTable.session_id, chat.id)) + .get() + .pipe(Effect.orDie) + expect(activity).toMatchObject({ state: "settled", terminal_reason: "assistant_completed" }) + if (!activity) return + expect( + yield* db + .select() + .from(SessionLegacyActivityAdmissionTable) + .where(eq(SessionLegacyActivityAdmissionTable.activity_id, activity.activity_id)) + .all() + .pipe(Effect.orDie), + ).toMatchObject([{ ordinal: 0, role: "trigger" }]) + const run = yield* db + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.activity_id, activity.activity_id)) + .get() + .pipe(Effect.orDie) + expect(run).toMatchObject({ state: "completed", terminal_reason: "assistant_completed" }) + expect( + yield* db + .select() + .from(SessionLegacyActivityTerminalTable) + .where(eq(SessionLegacyActivityTerminalTable.activity_id, activity.activity_id)) + .all() + .pipe(Effect.orDie), + ).toMatchObject([ + { + run_id: run?.run_id, + state: "settled", + reason_code: "assistant_completed", + source: "provider_final", + membership_ordinal: 0, + }, + ]) + }), + { git: true }, + 30_000, +) + +worldStateCompaction.instance( + "cancel after compaction terminalizes a dynamically claimed steer run before provider dispatch", + () => + Effect.gen(function* () { + const { dir, llm } = yield* useServerConfig(providerCfg) + const gate = yield* Deferred.make() + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const steer = yield* SessionSteer.Service + const compaction = yield* SessionCompaction.Service + const { db } = yield* Database.Service + const chat = yield* sessions.create({ title: "Compaction dynamic run cancel" }) + const seeded = yield* seed(chat.id, { finish: "stop" }) + seeded.assistant.tokens.input = 95_000 + seeded.assistant.tokens.total = 95_000 + yield* sessions.updateMessage(seeded.assistant) + yield* user(chat.id, "Start compaction, then cancel the queued steer run.") + yield* llm.hold("## Progress\n- durable summary", deferredAsPromise(gate)) + + const running = yield* prompt.loop({ sessionID: chat.id }).pipe(Effect.forkChild) + yield* llm.wait(1) + const admitted = yield* steer.admit({ + sessionID: chat.id, + prompt: new Prompt({ text: "CANCEL-DYNAMIC-RUN-BEFORE-DISPATCH" }), + }) + expect(yield* compaction.hasPending(chat.id)).toBe(true) + const marker = path.join(dir, "activity-admit-and-bind.json") + const previousPoint = process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT + const previousRoot = process.env.DEEPAGENT_CODE_TEST_ROOT + const previousMarker = process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER + + yield* Effect.acquireUseRelease( + Effect.sync(() => { + process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT = "after_admit_and_bind" + process.env.DEEPAGENT_CODE_TEST_ROOT = dir + process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER = marker + }), + () => + Effect.gen(function* () { + yield* Deferred.succeed(gate, void 0) + yield* pollWithTimeout( + Effect.promise(async () => ((await Bun.file(marker).exists()) ? true : undefined)), + "timed out waiting for the dynamically claimed steer run", + ) + expect(yield* llm.calls).toBe(1) + yield* awaitWithTimeout(prompt.cancel(chat.id), "timed out cancelling the dynamic steer run", "5 seconds") + yield* awaitWithTimeout(Fiber.await(running), "timed out joining the canceled dynamic steer run", "5 seconds") + }), + () => + Effect.sync(() => { + if (previousPoint === undefined) delete process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT + else process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT = previousPoint + if (previousRoot === undefined) delete process.env.DEEPAGENT_CODE_TEST_ROOT + else process.env.DEEPAGENT_CODE_TEST_ROOT = previousRoot + if (previousMarker === undefined) delete process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER + else process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER = previousMarker + }), + ) + + expect(yield* llm.calls).toBe(1) + expect( + yield* db + .select() + .from(SessionLegacyActivityTable) + .where(eq(SessionLegacyActivityTable.session_id, chat.id)) + .all() + .pipe(Effect.orDie), + ).toMatchObject([{ state: "interrupted", terminal_reason: "user_cancelled" }]) + const runs = yield* db + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.session_id, chat.id)) + .all() + .pipe(Effect.orDie) + expect(runs).toMatchObject([{ state: "interrupted", terminal_reason: "user_cancelled" }]) + expect( + yield* db + .select() + .from(SessionLegacyActivityTerminalTable) + .where(eq(SessionLegacyActivityTerminalTable.session_id, chat.id)) + .all() + .pipe(Effect.orDie), + ).toMatchObject([{ state: "interrupted", reason_code: "user_cancelled", source: "cancel" }]) + expect( + yield* db + .select() + .from(SessionIntentTable) + .where(eq(SessionIntentTable.admitted_message_id, admitted.id)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ + execution_mode: "run_now", + execution_state: "claimed", + execution_claim_id: runs[0]?.run_id, + }) + expect( + yield* db + .select() + .from(SessionSteerTable) + .where(eq(SessionSteerTable.id, admitted.id)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ consumed_seq: null, superseded_at: expect.any(Number) }) }), { git: true }, 30_000, @@ -2158,7 +2473,7 @@ it.instance("synthetic output continuation preserves the durable legacy activity .pipe(Effect.orDie), ).toEqual([expect.objectContaining({ state: "progress" }), expect.objectContaining({ state: "final" })]) expect(yield* db.select().from(SessionLegacyActivityTable).all().pipe(Effect.orDie)).toMatchObject([ - { state: "settled", terminal_reason: "stop" }, + { state: "settled", terminal_reason: "assistant_completed" }, ]) }), ) @@ -3555,9 +3870,7 @@ noLLMServer.instance( .pipe(Effect.orDie), ).toHaveLength(1) expect(yield* db.select().from(SessionActivityAdmissionTable).all().pipe(Effect.orDie)).toHaveLength(1) - expect(yield* db.select().from(SessionLegacyActivityTable).all().pipe(Effect.orDie)).toMatchObject([ - { state: "active", terminal_reason: null }, - ]) + expect(yield* db.select().from(SessionLegacyActivityTable).all().pipe(Effect.orDie)).toHaveLength(0) }), { config: cfg }, ) @@ -3946,6 +4259,115 @@ it.instance( 15_000, ) +it.instance( + "cancel terminalizes the durable run, supersedes attached steer, and permits the next prompt", + () => + Effect.gen(function* () { + const { dir, llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const steer = yield* SessionSteer.Service + const sessions = yield* Session.Service + const { db } = yield* Database.Service + const session = yield* sessions.create({ title: "Durable cancel lifecycle" }) + const marker = path.join(dir, "normal-prompt-admit-and-bind.json") + const previousPoint = process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT + const previousRoot = process.env.DEEPAGENT_CODE_TEST_ROOT + const previousMarker = process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER + const steered = yield* Effect.acquireUseRelease( + Effect.sync(() => { + process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT = "after_admit_and_bind" + process.env.DEEPAGENT_CODE_TEST_ROOT = dir + process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER = marker + }), + () => + Effect.gen(function* () { + const running = yield* prompt + .prompt({ + sessionID: session.id, + agent: "build", + parts: [{ type: "text", text: "Cancel this run" }], + }) + .pipe(Effect.forkChild) + yield* pollWithTimeout( + Effect.promise(async () => ((await Bun.file(marker).exists()) ? true : undefined)), + "timed out waiting for normal prompt admit-and-bind", + ) + expect(yield* llm.calls).toBe(0) + const admitted = yield* steer.admit({ + sessionID: session.id, + prompt: new Prompt({ text: "This steer belongs to the canceled run" }), + }) + expect( + yield* db + .select({ + activityID: SessionLegacyActivityAdmissionTable.activity_id, + executionMode: SessionIntentTable.execution_mode, + executionState: SessionIntentTable.execution_state, + }) + .from(SessionLegacyActivityAdmissionTable) + .innerJoin( + SessionActivityAdmissionTable, + eq(SessionActivityAdmissionTable.admission_id, SessionLegacyActivityAdmissionTable.admission_id), + ) + .innerJoin( + SessionIntentTable, + eq(SessionIntentTable.intent_id, SessionActivityAdmissionTable.legacy_intent_id), + ) + .where(eq(SessionActivityAdmissionTable.admitted_message_id, admitted.id)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ executionMode: "run_now", executionState: "pending" }) + yield* awaitWithTimeout(prompt.cancel(session.id), "timed out cancelling durable run", "5 seconds") + yield* awaitWithTimeout(Fiber.await(running), "timed out joining canceled durable run", "5 seconds") + return admitted + }), + () => + Effect.sync(() => { + if (previousPoint === undefined) delete process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT + else process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT = previousPoint + if (previousRoot === undefined) delete process.env.DEEPAGENT_CODE_TEST_ROOT + else process.env.DEEPAGENT_CODE_TEST_ROOT = previousRoot + if (previousMarker === undefined) delete process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER + else process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER = previousMarker + }), + ) + expect(yield* db.select().from(SessionLegacyActivityTable).all().pipe(Effect.orDie)).toMatchObject([ + { state: "interrupted", terminal_reason: "user_cancelled" }, + ]) + expect(yield* db.select().from(SessionLegacyActivityRunTable).all().pipe(Effect.orDie)).toMatchObject([ + { state: "interrupted", terminal_reason: "user_cancelled" }, + ]) + expect(yield* db.select().from(SessionLegacyActivityTerminalTable).all().pipe(Effect.orDie)).toMatchObject([ + { state: "interrupted", reason_code: "user_cancelled", source: "cancel" }, + ]) + expect(yield* db.select().from(SessionSteerTable).all().pipe(Effect.orDie)).toMatchObject([ + { consumed_seq: null, superseded_at: expect.any(Number) }, + ]) + expect( + yield* db + .select() + .from(SessionIntentTable) + .where(eq(SessionIntentTable.admitted_message_id, steered.id)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ execution_mode: "run_now", execution_state: "canceled" }) + + yield* llm.text("next prompt completed") + const next = yield* awaitWithTimeout( + prompt.prompt({ + sessionID: session.id, + agent: "build", + parts: [{ type: "text", text: "Start a new run" }], + }), + "timed out starting prompt after durable cancel", + "5 seconds", + ) + expect(next.parts.some((part) => part.type === "text" && part.text === "next prompt completed")).toBeTrue() + + }), + 15_000, +) + // Agent variant noLLMServer.instance( diff --git a/packages/deepagent-code/test/session/steer.test.ts b/packages/deepagent-code/test/session/steer.test.ts index 611bfc69a..baad448b6 100644 --- a/packages/deepagent-code/test/session/steer.test.ts +++ b/packages/deepagent-code/test/session/steer.test.ts @@ -67,7 +67,14 @@ import { MessageTable, SessionIntentTable, SessionSteerTable, SessionTable } fro import { eq } from "drizzle-orm" import { SessionMutationEpoch } from "../../src/session/mutation-epoch" import { SessionPromptIntent } from "../../src/session/prompt-intent" -import { SessionActivityAdmissionTable, SessionLegacyActivityAdmissionTable } from "../../src/session/activity-sql" +import { + SessionActivityAdmissionTable, + SessionActivityProgressTable, + SessionLegacyActivityAdmissionTable, + SessionLegacyActivityRunTable, + SessionLegacyActivityTable, + SessionLegacyActivityTerminalTable, +} from "../../src/session/activity-sql" void Log.init({ print: false }) @@ -787,6 +794,66 @@ on.instance( // Two model calls: the original + the follow-up that absorbed the steer. expect(yield* llm.calls).toBe(2) + const { db } = yield* Database.Service + const activities = yield* db + .select() + .from(SessionLegacyActivityTable) + .where(eq(SessionLegacyActivityTable.session_id, chat.id)) + .all() + .pipe(Effect.orDie) + expect(activities).toMatchObject([{ state: "settled", terminal_reason: "assistant_completed" }]) + const activity = activities[0] + expect(activity).toBeDefined() + if (!activity) return + expect( + yield* db + .select() + .from(SessionLegacyActivityAdmissionTable) + .where(eq(SessionLegacyActivityAdmissionTable.activity_id, activity.activity_id)) + .orderBy(SessionLegacyActivityAdmissionTable.ordinal) + .all() + .pipe(Effect.orDie), + ).toMatchObject([ + { ordinal: 0, role: "trigger" }, + { ordinal: 1, role: "steer" }, + ]) + expect( + yield* db + .select() + .from(SessionActivityProgressTable) + .where(eq(SessionActivityProgressTable.activity_id, activity.activity_id)) + .orderBy(SessionActivityProgressTable.revision) + .all() + .pipe(Effect.orDie), + ).toMatchObject([ + { revision: 0, input_membership_ordinal: 0, state: "progress" }, + { revision: 1, input_membership_ordinal: 1, state: "final" }, + ]) + const runs = yield* db + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.session_id, chat.id)) + .all() + .pipe(Effect.orDie) + expect(runs).toMatchObject([ + { activity_id: activity.activity_id, state: "completed", terminal_reason: "assistant_completed" }, + ]) + expect( + yield* db + .select() + .from(SessionLegacyActivityTerminalTable) + .where(eq(SessionLegacyActivityTerminalTable.activity_id, activity.activity_id)) + .all() + .pipe(Effect.orDie), + ).toMatchObject([ + { + run_id: runs[0]?.run_id, + state: "settled", + reason_code: "assistant_completed", + membership_ordinal: 1, + }, + ]) + // The steered message is persisted as an ordinary user message in history. const msgs = yield* sessions.messages({ sessionID: chat.id }) const steered = msgs.find((m) => m.info.role === "user" && m.info.id === MessageID.make(admitted.id)) From 87b383d73b7ea5cf5866ebbb92353e94db0c3ebe Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 22:50:27 +0800 Subject: [PATCH 07/44] fix(session): recover crash-ambiguous sessions safely --- packages/app/src/context/directory-sync.ts | 5 +- .../global-sync/session-prefetch.test.ts | 4 + .../context/global-sync/session-prefetch.ts | 3 + packages/app/src/i18n/en.ts | 24 + packages/app/src/i18n/zh.ts | 19 + packages/app/src/pages/layout.tsx | 3 +- packages/app/src/pages/session.tsx | 29 +- .../composer/session-composer-region.tsx | 23 +- .../session-provider-recovery-dock.test.ts | 167 +++ .../session-provider-recovery-dock.tsx | 313 +++++ packages/app/src/utils/diffs.test.ts | 47 +- packages/app/src/utils/diffs.ts | 32 +- packages/core/src/database/migration.gen.ts | 2 + ...20260812120000_legacy_provider_recovery.ts | 787 +++++++++++++ .../20260812140000_session_diff_manifest.ts | 11 + packages/core/src/event.ts | 176 ++- packages/core/src/session/projector.ts | 12 +- packages/core/src/session/sql.ts | 67 +- packages/core/src/snapshot.ts | 22 + packages/core/src/v1/session.ts | 26 + packages/core/test/event.test.ts | 288 +++++ ...y-provider-recovery-delete-cascade.test.ts | 495 ++++++++ packages/core/test/session-projector.test.ts | 27 + .../src/control-plane/workspace.ts | 417 ++++--- packages/deepagent-code/src/git/index.ts | 10 +- packages/deepagent-code/src/project/vcs.ts | 181 ++- .../instance/httpapi/groups/instance.ts | 15 + .../routes/instance/httpapi/groups/session.ts | 44 +- .../routes/instance/httpapi/groups/sync.ts | 22 +- .../instance/httpapi/groups/workspace.ts | 4 +- .../instance/httpapi/handlers/instance.ts | 18 +- .../httpapi/handlers/session-errors.ts | 22 + .../instance/httpapi/handlers/session.ts | 55 +- .../routes/instance/httpapi/handlers/sync.ts | 143 ++- .../instance/httpapi/handlers/workspace.ts | 14 +- .../middleware/sync-replay-body-limit.ts | 64 ++ .../server/routes/instance/httpapi/server.ts | 4 + .../src/session/legacy-provider-resolution.ts | 765 +++++++++++++ .../deepagent-code/src/session/message-v2.ts | 494 +++++++- .../src/session/prompt-epoch.sql.ts | 2 + .../src/session/prompt-epoch.ts | 3 + packages/deepagent-code/src/session/prompt.ts | 30 +- .../src/session/recovery-transfer-guard.ts | 75 ++ packages/deepagent-code/src/session/revert.ts | 145 ++- .../deepagent-code/src/session/session.ts | 300 ++++- .../deepagent-code/src/session/summary.ts | 118 +- packages/deepagent-code/src/snapshot/index.ts | 701 +++++++++--- .../src/sync/replay-protocol.ts | 61 + .../test/control-plane/workspace.test.ts | 160 ++- .../deepagent-code/test/project/vcs.test.ts | 108 +- .../test/server/httpapi-instance.test.ts | 23 + .../server/httpapi-provider-recovery.test.ts | 254 +++++ .../test/server/httpapi-session.test.ts | 4 +- .../test/server/httpapi-sync.test.ts | 471 +++++++- .../server/session-diff-missing-patch.test.ts | 108 +- .../test/server/session-messages.test.ts | 49 +- .../test/session/compaction.test.ts | 1 + .../legacy-provider-resolution.test.ts | 519 +++++++++ .../test/session/processor-effect.test.ts | 1 + .../test/session/prompt.test.ts | 2 + .../test/session/revert-compact.test.ts | 11 +- .../test/session/snapshot-tool-race.test.ts | 40 +- .../deepagent-code/test/session/steer.test.ts | 1 + .../test/snapshot/snapshot.test.ts | 132 +++ packages/sdk/js/src/gen/sdk.gen.ts | 102 +- packages/sdk/js/src/gen/types.gen.ts | 1014 ++++++++++++++++- packages/sdk/js/src/v2/gen/sdk.gen.ts | 102 +- packages/sdk/js/src/v2/gen/types.gen.ts | 1014 ++++++++++++++++- 68 files changed, 9712 insertions(+), 693 deletions(-) create mode 100644 packages/app/src/pages/session/composer/session-provider-recovery-dock.test.ts create mode 100644 packages/app/src/pages/session/composer/session-provider-recovery-dock.tsx create mode 100644 packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts create mode 100644 packages/core/src/database/migration/20260812140000_session_diff_manifest.ts create mode 100644 packages/core/test/legacy-provider-recovery-delete-cascade.test.ts create mode 100644 packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts create mode 100644 packages/deepagent-code/src/session/legacy-provider-resolution.ts create mode 100644 packages/deepagent-code/src/session/recovery-transfer-guard.ts create mode 100644 packages/deepagent-code/src/sync/replay-protocol.ts create mode 100644 packages/deepagent-code/test/server/httpapi-provider-recovery.test.ts create mode 100644 packages/deepagent-code/test/session/legacy-provider-resolution.test.ts diff --git a/packages/app/src/context/directory-sync.ts b/packages/app/src/context/directory-sync.ts index 202c635b2..d6bc4a949 100644 --- a/packages/app/src/context/directory-sync.ts +++ b/packages/app/src/context/directory-sync.ts @@ -6,6 +6,7 @@ import { clearSessionPrefetch, getSessionPrefetch, getSessionPrefetchPromise, + SESSION_MESSAGE_PAGE_LIMIT, setSessionPrefetch, } from "./global-sync/session-prefetch" import type { Message, Part } from "@deepagent-code/sdk/v2/client" @@ -267,7 +268,7 @@ export const createDirSyncContext = ( } const absolute = (path: string) => (current()[0].path.directory + "/" + path).replace("//", "/") const initialMessagePageSize = 80 - const historyMessagePageSize = 200 + const historyMessagePageSize = SESSION_MESSAGE_PAGE_LIMIT const inflight = new Map>() const trailing = new Set() const inflightMessages = new Map>() @@ -679,7 +680,7 @@ export const createDirSyncContext = ( const [, setStore] = serverSync.child(directory) touch(directory, setStore, sessionID) const key = keyFor(directory, sessionID) - const step = count ?? historyMessagePageSize + const step = Math.min(count ?? historyMessagePageSize, SESSION_MESSAGE_PAGE_LIMIT) if (meta.loading[key]) return if (meta.complete[key]) return const before = meta.cursor[key] diff --git a/packages/app/src/context/global-sync/session-prefetch.test.ts b/packages/app/src/context/global-sync/session-prefetch.test.ts index cf8ee77b4..0ea8e1918 100644 --- a/packages/app/src/context/global-sync/session-prefetch.test.ts +++ b/packages/app/src/context/global-sync/session-prefetch.test.ts @@ -4,6 +4,7 @@ import { clearSessionPrefetchDirectory, getSessionPrefetch, runSessionPrefetch, + SESSION_MESSAGE_PAGE_LIMIT, setSessionPrefetch, shouldSkipSessionPrefetch, } from "./session-prefetch" @@ -12,6 +13,9 @@ import { ServerScope } from "@/utils/server-scope" const scope = ServerScope.local describe("session prefetch", () => { + test("uses the bounded server page size", () => { + expect(SESSION_MESSAGE_PAGE_LIMIT).toBe(100) + }) test("stores and clears message metadata by directory", () => { clearSessionPrefetch(scope, "/tmp/a", ["ses_1"]) clearSessionPrefetch(scope, "/tmp/b", ["ses_1"]) diff --git a/packages/app/src/context/global-sync/session-prefetch.ts b/packages/app/src/context/global-sync/session-prefetch.ts index d2c3d86f4..553822989 100644 --- a/packages/app/src/context/global-sync/session-prefetch.ts +++ b/packages/app/src/context/global-sync/session-prefetch.ts @@ -4,6 +4,9 @@ const key = (scope: ServerScope, directory: string, sessionID: string) => Scoped export const SESSION_PREFETCH_TTL = 15_000 +// Keep every client page request within the server's bounded MessageV2 contract. +export const SESSION_MESSAGE_PAGE_LIMIT = 100 + type Meta = { limit: number cursor?: string diff --git a/packages/app/src/i18n/en.ts b/packages/app/src/i18n/en.ts index 0bfd2260f..80d048693 100644 --- a/packages/app/src/i18n/en.ts +++ b/packages/app/src/i18n/en.ts @@ -1624,4 +1624,28 @@ export const dict = { "session.context.state.denied": "Denied", "session.context.state.disabled": "Disabled", "session.context.state.broken": "Broken reference", + "session.providerRecovery.title": "Provider outcome is unknown", + "session.providerRecovery.description": + "The previous request may have reached the provider before the app restarted. Abandoning it preserves the incident record, excludes partial output, and does not undo provider-side effects.", + "session.providerRecovery.abandon": "Abandon and continue", + "session.providerRecovery.confirmTitle": "Abandon unknown provider outcome?", + "session.providerRecovery.confirmDescription": + "This does not cancel or undo work that may already have run at the provider. The partial assistant response will remain as incident evidence and will not be used as future model history.", + "session.providerRecovery.resolving": "Resolving...", + "session.providerRecovery.resolved": "Session recovery completed", + "session.providerRecovery.failed": "Session recovery failed", + "session.providerRecovery.conflict": "Recovery state changed", + "session.providerRecovery.conflictDescription": + "The recovery state changed before this action completed. The latest server state has been loaded.", + "session.providerRecovery.refreshFailed": "Recovery completed, but the conversation could not be refreshed", + "session.providerRecovery.refreshFailedDescription": + "Sending remains disabled until the recovered conversation and messages can be loaded from the server.", + "session.providerRecovery.loadFailed": "Could not verify provider recovery state", + "session.providerRecovery.loadFailedDescription": + "Message sending remains disabled until the server can confirm whether this session requires recovery.", + "session.providerRecovery.unsupported": "This continuation requires maintenance recovery and cannot be abandoned here.", + "session.providerRecovery.workspaceUnsupported": + "This shared workspace requires coordinated recovery and cannot be resolved from this app instance.", + "session.providerRecovery.baselineUnsupported": + "The source World State baseline is missing or invalid. Sending remains locked until maintenance recovery verifies it.", } diff --git a/packages/app/src/i18n/zh.ts b/packages/app/src/i18n/zh.ts index 6c01349b5..ceaf5a905 100644 --- a/packages/app/src/i18n/zh.ts +++ b/packages/app/src/i18n/zh.ts @@ -1566,4 +1566,23 @@ export const dict = { "session.context.state.denied": "无权访问", "session.context.state.disabled": "未启用", "session.context.state.broken": "引用损坏", + "session.providerRecovery.title": "Provider 结果未知", + "session.providerRecovery.description": + "应用重启前，请求可能已经到达 Provider。放弃后会保留事故记录、排除不完整输出，但不会撤销 Provider 侧可能已经发生的副作用。", + "session.providerRecovery.abandon": "放弃未知结果并继续", + "session.providerRecovery.confirmTitle": "放弃未知的 Provider 结果？", + "session.providerRecovery.confirmDescription": + "此操作不会取消或撤销 Provider 侧可能已经执行的工作。不完整的助手响应将作为事故证据保留，但不会进入后续模型历史。", + "session.providerRecovery.resolving": "处理中...", + "session.providerRecovery.resolved": "会话恢复完成", + "session.providerRecovery.failed": "会话恢复失败", + "session.providerRecovery.conflict": "恢复状态已变更", + "session.providerRecovery.conflictDescription": "提交期间恢复状态发生变化，已重新加载服务器上的最新状态。", + "session.providerRecovery.refreshFailed": "恢复已完成，但对话内容刷新失败", + "session.providerRecovery.refreshFailedDescription": "从服务器重新加载已恢复的对话和消息之前，发送将保持禁用。", + "session.providerRecovery.loadFailed": "无法确认 Provider 恢复状态", + "session.providerRecovery.loadFailedDescription": "在服务器确认该会话是否需要恢复之前，消息发送将保持禁用。", + "session.providerRecovery.unsupported": "该 continuation 需要维护恢复，当前不能在这里放弃。", + "session.providerRecovery.workspaceUnsupported": "共享工作区需要协调恢复，当前应用实例不能直接处理。", + "session.providerRecovery.baselineUnsupported": "源 World State baseline 缺失或无效；维护恢复验证完成前，发送将保持锁定。", } satisfies Partial> diff --git a/packages/app/src/pages/layout.tsx b/packages/app/src/pages/layout.tsx index c2af49df6..799035d4b 100644 --- a/packages/app/src/pages/layout.tsx +++ b/packages/app/src/pages/layout.tsx @@ -44,6 +44,7 @@ import { getSessionPrefetch, isSessionPrefetchCurrent, runSessionPrefetch, + SESSION_MESSAGE_PAGE_LIMIT, setSessionPrefetch, shouldSkipSessionPrefetch, } from "@/context/global-sync/session-prefetch" @@ -685,7 +686,7 @@ export default function Layout(props: ParentProps<{ onStartupRestoreSettled?: () running: number } - const prefetchChunk = 200 + const prefetchChunk = SESSION_MESSAGE_PAGE_LIMIT const prefetchConcurrency = 2 const prefetchPendingLimit = 10 const span = 4 diff --git a/packages/app/src/pages/session.tsx b/packages/app/src/pages/session.tsx index 72b4804c5..96f2616c8 100644 --- a/packages/app/src/pages/session.tsx +++ b/packages/app/src/pages/session.tsx @@ -344,6 +344,7 @@ export default function Page() { } const info = createMemo(() => (params.id ? sync.session.get(params.id) : undefined)) + const diffManifest = createMemo(() => info()?.summary?.diffManifest) const isChildSession = createMemo(() => !!info()?.parentID) const diffs = createMemo(() => (params.id ? list(sync.data.session_diff[params.id]) : [])) const canReview = createMemo(() => !!sync.project) @@ -991,15 +992,25 @@ export default function Page() { } return ( - option && setStore("changes", option)} + variant="ghost" + size="small" + valueClass="text-14-medium" + /> + + + {language.t("session.context.summary.partial")} + + + ) } diff --git a/packages/app/src/pages/session/composer/session-composer-region.tsx b/packages/app/src/pages/session/composer/session-composer-region.tsx index 65e13f527..62e738464 100644 --- a/packages/app/src/pages/session/composer/session-composer-region.tsx +++ b/packages/app/src/pages/session/composer/session-composer-region.tsx @@ -20,6 +20,7 @@ import { GoalStatusBar } from "@/components/deepagent/goal-status-bar" import { GoalStartButton } from "@/components/deepagent/goal-start-button" import type { FollowupDraft } from "@/components/prompt-input/submit" import { createResizeObserver } from "@solid-primitives/resize-observer" +import { SessionProviderRecoveryDock } from "@/pages/session/composer/session-provider-recovery-dock" export function SessionComposerRegion(props: { state: SessionComposerState @@ -71,6 +72,16 @@ export function SessionComposerRegion(props: { const parentID = createMemo(() => info()?.parentID) const child = createMemo(() => !!parentID()) const showComposer = createMemo(() => !props.state.blocked() || child()) + const [recoveryStore, setRecoveryStore] = createStore({ + sessionID: route.params.id, + pending: !!route.params.id, + }) + + createEffect(() => { + const sessionID = route.params.id + if (recoveryStore.sessionID === sessionID) return + setRecoveryStore({ sessionID, pending: !!sessionID }) + }) const previewPrompt = () => prompt @@ -189,6 +200,16 @@ export function SessionComposerRegion(props: { )} + + { + if (recoveryStore.sessionID !== route.params.id) return + setRecoveryStore("pending", pending) + }} + /> + + diff --git a/packages/app/src/pages/session/composer/session-provider-recovery-dock.test.ts b/packages/app/src/pages/session/composer/session-provider-recovery-dock.test.ts new file mode 100644 index 000000000..0c839cbda --- /dev/null +++ b/packages/app/src/pages/session/composer/session-provider-recovery-dock.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, test } from "bun:test" +import type { ProviderRecovery } from "./session-provider-recovery-dock" +import { + providerRecoveryCommandID, + providerRecoveryFingerprint, + providerRecoveryPending, + providerRecoveryResolveInput, + resolveProviderRecovery, +} from "./session-provider-recovery-dock" + +const recovery = (overrides: Partial = {}): ProviderRecovery => ({ + receiptID: "receipt-1", + sessionID: "session-1", + assistantMessageID: "message-1", + providerID: "provider", + modelID: "model", + providerState: "indeterminate_after_crash", + promptEpoch: 3, + promptWindowID: "window-1", + historyHash: "history-1", + requestHash: "request-1", + sessionMutationEpoch: 7, + continuationRecoverySupported: true, + workspaceRecoverySupported: true, + sourceWorldStateBaselineStatus: "available", + worldStateBaselineHash: "wsb1-baseline", + ...overrides, +}) + +describe("provider recovery command", () => { + test("keeps the command id stable for an exact retry", async () => { + const item = recovery() + expect(await providerRecoveryCommandID(item)).toBe(await providerRecoveryCommandID({ ...item })) + expect((await providerRecoveryCommandID(item)).length).toBe(82) + }) + + test("changes the command id when any recovery authority field changes", async () => { + const item = recovery() + expect(providerRecoveryFingerprint(item)).not.toBe( + providerRecoveryFingerprint(recovery({ sessionMutationEpoch: item.sessionMutationEpoch + 1 })), + ) + expect(await providerRecoveryCommandID(item)).not.toBe( + await providerRecoveryCommandID(recovery({ sessionMutationEpoch: item.sessionMutationEpoch + 1 })), + ) + expect(await providerRecoveryCommandID(item)).not.toBe( + await providerRecoveryCommandID(recovery({ historyHash: "history-2" })), + ) + }) + + test("bounds the command id for a maximum-length receipt", async () => { + expect((await providerRecoveryCommandID(recovery({ receiptID: "r".repeat(200) }))).length).toBe(82) + }) + + test("builds an abandon-only request with the complete expected authority", async () => { + expect(await providerRecoveryResolveInput(recovery())).toEqual({ + commandID: await providerRecoveryCommandID(recovery()), + receiptID: "receipt-1", + decision: "abandoned", + expected: { + providerState: "indeterminate_after_crash", + promptEpoch: 3, + sessionMutationEpoch: 7, + requestHash: "request-1", + historyHash: "history-1", + worldStateBaselineHash: "wsb1-baseline", + }, + }) + }) +}) + +describe("provider recovery admission", () => { + test("fails closed while loading or after a list failure", () => { + expect(providerRecoveryPending({ loading: true, error: undefined, recoveries: undefined })).toBe(true) + expect(providerRecoveryPending({ loading: false, error: new Error("offline"), recoveries: undefined })).toBe(true) + expect( + providerRecoveryPending({ loading: false, error: undefined, recoveries: [], settlementFailed: true }), + ).toBe(true) + }) + + test("blocks for a descriptor and unblocks only for a confirmed empty list", () => { + expect(providerRecoveryPending({ loading: false, error: undefined, recoveries: [recovery()] })).toBe(true) + expect(providerRecoveryPending({ loading: false, error: undefined, recoveries: [] })).toBe(false) + }) +}) + +describe("provider recovery settlement", () => { + test("uses the exact idempotent request and refreshes after success", async () => { + const requests: unknown[] = [] + let refreshes = 0 + const result = await resolveProviderRecovery({ + sessionID: "session-1", + recovery: recovery(), + resolve: async (request) => { + requests.push(request) + return {} + }, + refresh: async () => { + refreshes += 1 + return "cleared" + }, + }) + + expect(result).toEqual({ status: "resolved" }) + expect(requests).toEqual([{ sessionID: "session-1", ...(await providerRecoveryResolveInput(recovery())) }]) + expect(refreshes).toBe(1) + }) + + test("classifies a conflict and refreshes authority before returning", async () => { + let refreshes = 0 + const error = { name: "ConflictError", data: { message: "stale recovery state" } } + const result = await resolveProviderRecovery({ + sessionID: "session-1", + recovery: recovery(), + resolve: async () => ({ error, response: { status: 409 } }), + refresh: async () => { + refreshes += 1 + return "retained" + }, + }) + + expect(result).toEqual({ status: "conflict", error, refreshed: "retained" }) + expect(refreshes).toBe(1) + }) + + test("refreshes authority after a network-unknown result", async () => { + let refreshes = 0 + const error = new Error("connection closed") + const result = await resolveProviderRecovery({ + sessionID: "session-1", + recovery: recovery(), + resolve: async () => { + throw error + }, + refresh: async () => { + refreshes += 1 + return "failed" + }, + }) + + expect(result).toEqual({ status: "failed", error, refreshed: "failed" }) + expect(refreshes).toBe(1) + }) + + test("reports a completed command whose canonical refresh failed", async () => { + expect( + await resolveProviderRecovery({ + sessionID: "session-1", + recovery: recovery(), + resolve: async () => ({}), + refresh: async () => "failed", + }), + ).toEqual({ status: "refresh_failed" }) + }) + + test("settles a network-unknown command when canonical state no longer contains the receipt", async () => { + expect( + await resolveProviderRecovery({ + sessionID: "session-1", + recovery: recovery(), + resolve: async () => { + throw new Error("connection closed after dispatch") + }, + refresh: async () => "cleared", + }), + ).toEqual({ status: "resolved" }) + }) +}) diff --git a/packages/app/src/pages/session/composer/session-provider-recovery-dock.tsx b/packages/app/src/pages/session/composer/session-provider-recovery-dock.tsx new file mode 100644 index 000000000..6adc93721 --- /dev/null +++ b/packages/app/src/pages/session/composer/session-provider-recovery-dock.tsx @@ -0,0 +1,313 @@ +import type { SessionProviderResolutionListResponse } from "@deepagent-code/sdk/v2" +import { hash } from "@deepagent-code/core/util/encode" +import { Button } from "@deepagent-code/ui/button" +import { useDialog } from "@deepagent-code/ui/context/dialog" +import { Dialog } from "@deepagent-code/ui/dialog" +import { Icon } from "@deepagent-code/ui/icon" +import { createEffect, createResource, onCleanup, Show } from "solid-js" +import { createStore } from "solid-js/store" +import { useLanguage } from "@/context/language" +import { useSDK } from "@/context/sdk" +import { useSync } from "@/context/sync" +import { formatServerError } from "@/utils/server-errors" +import { showToast } from "@/utils/toast" + +export type ProviderRecovery = SessionProviderResolutionListResponse[number] + +export function providerRecoveryFingerprint(item: ProviderRecovery) { + return JSON.stringify([ + item.receiptID, + item.providerState, + item.promptEpoch, + item.sessionMutationEpoch, + item.requestHash, + item.historyHash, + item.worldStateBaselineHash, + ]) +} + +export async function providerRecoveryCommandID(item: ProviderRecovery) { + return `provider-recovery-${await hash(providerRecoveryFingerprint(item))}` +} + +export function providerRecoveryPending(input: { + loading: boolean + error: unknown + recoveries: readonly ProviderRecovery[] | undefined + settlementFailed?: boolean +}) { + return input.loading || input.error !== undefined || input.settlementFailed === true || (input.recoveries?.length ?? 0) > 0 +} + +export async function providerRecoveryResolveInput(item: ProviderRecovery) { + return { + commandID: await providerRecoveryCommandID(item), + receiptID: item.receiptID, + decision: "abandoned" as const, + expected: { + providerState: item.providerState, + promptEpoch: item.promptEpoch, + sessionMutationEpoch: item.sessionMutationEpoch, + requestHash: item.requestHash, + historyHash: item.historyHash, + worldStateBaselineHash: item.worldStateBaselineHash!, + }, + } +} + +export async function resolveProviderRecovery(input: { + sessionID: string + recovery: ProviderRecovery + resolve: (request: Awaited> & { sessionID: string }) => Promise<{ + error?: unknown + response?: { status?: number } + }> + refresh: () => Promise<"cleared" | "retained" | "failed"> +}) { + const request = await providerRecoveryResolveInput(input.recovery) + const outcome = await input + .resolve({ sessionID: input.sessionID, ...request }) + .then( + (result) => ({ result }), + (error: unknown) => ({ error }), + ) + + if ("error" in outcome) { + const refreshed = await input.refresh() + if (refreshed === "cleared") return { status: "resolved" as const } + return { status: "failed" as const, error: outcome.error, refreshed } + } + + if (outcome.result.error !== undefined) { + const refreshed = await input.refresh() + if (refreshed === "cleared") return { status: "resolved" as const } + return { + status: outcome.result.response?.status === 409 ? ("conflict" as const) : ("failed" as const), + error: outcome.result.error, + refreshed, + } + } + + const refreshed = await input.refresh() + return { status: refreshed === "cleared" ? ("resolved" as const) : ("refresh_failed" as const) } +} + +export function SessionProviderRecoveryDock(props: { + sessionID: string + onPendingChange: (pending: boolean) => void +}) { + const sdk = useSDK() + const sync = useSync() + const language = useLanguage() + const dialog = useDialog() + const [store, setStore] = createStore({ + resolving: undefined as string | undefined, + status: "idle" as "idle" | "conflict" | "failed", + detail: undefined as string | undefined, + }) + const [recoveries, { refetch }] = createResource( + () => props.sessionID, + async (sessionID) => { + const result = await sdk.client.session.providerResolutionList({ sessionID }) + if (result.error) throw result.error + return result.data ?? [] + }, + ) + const recovery = () => recoveries()?.[0] + + createEffect(() => { + props.sessionID + setStore({ resolving: undefined, status: "idle", detail: undefined }) + }) + + createEffect(() => { + props.onPendingChange( + providerRecoveryPending({ + loading: recoveries.loading, + error: recoveries.error, + recoveries: recoveries(), + settlementFailed: store.status === "failed", + }), + ) + }) + onCleanup(() => props.onPendingChange(false)) + + const refresh = async (sessionID: string, receiptID: string) => { + if (props.sessionID !== sessionID) return "failed" as const + const results = await Promise.allSettled([ + refetch(), + sync.session.sync(sessionID, { force: true }), + ]) + if (results.some((result) => result.status === "rejected")) return "failed" as const + const descriptors = results[0].status === "fulfilled" ? results[0].value : undefined + return descriptors?.some((item) => item.receiptID === receiptID) ? ("retained" as const) : ("cleared" as const) + } + + const abandon = async (item: ProviderRecovery) => { + if (store.resolving) return + const sessionID = props.sessionID + setStore({ resolving: item.receiptID, status: "idle", detail: undefined }) + const outcome = await resolveProviderRecovery({ + sessionID, + recovery: item, + resolve: (request) => sdk.client.session.providerResolutionResolve(request), + refresh: () => refresh(sessionID, item.receiptID), + }) + if (props.sessionID !== sessionID) return + setStore("resolving", undefined) + + if (outcome.status === "failed") { + setStore({ status: "failed", detail: formatServerError(outcome.error, language.t) }) + showToast({ + variant: "error", + title: language.t("session.providerRecovery.failed"), + description: store.detail, + }) + return + } + + if (outcome.status === "conflict") { + setStore({ + status: "conflict", + detail: formatServerError(outcome.error, language.t), + }) + showToast({ + variant: "default", + title: language.t("session.providerRecovery.conflict"), + description: store.detail, + }) + return + } + + if (outcome.status === "resolved") { + setStore({ status: "idle", detail: undefined }) + showToast({ variant: "success", title: language.t("session.providerRecovery.resolved") }) + return + } + + setStore({ status: "failed", detail: undefined }) + showToast({ variant: "error", title: language.t("session.providerRecovery.refreshFailed") }) + } + + const retry = async () => { + const results = await Promise.allSettled([refetch(), sync.session.sync(props.sessionID, { force: true })]) + setStore("status", results.every((result) => result.status === "fulfilled") ? "idle" : "failed") + } + + const confirm = (item: ProviderRecovery) => + dialog.show(() => ( + +
+
{language.t("session.providerRecovery.confirmDescription")}
+
+ + +
+
+ + )) + + return ( + +
+
+ +
+
+ {language.t( + recoveries.error ? "session.providerRecovery.loadFailed" : "session.providerRecovery.refreshFailed", + )} +
+
+ {language.t( + recoveries.error + ? "session.providerRecovery.loadFailedDescription" + : "session.providerRecovery.refreshFailedDescription", + )} +
+
+ +
+
+ + } + > + {(item) => { + const supported = () => + item.continuationRecoverySupported && + item.workspaceRecoverySupported && + item.sourceWorldStateBaselineStatus === "available" + return ( +
+
+ +
+
{language.t("session.providerRecovery.title")}
+
+ {language.t("session.providerRecovery.description")} +
+
+ {item.providerID} / {item.modelID} +
+
+ +
+ +
+ {language.t("session.providerRecovery.unsupported")} +
+ + +
+ {language.t("session.providerRecovery.workspaceUnsupported")} +
+ + +
+ {language.t("session.providerRecovery.baselineUnsupported")} +
+ + +
+ {language.t( + store.status === "conflict" + ? "session.providerRecovery.conflictDescription" + : "session.providerRecovery.failed", + )} + : {store.detail} +
+ +
+ ) + }} + + ) +} diff --git a/packages/app/src/utils/diffs.test.ts b/packages/app/src/utils/diffs.test.ts index 137a54d4c..a9e35f834 100644 --- a/packages/app/src/utils/diffs.test.ts +++ b/packages/app/src/utils/diffs.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test" import type { SnapshotFileDiff } from "@deepagent-code/sdk/v2" import type { Message } from "@deepagent-code/sdk/v2/client" -import { diffs, message } from "./diffs" +import { DIFF_PROJECTION_LIMITS, diffs, message } from "./diffs" const item = { file: "src/app.ts", @@ -28,10 +28,51 @@ describe("diffs", () => { expect( diffs([ item, - { file: "src/bad.ts", additions: 1, deletions: 1 }, + { file: "src/metadata-only.ts", additions: 1, deletions: 1 }, { patch: item.patch, additions: 1, deletions: 1 }, ]), - ).toEqual([item]) + ).toEqual([item, { file: "src/metadata-only.ts", additions: 1, deletions: 1, patch: "" }]) + }) + + test("keeps metadata-only summary diffs emitted by the server", () => { + expect(diffs([{ file: "src/app.ts", additions: 2, deletions: 1, status: "modified" }])).toEqual([ + { file: "src/app.ts", patch: "", additions: 2, deletions: 1, status: "modified" }, + ]) + }) + + test("bounds legacy diff arrays and strips oversized patches", () => { + const oversized = { + ...item, + file: "src/oversized.ts", + patch: "x".repeat(DIFF_PROJECTION_LIMITS.patchCharsPerFile + 1), + } + const input = [ + oversized, + ...Array.from({ length: DIFF_PROJECTION_LIMITS.files + 10 }, (_, index) => ({ + ...item, + file: `src/file-${index}.ts`, + })), + ] + + const result = diffs(input) + expect(result).toHaveLength(DIFF_PROJECTION_LIMITS.files) + expect(result[0]).toMatchObject({ file: "src/oversized.ts", patch: "" }) + }) + + test("bounds cumulative patch text retained by the client store", () => { + const patch = "x".repeat(Math.floor(DIFF_PROJECTION_LIMITS.patchCharsTotal / 3)) + const result = diffs( + Array.from({ length: 5 }, (_, index) => ({ + ...item, + file: `src/file-${index}.ts`, + patch, + })), + ) + + expect(result.reduce((total, value) => total + (value.patch?.length ?? 0), 0)).toBeLessThanOrEqual( + DIFF_PROJECTION_LIMITS.patchCharsTotal, + ) + expect(result.some((value) => value.patch === "")).toBe(true) }) }) diff --git a/packages/app/src/utils/diffs.ts b/packages/app/src/utils/diffs.ts index 7ff361728..9048f772f 100644 --- a/packages/app/src/utils/diffs.ts +++ b/packages/app/src/utils/diffs.ts @@ -3,10 +3,16 @@ import type { Message } from "@deepagent-code/sdk/v2/client" type Diff = SnapshotFileDiff | VcsFileDiff +export const DIFF_PROJECTION_LIMITS = { + files: 200, + patchCharsPerFile: 256 * 1024, + patchCharsTotal: 1024 * 1024, +} as const + function diff(value: unknown): value is Diff { if (!value || typeof value !== "object" || Array.isArray(value)) return false if (!("file" in value) || typeof value.file !== "string") return false - if (!("patch" in value) || typeof value.patch !== "string") return false + if ("patch" in value && value.patch !== undefined && typeof value.patch !== "string") return false if (!("additions" in value) || typeof value.additions !== "number") return false if (!("deletions" in value) || typeof value.deletions !== "number") return false if (!("status" in value) || value.status === undefined) return true @@ -18,11 +24,25 @@ function object(value: unknown): value is Record { } export function diffs(value: unknown): Diff[] { - if (Array.isArray(value) && value.every(diff)) return value - if (Array.isArray(value)) return value.filter(diff) - if (diff(value)) return [value] - if (!object(value)) return [] - return Object.values(value).filter(diff) + const items = Array.isArray(value) + ? value.filter(diff) + : diff(value) + ? [value] + : object(value) + ? Object.values(value).filter(diff) + : [] + let patchChars = 0 + return items.slice(0, DIFF_PROJECTION_LIMITS.files).map((item) => { + const patch = item.patch ?? "" + if ( + patch.length > DIFF_PROJECTION_LIMITS.patchCharsPerFile || + patchChars + patch.length > DIFF_PROJECTION_LIMITS.patchCharsTotal + ) { + return { ...item, patch: "" } + } + patchChars += patch.length + return { ...item, patch } + }) } export function message(value: Message): Message { diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index cda868844..d0345dbb9 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -80,6 +80,8 @@ export const migrations = ( import("./migration/20260810170000_part_integrity_backfill"), import("./migration/20260811090000_legacy_activity_progress"), import("./migration/20260811100000_legacy_activity_owner"), + import("./migration/20260812120000_legacy_provider_recovery"), import("./migration/20260812130000_legacy_activity_lifecycle_expand"), + import("./migration/20260812140000_session_diff_manifest"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts b/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts new file mode 100644 index 000000000..425f51b21 --- /dev/null +++ b/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts @@ -0,0 +1,787 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812120000_legacy_provider_recovery", + up(tx) { + return Effect.gen(function* () { + // The original receipt table predated Session FK ownership. Rebuild it with its two FK + // children detached so SQLite cannot cascade evidence away or retain references to a renamed + // old table. Historical orphan receipts are intentionally omitted from the rebuilt authority. + yield* tx.run(` + CREATE TEMP TABLE legacy_provider_recovery_argument_receipt AS + SELECT * FROM session_tool_argument_receipt + `) + yield* tx.run(` + CREATE TEMP TABLE legacy_provider_recovery_activity_progress AS + SELECT * FROM session_activity_progress + `) + yield* tx.run("DROP TABLE session_tool_argument_receipt") + yield* tx.run("DROP TABLE session_activity_progress") + yield* tx.run(` + CREATE TABLE session_tool_request_receipt_rebuilt ( + receipt_id TEXT NOT NULL PRIMARY KEY, + request_ordinal INTEGER NOT NULL CHECK (request_ordinal >= 1), + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + user_message_id TEXT NOT NULL, + assistant_message_id TEXT, + provider_attempt_id TEXT, + provider_id TEXT NOT NULL, + model_id TEXT NOT NULL, + protocol TEXT, + registry_tool_ids TEXT NOT NULL, + permission_filtered_tool_ids TEXT NOT NULL, + final_offered_tool_ids TEXT NOT NULL, + call_ids TEXT NOT NULL DEFAULT '[]', + tool_definition_hash TEXT, + tool_choice_mode TEXT, + adapter_tool_capability TEXT, + adapter_lowering_outcome TEXT, + estimated_input_tokens INTEGER, + physical_input_budget INTEGER, + reserved_output_tokens INTEGER, + safety_margin_tokens INTEGER, + context_limit_provenance TEXT, + request_state TEXT NOT NULL CHECK (request_state IN ('prepared','dispatched','rejected')), + request_error_code TEXT, + created_at INTEGER NOT NULL, + prompt_epoch INTEGER, + prompt_window_id TEXT, + effective_history_hash TEXT, + world_state_baseline_hash TEXT, + prompt_cache_key TEXT, + provider_request_hash TEXT, + response_chain_reuse_decision TEXT, + response_chain_refusal_reason TEXT, + request_input_hash TEXT, + final_request_hash TEXT, + provider_state TEXT NOT NULL DEFAULT 'preparing', + adapter_prepared_at INTEGER, + dispatching_at INTEGER, + streaming_at INTEGER, + terminal_at INTEGER, + response_fingerprint TEXT, + owner_token TEXT + ) + `) + yield* tx.run(` + INSERT INTO session_tool_request_receipt_rebuilt + SELECT receipt.* + FROM session_tool_request_receipt receipt + JOIN session ON session.id = receipt.session_id + `) + yield* tx.run("DROP TABLE session_tool_request_receipt") + yield* tx.run("ALTER TABLE session_tool_request_receipt_rebuilt RENAME TO session_tool_request_receipt") + yield* tx.run(` + CREATE UNIQUE INDEX session_tool_request_receipt_session_ordinal_idx + ON session_tool_request_receipt(session_id, request_ordinal) + `) + yield* tx.run(` + CREATE INDEX session_tool_request_receipt_session_idx + ON session_tool_request_receipt(session_id, created_at) + `) + yield* tx.run(` + CREATE INDEX session_tool_request_receipt_msg_idx + ON session_tool_request_receipt(assistant_message_id) + `) + yield* tx.run(` + CREATE INDEX session_tool_request_receipt_prompt_window_idx + ON session_tool_request_receipt(session_id, prompt_epoch, prompt_window_id) + `) + yield* tx.run(` + CREATE INDEX session_tool_request_receipt_provider_state_idx + ON session_tool_request_receipt(session_id, provider_state, created_at) + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_provider_transition + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_state != OLD.provider_state AND NOT ( + (OLD.provider_state = 'preparing' AND NEW.provider_state IN ('prepared', 'failed')) OR + (OLD.provider_state = 'prepared' AND NEW.provider_state IN ('dispatching', 'failed')) OR + (OLD.provider_state = 'dispatching' AND NEW.provider_state IN ('streaming', 'settled', 'failed', 'indeterminate_after_crash')) OR + (OLD.provider_state = 'streaming' AND NEW.provider_state IN ('settled', 'failed', 'indeterminate_after_crash')) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal provider receipt transition'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_dispatch_guard + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_state = 'dispatching' AND ( + NEW.final_request_hash IS NULL OR NEW.adapter_prepared_at IS NULL OR + NEW.prompt_epoch IS NULL OR NEW.prompt_window_id IS NULL OR + NEW.effective_history_hash IS NULL + ) + BEGIN + SELECT RAISE(ABORT, 'provider dispatch requires final request authority'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_binding_immutable + BEFORE UPDATE ON session_tool_request_receipt + WHEN NEW.receipt_id != OLD.receipt_id + OR NEW.request_ordinal != OLD.request_ordinal + OR NEW.session_id != OLD.session_id + OR NEW.user_message_id != OLD.user_message_id + OR NEW.assistant_message_id IS NOT OLD.assistant_message_id + OR NEW.provider_id != OLD.provider_id + OR NEW.model_id != OLD.model_id + OR NEW.protocol IS NOT OLD.protocol + OR NEW.prompt_epoch IS NOT OLD.prompt_epoch + OR NEW.prompt_window_id IS NOT OLD.prompt_window_id + OR NEW.effective_history_hash IS NOT OLD.effective_history_hash + OR NEW.world_state_baseline_hash IS NOT OLD.world_state_baseline_hash + OR NEW.request_input_hash IS NOT OLD.request_input_hash + OR NEW.owner_token IS NOT OLD.owner_token + OR NEW.created_at != OLD.created_at + OR (OLD.final_request_hash IS NOT NULL AND NEW.final_request_hash IS NOT OLD.final_request_hash) + OR (OLD.prompt_cache_key IS NOT NULL AND NEW.prompt_cache_key IS NOT OLD.prompt_cache_key) + OR (OLD.response_fingerprint IS NOT NULL AND NEW.response_fingerprint IS NOT OLD.response_fingerprint) + BEGIN + SELECT RAISE(ABORT, 'provider receipt binding is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_response_guard + BEFORE UPDATE OF response_fingerprint ON session_tool_request_receipt + WHEN NEW.response_fingerprint IS NOT OLD.response_fingerprint AND ( + OLD.response_fingerprint IS NOT NULL OR NEW.response_fingerprint IS NULL OR + NEW.provider_state NOT IN ('settled', 'failed') + ) + BEGIN + SELECT RAISE(ABORT, 'provider response fingerprint requires terminal receipt'); + END + `) + yield* tx.run(` + CREATE TABLE session_tool_argument_receipt ( + receipt_id TEXT NOT NULL, + layer TEXT NOT NULL CHECK (layer IN ('raw_frame','ai_sdk_input','adapter_assembly','processor_decoded')), + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + call_id TEXT, + tool_name TEXT, + event_type TEXT NOT NULL, + payload_hash TEXT, + payload_length INTEGER CHECK (payload_length IS NULL OR payload_length >= 0), + payload_keys TEXT NOT NULL, + unavailable_reason TEXT, + created_at INTEGER NOT NULL, + validation_outcome TEXT NOT NULL DEFAULT 'not_evaluated' CHECK (validation_outcome IN ( + 'not_evaluated','schema_valid','schema_invalid','semantic_valid','semantic_invalid', + 'conflict','no_progress' + )), + PRIMARY KEY (receipt_id, layer, ordinal), + FOREIGN KEY (receipt_id) REFERENCES session_tool_request_receipt(receipt_id) ON DELETE CASCADE, + CHECK (call_id IS NULL OR length(trim(call_id)) > 0), + CHECK (tool_name IS NULL OR length(trim(tool_name)) > 0), + CHECK ( + (payload_hash IS NOT NULL AND length(payload_hash) = 64 AND payload_length IS NOT NULL AND unavailable_reason IS NULL) OR + (payload_hash IS NULL AND payload_length IS NULL AND unavailable_reason IS NOT NULL AND length(trim(unavailable_reason)) > 0) + ) + ) + `) + yield* tx.run(` + INSERT INTO session_tool_argument_receipt + SELECT argument.* + FROM legacy_provider_recovery_argument_receipt argument + JOIN session_tool_request_receipt receipt ON receipt.receipt_id = argument.receipt_id + `) + yield* tx.run("DROP TABLE legacy_provider_recovery_argument_receipt") + yield* tx.run(` + CREATE INDEX session_tool_argument_receipt_call_idx + ON session_tool_argument_receipt(receipt_id, call_id, layer, ordinal) + `) + yield* tx.run(` + CREATE INDEX session_tool_argument_receipt_created_idx + ON session_tool_argument_receipt(created_at) + `) + yield* tx.run(` + CREATE TABLE session_activity_progress ( + activity_id TEXT NOT NULL REFERENCES session_legacy_activity(activity_id) ON DELETE CASCADE, + revision INTEGER NOT NULL CHECK (revision >= 0), + assistant_message_id TEXT NOT NULL UNIQUE REFERENCES message(id) ON DELETE CASCADE, + text_part_id TEXT REFERENCES part(id) ON DELETE SET NULL, + provider_receipt_id TEXT NOT NULL UNIQUE REFERENCES session_tool_request_receipt(receipt_id), + state TEXT NOT NULL CHECK (state IN ('provisional', 'progress', 'final', 'interrupted', 'recovery_required')), + finish_observed TEXT, + response_fingerprint TEXT, + created_at INTEGER NOT NULL, + settled_at INTEGER, + PRIMARY KEY (activity_id, revision), + CHECK ( + (state = 'provisional' AND settled_at IS NULL AND response_fingerprint IS NULL) OR + (state != 'provisional' AND settled_at IS NOT NULL) + ) + ) + `) + yield* tx.run(` + INSERT INTO session_activity_progress + SELECT progress.* + FROM legacy_provider_recovery_activity_progress progress + JOIN session_tool_request_receipt receipt ON receipt.receipt_id = progress.provider_receipt_id + `) + yield* tx.run("DROP TABLE legacy_provider_recovery_activity_progress") + yield* tx.run(` + CREATE TRIGGER session_activity_progress_validate_insert + BEFORE INSERT ON session_activity_progress + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 + FROM session_legacy_activity activity + JOIN message assistant ON assistant.id = NEW.assistant_message_id + JOIN session_tool_request_receipt receipt ON receipt.receipt_id = NEW.provider_receipt_id + WHERE activity.activity_id = NEW.activity_id + AND activity.state = 'active' + AND assistant.session_id = activity.session_id + AND receipt.session_id = activity.session_id + AND receipt.assistant_message_id = NEW.assistant_message_id + ) THEN RAISE(ABORT, 'activity progress ownership mismatch') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_progress_legal_update + BEFORE UPDATE ON session_activity_progress + WHEN NEW.activity_id != OLD.activity_id + OR NEW.revision != OLD.revision + OR NEW.assistant_message_id != OLD.assistant_message_id + OR NEW.provider_receipt_id != OLD.provider_receipt_id + OR NEW.created_at != OLD.created_at + OR OLD.state != 'provisional' + OR NEW.state NOT IN ('progress', 'final', 'interrupted', 'recovery_required') + OR NEW.settled_at IS NULL + BEGIN + SELECT RAISE(ABORT, 'illegal session_activity_progress transition'); + END + `) + + // SQLite cannot add a CHECK literal in place. Rebuild the PromptEpoch authority and its two + // FK-owned child tables together so recovery is a first-class reason without cascading away + // existing membership or World State rows. + yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_validate_insert") + yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_validate_update") + yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_message_validate_insert") + yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_message_validate_update") + yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_message_owner_immutable") + yield* tx.run(` + CREATE TEMP TABLE legacy_provider_recovery_epoch_message AS + SELECT * FROM session_prompt_epoch_message + `) + yield* tx.run(` + CREATE TEMP TABLE legacy_provider_recovery_world_state AS + SELECT * FROM session_world_state_baseline + `) + yield* tx.run("DROP TABLE session_prompt_epoch_message") + yield* tx.run("DROP TABLE session_world_state_baseline") + yield* tx.run(` + CREATE TABLE session_prompt_epoch_rebuilt ( + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + epoch INTEGER NOT NULL CHECK (epoch >= 0), + state TEXT NOT NULL CHECK (state IN ('active', 'retired')), + checkpoint_user_id TEXT, + checkpoint_assistant_id TEXT, + retained_tail_start_id TEXT, + source_end_message_id TEXT, + checkpoint_hash TEXT, + reason TEXT NOT NULL CHECK (reason IN + ('bootstrap', 'compaction', 'recovery', 'model', 'agent', 'directory', + 'workspace', 'tools', 'permission', 'renderer')), + created_at INTEGER NOT NULL, + retired_at INTEGER, + projection_version INTEGER, + canonicalization_version INTEGER, + base_message_count INTEGER, + effective_history_hash TEXT, + first_window_id TEXT, + previous_window_id TEXT, + window_id TEXT, + world_state_baseline_hash TEXT, + authority_state TEXT CHECK (authority_state IN ('legacy_pending', 'ready', 'recovery_required')), + recovery_reason TEXT, + recovery_resolution_id TEXT, + PRIMARY KEY (session_id, epoch) + ) + `) + yield* tx.run(` + INSERT INTO session_prompt_epoch_rebuilt ( + session_id, epoch, state, checkpoint_user_id, checkpoint_assistant_id, + retained_tail_start_id, source_end_message_id, checkpoint_hash, reason, + created_at, retired_at, projection_version, canonicalization_version, + base_message_count, effective_history_hash, first_window_id, previous_window_id, + window_id, world_state_baseline_hash, authority_state, recovery_reason, + recovery_resolution_id + ) + SELECT + session_id, epoch, state, checkpoint_user_id, checkpoint_assistant_id, + retained_tail_start_id, source_end_message_id, checkpoint_hash, reason, + created_at, retired_at, projection_version, canonicalization_version, + base_message_count, effective_history_hash, first_window_id, previous_window_id, + window_id, world_state_baseline_hash, authority_state, recovery_reason, NULL + FROM session_prompt_epoch + `) + yield* tx.run("DROP TABLE session_prompt_epoch") + yield* tx.run("ALTER TABLE session_prompt_epoch_rebuilt RENAME TO session_prompt_epoch") + yield* tx.run(` + CREATE UNIQUE INDEX session_prompt_epoch_active_idx + ON session_prompt_epoch (session_id) WHERE state = 'active' + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_prompt_epoch_window_idx + ON session_prompt_epoch (window_id) WHERE window_id IS NOT NULL + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_prompt_epoch_recovery_resolution_idx + ON session_prompt_epoch (recovery_resolution_id) + WHERE recovery_resolution_id IS NOT NULL + `) + yield* tx.run(` + CREATE TABLE session_prompt_epoch_message ( + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + prompt_epoch INTEGER NOT NULL, + ordinal INTEGER NOT NULL CHECK (ordinal >= 0), + message_id TEXT NOT NULL REFERENCES message(id) ON DELETE CASCADE, + PRIMARY KEY (session_id, prompt_epoch, ordinal), + UNIQUE (session_id, prompt_epoch, message_id), + FOREIGN KEY (session_id, prompt_epoch) + REFERENCES session_prompt_epoch(session_id, epoch) ON DELETE CASCADE + ) + `) + yield* tx.run(` + INSERT INTO session_prompt_epoch_message + SELECT * FROM legacy_provider_recovery_epoch_message + `) + yield* tx.run("DROP TABLE legacy_provider_recovery_epoch_message") + yield* tx.run(` + CREATE INDEX session_prompt_epoch_message_lookup_idx + ON session_prompt_epoch_message (session_id, prompt_epoch, message_id) + `) + yield* tx.run(` + CREATE TABLE session_world_state_baseline ( + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + prompt_epoch INTEGER NOT NULL, + section_id TEXT NOT NULL, + snapshot TEXT NOT NULL, + fragment TEXT NOT NULL, + fragment_hash TEXT NOT NULL, + provenance TEXT NOT NULL CHECK (provenance IN + ('native', 'fork_rebuilt', 'legacy_migration', 'recovery_copied', 'recovery_recollected')), + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, prompt_epoch, section_id), + FOREIGN KEY (session_id, prompt_epoch) + REFERENCES session_prompt_epoch(session_id, epoch) ON DELETE CASCADE + ) + `) + yield* tx.run(` + INSERT INTO session_world_state_baseline + SELECT * FROM legacy_provider_recovery_world_state + `) + yield* tx.run("DROP TABLE legacy_provider_recovery_world_state") + yield* tx.run(` + CREATE INDEX session_world_state_baseline_epoch_idx + ON session_world_state_baseline (session_id, prompt_epoch) + `) + yield* tx.run(` + CREATE TRIGGER session_prompt_epoch_message_validate_insert + BEFORE INSERT ON session_prompt_epoch_message + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_prompt_epoch + WHERE session_id = NEW.session_id AND epoch = NEW.prompt_epoch + ) THEN RAISE(ABORT, 'prompt_epoch_message_epoch_missing') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM message WHERE id = NEW.message_id AND session_id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_message_cross_session') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_prompt_epoch_message_validate_update + BEFORE UPDATE ON session_prompt_epoch_message + BEGIN + SELECT CASE WHEN NEW.session_id IS NOT OLD.session_id OR + NEW.prompt_epoch IS NOT OLD.prompt_epoch OR NEW.ordinal IS NOT OLD.ordinal OR + NEW.message_id IS NOT OLD.message_id + THEN RAISE(ABORT, 'prompt_epoch_message_binding_immutable') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_prompt_epoch_message_owner_immutable + BEFORE UPDATE OF session_id ON message + WHEN EXISTS ( + SELECT 1 FROM session_prompt_epoch + WHERE checkpoint_user_id = OLD.id OR checkpoint_assistant_id = OLD.id OR + retained_tail_start_id = OLD.id OR source_end_message_id = OLD.id + ) + BEGIN + SELECT RAISE(ABORT, 'prompt_epoch_referenced_message_owner_immutable'); + END + `) + + yield* tx.run(` + CREATE TABLE session_tool_request_resolution_command ( + command_id TEXT NOT NULL PRIMARY KEY, + request_hash TEXT NOT NULL, + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + receipt_id TEXT NOT NULL REFERENCES session_tool_request_receipt(receipt_id) ON DELETE CASCADE, + result_resolution_id TEXT, + created_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + CREATE INDEX session_tool_request_resolution_command_session_idx + ON session_tool_request_resolution_command (session_id, created_at) + `) + yield* tx.run(` + CREATE TABLE session_tool_request_resolution ( + resolution_id TEXT NOT NULL PRIMARY KEY, + receipt_id TEXT NOT NULL UNIQUE REFERENCES session_tool_request_receipt(receipt_id) ON DELETE CASCADE, + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + legacy_activity_id TEXT, + assistant_message_id TEXT NOT NULL REFERENCES message(id) ON DELETE RESTRICT, + source_prompt_epoch INTEGER NOT NULL, + source_window_id TEXT NOT NULL, + source_effective_history_hash TEXT NOT NULL, + source_request_hash TEXT NOT NULL, + source_mutation_epoch INTEGER NOT NULL, + expected_provider_state TEXT NOT NULL CHECK (expected_provider_state = 'indeterminate_after_crash'), + decision TEXT NOT NULL CHECK (decision = 'abandoned'), + actor_type TEXT NOT NULL CHECK (actor_type IN ('user', 'administrator', 'system-verifier')), + actor_id TEXT NOT NULL, + reason TEXT NOT NULL CHECK (length(reason) BETWEEN 1 AND 2000), + risk_acknowledged INTEGER NOT NULL CHECK (risk_acknowledged = 0), + safe_end_message_id TEXT REFERENCES message(id) ON DELETE RESTRICT, + safe_history_hash TEXT NOT NULL, + safe_message_ids TEXT NOT NULL, + ambiguity_message_id TEXT NOT NULL REFERENCES message(id) ON DELETE RESTRICT, + physical_message_high_water TEXT NOT NULL REFERENCES message(id) ON DELETE RESTRICT, + successor_prompt_epoch INTEGER NOT NULL, + successor_window_id TEXT NOT NULL, + successor_history_hash TEXT NOT NULL, + successor_mutation_epoch INTEGER NOT NULL, + created_at INTEGER NOT NULL, + FOREIGN KEY (session_id, source_prompt_epoch) + REFERENCES session_prompt_epoch(session_id, epoch) + ) + `) + yield* tx.run(` + CREATE INDEX session_tool_request_resolution_session_idx + ON session_tool_request_resolution (session_id, created_at) + `) + yield* tx.run(` + CREATE TABLE session_prompt_epoch_recovery ( + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + prompt_epoch INTEGER NOT NULL, + resolution_id TEXT NOT NULL UNIQUE, + source_prompt_epoch INTEGER NOT NULL, + source_mutation_epoch INTEGER NOT NULL, + successor_mutation_epoch INTEGER NOT NULL, + ambiguity_message_id TEXT NOT NULL REFERENCES message(id) ON DELETE RESTRICT, + physical_message_high_water TEXT NOT NULL REFERENCES message(id) ON DELETE RESTRICT, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, prompt_epoch), + FOREIGN KEY (session_id, prompt_epoch) + REFERENCES session_prompt_epoch(session_id, epoch) ON DELETE CASCADE, + FOREIGN KEY (session_id, source_prompt_epoch) + REFERENCES session_prompt_epoch(session_id, epoch), + FOREIGN KEY (resolution_id) + REFERENCES session_tool_request_resolution(resolution_id) + ) + `) + + yield* tx.run(` + CREATE TRIGGER session_tool_request_resolution_validate_insert + BEFORE INSERT ON session_tool_request_resolution + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_tool_request_receipt receipt + WHERE receipt.receipt_id = NEW.receipt_id + AND receipt.session_id = NEW.session_id + AND receipt.assistant_message_id = NEW.assistant_message_id + AND receipt.provider_attempt_id IS NULL + AND receipt.provider_state = NEW.expected_provider_state + AND receipt.prompt_epoch = NEW.source_prompt_epoch + AND receipt.prompt_window_id = NEW.source_window_id + AND receipt.effective_history_hash = NEW.source_effective_history_hash + AND COALESCE(receipt.final_request_hash, receipt.provider_request_hash, receipt.request_input_hash) = NEW.source_request_hash + ) THEN RAISE(ABORT, 'legacy_provider_resolution_receipt_binding_invalid') END; + SELECT CASE WHEN NEW.legacy_activity_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_activity_progress progress + JOIN session_legacy_activity activity ON activity.activity_id = progress.activity_id + WHERE progress.provider_receipt_id = NEW.receipt_id + AND progress.activity_id = NEW.legacy_activity_id + AND activity.session_id = NEW.session_id + AND progress.state = 'recovery_required' + AND activity.state = 'recovery_required' + ) THEN RAISE(ABORT, 'legacy_provider_resolution_activity_binding_invalid') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_prompt_epoch source + WHERE source.session_id = NEW.session_id + AND source.epoch = NEW.source_prompt_epoch + AND source.window_id = NEW.source_window_id + AND source.state = 'retired' + AND source.authority_state = 'recovery_required' + ) THEN RAISE(ABORT, 'legacy_provider_resolution_source_binding_invalid') END; + SELECT CASE WHEN NEW.successor_prompt_epoch <= NEW.source_prompt_epoch + THEN RAISE(ABORT, 'legacy_provider_resolution_successor_not_newer') END; + SELECT CASE WHEN NEW.successor_mutation_epoch != NEW.source_mutation_epoch + 1 + THEN RAISE(ABORT, 'legacy_provider_resolution_mutation_epoch_invalid') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_resolution_command_validate_insert + BEFORE INSERT ON session_tool_request_resolution_command + BEGIN + SELECT CASE WHEN NEW.result_resolution_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_tool_request_resolution resolution + WHERE resolution.resolution_id = NEW.result_resolution_id + AND resolution.session_id = NEW.session_id + AND resolution.receipt_id = NEW.receipt_id + ) THEN RAISE(ABORT, 'legacy_provider_resolution_command_result_invalid') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_resolution_immutable_update + BEFORE UPDATE ON session_tool_request_resolution + BEGIN + SELECT RAISE(ABORT, 'legacy_provider_resolution_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_resolution_immutable_delete + BEFORE DELETE ON session_tool_request_resolution + WHEN EXISTS (SELECT 1 FROM session WHERE id = OLD.session_id) + BEGIN + SELECT RAISE(ABORT, 'legacy_provider_resolution_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_resolution_command_immutable_update + BEFORE UPDATE ON session_tool_request_resolution_command + BEGIN + SELECT CASE WHEN OLD.result_resolution_id IS NOT NULL OR + NEW.command_id IS NOT OLD.command_id OR NEW.request_hash IS NOT OLD.request_hash OR + NEW.session_id IS NOT OLD.session_id OR NEW.receipt_id IS NOT OLD.receipt_id OR + NEW.created_at IS NOT OLD.created_at OR NEW.result_resolution_id IS NULL OR NOT EXISTS ( + SELECT 1 FROM session_tool_request_resolution resolution + WHERE resolution.resolution_id = NEW.result_resolution_id + AND resolution.session_id = NEW.session_id + AND resolution.receipt_id = NEW.receipt_id + ) + THEN RAISE(ABORT, 'legacy_provider_resolution_command_immutable') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_resolution_command_immutable_delete + BEFORE DELETE ON session_tool_request_resolution_command + WHEN EXISTS (SELECT 1 FROM session WHERE id = OLD.session_id) + BEGIN + SELECT RAISE(ABORT, 'legacy_provider_resolution_command_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_prompt_epoch_recovery_validate_insert + BEFORE INSERT ON session_prompt_epoch_recovery + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_prompt_epoch successor + WHERE successor.session_id = NEW.session_id + AND successor.epoch = NEW.prompt_epoch + AND successor.reason = 'recovery' + AND successor.authority_state = 'ready' + AND successor.recovery_resolution_id = NEW.resolution_id + AND successor.source_end_message_id = NEW.physical_message_high_water + ) THEN RAISE(ABORT, 'prompt_epoch_recovery_successor_invalid') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_prompt_epoch source + WHERE source.session_id = NEW.session_id + AND source.epoch = NEW.source_prompt_epoch + AND source.state = 'retired' + AND source.authority_state = 'recovery_required' + ) THEN RAISE(ABORT, 'prompt_epoch_recovery_source_invalid') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_tool_request_resolution resolution + WHERE resolution.resolution_id = NEW.resolution_id + AND resolution.session_id = NEW.session_id + AND resolution.source_prompt_epoch = NEW.source_prompt_epoch + AND resolution.source_mutation_epoch = NEW.source_mutation_epoch + ) THEN RAISE(ABORT, 'prompt_epoch_recovery_resolution_source_invalid') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_tool_request_resolution resolution + JOIN session_prompt_epoch successor + ON successor.session_id = NEW.session_id AND successor.epoch = NEW.prompt_epoch + WHERE resolution.resolution_id = NEW.resolution_id + AND resolution.session_id = NEW.session_id + AND resolution.successor_prompt_epoch = NEW.prompt_epoch + AND resolution.successor_window_id = successor.window_id + AND resolution.successor_history_hash = successor.effective_history_hash + AND resolution.safe_history_hash = successor.effective_history_hash + AND resolution.successor_mutation_epoch = NEW.successor_mutation_epoch + ) THEN RAISE(ABORT, 'prompt_epoch_recovery_resolution_successor_invalid') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_tool_request_resolution resolution + WHERE resolution.resolution_id = NEW.resolution_id + AND resolution.ambiguity_message_id = NEW.ambiguity_message_id + AND resolution.physical_message_high_water = NEW.physical_message_high_water + ) THEN RAISE(ABORT, 'prompt_epoch_recovery_resolution_boundary_invalid') END; + SELECT CASE WHEN NEW.successor_mutation_epoch != NEW.source_mutation_epoch + 1 + THEN RAISE(ABORT, 'prompt_epoch_recovery_mutation_epoch_invalid') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_prompt_epoch_recovery_immutable_update + BEFORE UPDATE ON session_prompt_epoch_recovery + BEGIN + SELECT RAISE(ABORT, 'prompt_epoch_recovery_binding_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_prompt_epoch_recovery_immutable_delete + BEFORE DELETE ON session_prompt_epoch_recovery + WHEN EXISTS (SELECT 1 FROM session WHERE id = OLD.session_id) + BEGIN + SELECT RAISE(ABORT, 'prompt_epoch_recovery_binding_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_parent_cleanup + AFTER DELETE ON session + BEGIN + DELETE FROM session_tool_request_receipt WHERE session_id = OLD.id; + END + `) + + yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_validate_insert") + yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_validate_update") + yield* tx.run(` + CREATE TRIGGER session_prompt_epoch_validate_insert + BEFORE INSERT ON session_prompt_epoch + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session WHERE id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_session_missing') END; + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.checkpoint_user_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM message WHERE id = NEW.checkpoint_user_id AND session_id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_checkpoint_user_cross_session') END; + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.checkpoint_assistant_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM message WHERE id = NEW.checkpoint_assistant_id AND session_id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_checkpoint_assistant_cross_session') END; + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.retained_tail_start_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM message WHERE id = NEW.retained_tail_start_id AND session_id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_retained_tail_cross_session') END; + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.source_end_message_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM message WHERE id = NEW.source_end_message_id AND session_id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_source_end_cross_session') END; + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.previous_window_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_prompt_epoch + WHERE session_id = NEW.session_id AND window_id = NEW.previous_window_id + ) THEN RAISE(ABORT, 'prompt_epoch_previous_window_cross_session') END; + SELECT CASE WHEN NEW.authority_state = 'ready' AND ( + NEW.projection_version IS NULL OR NEW.canonicalization_version IS NULL OR + NEW.base_message_count IS NULL OR NEW.base_message_count < 0 OR + NEW.effective_history_hash IS NULL OR NEW.first_window_id IS NULL OR NEW.window_id IS NULL OR + (NEW.recovery_resolution_id IS NOT NULL AND NEW.world_state_baseline_hash IS NULL) OR + (NEW.recovery_resolution_id IS NULL AND NEW.epoch > 0 AND ( + NEW.checkpoint_user_id IS NULL OR NEW.checkpoint_assistant_id IS NULL OR + NEW.checkpoint_hash IS NULL OR NEW.world_state_baseline_hash IS NULL + )) + ) THEN RAISE(ABORT, 'prompt_epoch_ready_binding_incomplete') END; + SELECT CASE WHEN NEW.recovery_resolution_id IS NOT NULL AND ( + NEW.authority_state != 'ready' OR NEW.checkpoint_user_id IS NOT NULL OR + NEW.checkpoint_assistant_id IS NOT NULL OR NEW.retained_tail_start_id IS NOT NULL OR + NEW.reason != 'recovery' + ) THEN RAISE(ABORT, 'prompt_epoch_recovery_binding_invalid') END; + SELECT CASE WHEN NEW.recovery_resolution_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_tool_request_resolution resolution + WHERE resolution.resolution_id = NEW.recovery_resolution_id + AND resolution.session_id = NEW.session_id + AND resolution.successor_prompt_epoch = NEW.epoch + ) THEN RAISE(ABORT, 'prompt_epoch_recovery_resolution_invalid') END; + SELECT CASE WHEN NEW.reason = 'recovery' AND NEW.recovery_resolution_id IS NULL + THEN RAISE(ABORT, 'prompt_epoch_recovery_resolution_missing') END; + SELECT CASE WHEN NEW.authority_state = 'recovery_required' AND NEW.recovery_reason IS NULL + THEN RAISE(ABORT, 'prompt_epoch_recovery_without_reason') END; + SELECT CASE WHEN NEW.authority_state != 'recovery_required' AND NEW.recovery_reason IS NOT NULL + THEN RAISE(ABORT, 'prompt_epoch_nonrecovery_with_reason') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_prompt_epoch_validate_update + BEFORE UPDATE ON session_prompt_epoch + BEGIN + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.checkpoint_user_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM message WHERE id = NEW.checkpoint_user_id AND session_id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_checkpoint_user_cross_session') END; + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.checkpoint_assistant_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM message WHERE id = NEW.checkpoint_assistant_id AND session_id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_checkpoint_assistant_cross_session') END; + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.retained_tail_start_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM message WHERE id = NEW.retained_tail_start_id AND session_id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_retained_tail_cross_session') END; + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.source_end_message_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM message WHERE id = NEW.source_end_message_id AND session_id = NEW.session_id + ) THEN RAISE(ABORT, 'prompt_epoch_source_end_cross_session') END; + SELECT CASE WHEN NEW.authority_state IS NOT 'recovery_required' AND + NEW.previous_window_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_prompt_epoch + WHERE session_id = NEW.session_id AND window_id = NEW.previous_window_id + ) THEN RAISE(ABORT, 'prompt_epoch_previous_window_cross_session') END; + SELECT CASE WHEN OLD.authority_state = 'ready' AND ( + NEW.session_id IS NOT OLD.session_id OR NEW.epoch IS NOT OLD.epoch OR + NEW.checkpoint_user_id IS NOT OLD.checkpoint_user_id OR + NEW.checkpoint_assistant_id IS NOT OLD.checkpoint_assistant_id OR + NEW.retained_tail_start_id IS NOT OLD.retained_tail_start_id OR + NEW.source_end_message_id IS NOT OLD.source_end_message_id OR + NEW.checkpoint_hash IS NOT OLD.checkpoint_hash OR + NEW.projection_version IS NOT OLD.projection_version OR + NEW.canonicalization_version IS NOT OLD.canonicalization_version OR + NEW.base_message_count IS NOT OLD.base_message_count OR + NEW.effective_history_hash IS NOT OLD.effective_history_hash OR + NEW.first_window_id IS NOT OLD.first_window_id OR + NEW.previous_window_id IS NOT OLD.previous_window_id OR + NEW.window_id IS NOT OLD.window_id OR + NEW.world_state_baseline_hash IS NOT OLD.world_state_baseline_hash OR + NEW.recovery_resolution_id IS NOT OLD.recovery_resolution_id OR NEW.reason IS NOT OLD.reason + ) THEN RAISE(ABORT, 'prompt_epoch_ready_binding_immutable') END; + SELECT CASE WHEN OLD.authority_state = 'recovery_required' AND NEW.authority_state != 'recovery_required' + THEN RAISE(ABORT, 'prompt_epoch_recovery_state_immutable') END; + SELECT CASE WHEN OLD.authority_state = 'ready' AND NEW.authority_state NOT IN ('ready', 'recovery_required') + THEN RAISE(ABORT, 'prompt_epoch_invalid_authority_transition') END; + SELECT CASE WHEN NEW.authority_state = 'ready' AND ( + NEW.projection_version IS NULL OR NEW.canonicalization_version IS NULL OR + NEW.base_message_count IS NULL OR NEW.base_message_count < 0 OR + NEW.effective_history_hash IS NULL OR NEW.first_window_id IS NULL OR NEW.window_id IS NULL OR + (NEW.recovery_resolution_id IS NOT NULL AND NEW.world_state_baseline_hash IS NULL) OR + (NEW.recovery_resolution_id IS NULL AND NEW.epoch > 0 AND ( + NEW.checkpoint_user_id IS NULL OR NEW.checkpoint_assistant_id IS NULL OR + NEW.checkpoint_hash IS NULL OR NEW.world_state_baseline_hash IS NULL + )) + ) THEN RAISE(ABORT, 'prompt_epoch_ready_binding_incomplete') END; + SELECT CASE WHEN NEW.recovery_resolution_id IS NOT NULL AND ( + NEW.authority_state != 'ready' OR NEW.checkpoint_user_id IS NOT NULL OR + NEW.checkpoint_assistant_id IS NOT NULL OR NEW.retained_tail_start_id IS NOT NULL OR + NEW.reason != 'recovery' + ) THEN RAISE(ABORT, 'prompt_epoch_recovery_binding_invalid') END; + SELECT CASE WHEN NEW.recovery_resolution_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_tool_request_resolution resolution + WHERE resolution.resolution_id = NEW.recovery_resolution_id + AND resolution.session_id = NEW.session_id + AND resolution.successor_prompt_epoch = NEW.epoch + ) THEN RAISE(ABORT, 'prompt_epoch_recovery_resolution_invalid') END; + SELECT CASE WHEN NEW.reason = 'recovery' AND NEW.recovery_resolution_id IS NULL + THEN RAISE(ABORT, 'prompt_epoch_recovery_resolution_missing') END; + SELECT CASE WHEN NEW.authority_state = 'recovery_required' AND NEW.recovery_reason IS NULL + THEN RAISE(ABORT, 'prompt_epoch_recovery_without_reason') END; + SELECT CASE WHEN NEW.authority_state != 'recovery_required' AND NEW.recovery_reason IS NOT NULL + THEN RAISE(ABORT, 'prompt_epoch_nonrecovery_with_reason') END; + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812140000_session_diff_manifest.ts b/packages/core/src/database/migration/20260812140000_session_diff_manifest.ts new file mode 100644 index 000000000..355aeb55d --- /dev/null +++ b/packages/core/src/database/migration/20260812140000_session_diff_manifest.ts @@ -0,0 +1,11 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812140000_session_diff_manifest", + up(tx) { + return Effect.gen(function* () { + yield* tx.run("ALTER TABLE session ADD COLUMN summary_diff_manifest TEXT") + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index 30b141685..9a5307fb5 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -1,7 +1,7 @@ export * as EventV2 from "./event" import { Cause, Context, Effect, Layer, Option, PubSub, Schema, Stream } from "effect" -import { and, asc, eq, gt } from "drizzle-orm" +import { and, eq, gt, inArray, sql } from "drizzle-orm" import { Database } from "./database/database" import { EventSequenceTable, EventTable } from "./event/sql" import { Location } from "./location" @@ -78,6 +78,20 @@ export class InvalidSyncEventError extends Schema.TaggedErrorClass()( + "EventV2.EncodedPayloadTooLarge", + { + type: Schema.String, + encodedBytes: NonNegativeInt, + limitBytes: NonNegativeInt, + message: Schema.String, + }, +) {} + export function versionedType(type: string, version: number) { return `${type}.${version}` } @@ -90,6 +104,21 @@ type SyncDefinition = Definition & { } const syncRegistry = new Map() +function admitEncodedPayload(definition: SyncDefinition, data: unknown) { + const encoded = definition.encode(data) as Record + const encodedBytes = Buffer.byteLength(JSON.stringify(encoded)) + if (encodedBytes > MAX_ENCODED_PAYLOAD_BYTES) + return { + error: new EncodedPayloadTooLargeError({ + type: definition.type, + encodedBytes, + limitBytes: MAX_ENCODED_PAYLOAD_BYTES, + message: `Encoded event payload is ${encodedBytes} bytes; limit is ${MAX_ENCODED_PAYLOAD_BYTES} bytes`, + }), + } as const + return { encoded } as const +} + // Synchronized events cross a JSON boundary, so their data schemas must encode and decode without services. const syncCodec = (definition: Definition) => definition.data as Schema.Codec @@ -182,6 +211,7 @@ export class Service extends Context.Service()("@deepagent-c export interface LayerOptions { readonly beforeAggregateRead?: (aggregateID: string) => Effect.Effect + readonly afterAggregateReadMetadata?: (aggregateID: string, eventIDs: readonly ID[]) => Effect.Effect } export const layerWith = (options?: LayerOptions) => @@ -258,6 +288,12 @@ export const layerWith = (options?: LayerOptions) => }), ) } + const admission = admitEncodedPayload( + syncRegistry.get(versionedType(definition.type, sync.version))!, + event.data, + ) + if ("error" in admission) return yield* Effect.die(admission.error) + const encoded = admission.encoded const list = projectors.get(event.type) ?? [] return yield* Effect.uninterruptible( Effect.gen(function* () { @@ -272,9 +308,6 @@ export const layerWith = (options?: LayerOptions) => .get() .pipe(Effect.orDie) const latest = row?.seq ?? -1 - const encoded = syncRegistry - .get(versionedType(definition.type, sync.version))! - .encode(event.data) as Record if (input?.strictOwner && row?.ownerID && row.ownerID !== input.ownerID) { yield* Effect.die( new InvalidSyncEventError({ @@ -549,6 +582,17 @@ export const layerWith = (options?: LayerOptions) => ) } } + for (const event of events) { + const definition = syncRegistry.get(event.type) + if (!definition) { + yield* Effect.die( + new InvalidSyncEventError({ type: event.type, message: `Unknown sync event type ${event.type}` }), + ) + continue + } + const admission = admitEncodedPayload(definition, definition.decode(event.data)) + if ("error" in admission) return yield* Effect.die(admission.error) + } for (const event of events) { yield* replay(event, options) } @@ -600,29 +644,85 @@ export const layerWith = (options?: LayerOptions) => } } + const decodeStoredEventData = Schema.decodeUnknownSync(Schema.UnknownFromJsonString) + const readAfter = (aggregateID: string, after: number) => - (options?.beforeAggregateRead?.(aggregateID) ?? Effect.void).pipe( - Effect.andThen( - db - .select() - .from(EventTable) - .where(and(eq(EventTable.aggregate_id, aggregateID), gt(EventTable.seq, after))) - .orderBy(asc(EventTable.seq)) - .all(), - ), - Effect.orDie, - Effect.map((rows) => - rows.map((event) => - decodeSerializedEvent({ - id: event.id, - aggregateID: event.aggregate_id, - seq: event.seq, - type: event.type, - data: event.data, + Effect.gen(function* () { + yield* options?.beforeAggregateRead?.(aggregateID) ?? Effect.void + const candidates = yield* db + .all<{ + id: ID + seq: number + type: string + bytes: number + cumulativeBytes: number + }>(sql` + WITH candidate AS ( + SELECT + ${EventTable.id} AS id, + ${EventTable.seq} AS seq, + ${EventTable.type} AS type, + length(CAST(${EventTable.data} AS BLOB)) AS bytes + FROM ${EventTable} + WHERE ${EventTable.aggregate_id} = ${aggregateID} + AND ${EventTable.seq} > ${after} + ORDER BY ${EventTable.seq} ASC + LIMIT ${AGGREGATE_READ_BATCH_EVENTS + 1} + ) + SELECT + id, + seq, + type, + bytes, + sum(bytes) OVER (ORDER BY seq ASC ROWS UNBOUNDED PRECEDING) AS cumulativeBytes + FROM candidate + ORDER BY seq ASC + `) + .pipe(Effect.orDie) + const oversized = candidates[0] + if (oversized && oversized.bytes > AGGREGATE_READ_BATCH_BYTES) + return yield* Effect.die( + new EncodedPayloadTooLargeError({ + type: oversized.type, + encodedBytes: oversized.bytes, + limitBytes: AGGREGATE_READ_BATCH_BYTES, + message: `Stored event payload is ${oversized.bytes} bytes; aggregate read limit is ${AGGREGATE_READ_BATCH_BYTES} bytes`, }), - ), - ), - ) + ) + const metadata = candidates + .slice(0, AGGREGATE_READ_BATCH_EVENTS) + .filter((event) => event.cumulativeBytes <= AGGREGATE_READ_BATCH_BYTES) + if (metadata.length === 0) return { events: [], more: false } + yield* options?.afterAggregateReadMetadata?.( + aggregateID, + metadata.map((event) => event.id), + ) ?? Effect.void + const rows = yield* db + .select({ id: EventTable.id, data: sql`${EventTable.data}` }) + .from(EventTable) + .where(inArray(EventTable.id, metadata.map((event) => event.id))) + .all() + .pipe(Effect.orDie) + const dataByID = new Map(rows.map((row) => [row.id, row.data])) + const missing = metadata.find((event) => !dataByID.has(event.id)) + if (missing) + return yield* Effect.die( + new InvalidSyncEventError({ + type: missing.type, + message: `Stored event ${missing.id} disappeared while reading aggregate ${aggregateID}`, + }), + ) + const events = metadata.map((event) => + decodeSerializedEvent({ + id: event.id, + aggregateID, + seq: event.seq, + type: event.type, + data: decodeStoredEventData(dataByID.get(event.id)!) as Record, + }), + ) + return { events, more: candidates.length > metadata.length } + }) const subscribeSynchronized = (aggregateID: string) => Effect.gen(function* () { @@ -652,19 +752,23 @@ export const layerWith = (options?: LayerOptions) => Effect.gen(function* () { const synchronized = yield* subscribeSynchronized(input.aggregateID) let cursor = input.after ?? -1 - const read = Effect.suspend(() => readAfter(input.aggregateID, cursor)).pipe( - Effect.tap((events) => - Effect.sync(() => { - cursor = events.at(-1)?.cursor ?? cursor - }), - ), - ) - const historical = yield* read + const drain = () => + Stream.paginate(cursor, (after) => + readAfter(input.aggregateID, after).pipe( + Effect.map((page) => { + const next = page.events.at(-1)?.cursor + if (next !== undefined) cursor = next + return [ + page.events, + page.more && next !== undefined ? Option.some(next) : Option.none(), + ] as const + }), + ), + ) const live = Stream.fromSubscription(synchronized).pipe( - Stream.mapEffect(() => read), - Stream.flattenIterable, + Stream.flatMap(() => drain()), ) - return Stream.concat(Stream.fromIterable(historical), live) + return Stream.concat(drain(), live) }), ) diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index 5ad0462ab..78f22271e 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -60,6 +60,9 @@ function sessionRow(info: SessionV1.SessionInfo): typeof SessionTable.$inferInse summary_deletions: info.summary?.deletions, summary_files: info.summary?.files, summary_diffs: info.summary?.diffs ? [...info.summary.diffs] : undefined, + summary_diff_manifest: info.summary?.diffManifest + ? { ...info.summary.diffManifest, truncationReasons: [...info.summary.diffManifest.truncationReasons] } + : undefined, metadata: info.metadata, cost: info.cost ?? 0, tokens_input: (info.tokens ?? { input: 0 }).input, @@ -77,6 +80,13 @@ function sessionRow(info: SessionV1.SessionInfo): typeof SessionTable.$inferInse } } +function sessionUpdateRow(info: SessionV1.SessionInfo) { + const row = sessionRow(info) + if (info.summary?.diffs !== undefined) return row + const { summary_diffs: _, ...bounded } = row + return bounded +} + function messageData( info: (typeof SessionV1.Event.MessageUpdated.Type)["data"]["info"], ): typeof MessageTable.$inferInsert.data { @@ -242,7 +252,7 @@ export const layer = Layer.effectDiscard( yield* events.project(SessionV1.Event.Updated, (event) => db .update(SessionTable) - .set(sessionRow(event.data.info)) + .set(sessionUpdateRow(event.data.info)) .where(eq(SessionTable.id, event.data.sessionID)) .run() .pipe(Effect.orDie), diff --git a/packages/core/src/session/sql.ts b/packages/core/src/session/sql.ts index c76ddbb54..105eec510 100644 --- a/packages/core/src/session/sql.ts +++ b/packages/core/src/session/sql.ts @@ -48,6 +48,7 @@ export const SessionTable = sqliteTable( summary_deletions: integer(), summary_files: integer(), summary_diffs: text({ mode: "json" }).$type(), + summary_diff_manifest: text({ mode: "json" }).$type(), metadata: text({ mode: "json" }).$type>(), cost: real().notNull().default(0), tokens_input: integer().notNull().default(0), @@ -414,7 +415,9 @@ export const SessionWorldStateBaselineTable = sqliteTable( snapshot: text({ mode: "json" }).$type().notNull(), fragment: text().notNull(), fragment_hash: text().notNull(), - provenance: text().$type<"native" | "fork_rebuilt" | "legacy_migration">().notNull(), + provenance: text() + .$type<"native" | "fork_rebuilt" | "legacy_migration" | "recovery_copied" | "recovery_recollected">() + .notNull(), created_at: integer().notNull(), }, (table) => [ @@ -423,6 +426,68 @@ export const SessionWorldStateBaselineTable = sqliteTable( ], ) +export const SessionToolRequestResolutionCommandTable = sqliteTable( + "session_tool_request_resolution_command", + { + command_id: text().primaryKey(), + request_hash: text().notNull(), + session_id: text().$type().notNull(), + receipt_id: text().notNull(), + result_resolution_id: text(), + created_at: integer().notNull(), + }, + (table) => [index("session_tool_request_resolution_command_session_idx").on(table.session_id, table.created_at)], +) + +export const SessionToolRequestResolutionTable = sqliteTable( + "session_tool_request_resolution", + { + resolution_id: text().primaryKey(), + receipt_id: text().notNull().unique(), + session_id: text().$type().notNull(), + legacy_activity_id: text(), + assistant_message_id: text().$type().notNull(), + source_prompt_epoch: integer().notNull(), + source_window_id: text().notNull(), + source_effective_history_hash: text().notNull(), + source_request_hash: text().notNull(), + source_mutation_epoch: integer().notNull(), + expected_provider_state: text().$type<"indeterminate_after_crash">().notNull(), + decision: text().$type<"abandoned">().notNull(), + actor_type: text().$type<"user" | "administrator" | "system-verifier">().notNull(), + actor_id: text().notNull(), + reason: text().notNull(), + risk_acknowledged: integer({ mode: "boolean" }).notNull(), + safe_end_message_id: text().$type(), + safe_history_hash: text().notNull(), + safe_message_ids: text({ mode: "json" }).$type().notNull(), + ambiguity_message_id: text().$type().notNull(), + physical_message_high_water: text().$type().notNull(), + successor_prompt_epoch: integer().notNull(), + successor_window_id: text().notNull(), + successor_history_hash: text().notNull(), + successor_mutation_epoch: integer().notNull(), + created_at: integer().notNull(), + }, + (table) => [index("session_tool_request_resolution_session_idx").on(table.session_id, table.created_at)], +) + +export const SessionPromptEpochRecoveryTable = sqliteTable( + "session_prompt_epoch_recovery", + { + session_id: text().$type().notNull(), + prompt_epoch: integer().notNull(), + resolution_id: text().notNull().unique(), + source_prompt_epoch: integer().notNull(), + source_mutation_epoch: integer().notNull(), + successor_mutation_epoch: integer().notNull(), + ambiguity_message_id: text().$type().notNull(), + physical_message_high_water: text().$type().notNull(), + created_at: integer().notNull(), + }, + (table) => [primaryKey({ columns: [table.session_id, table.prompt_epoch] })], +) + export const SessionContextEpochTable = sqliteTable("session_context_epoch", { session_id: text() .$type() diff --git a/packages/core/src/snapshot.ts b/packages/core/src/snapshot.ts index b39c0f7f0..4803dbaea 100644 --- a/packages/core/src/snapshot.ts +++ b/packages/core/src/snapshot.ts @@ -6,4 +6,26 @@ export namespace Snapshot { deletions: number status?: "added" | "deleted" | "modified" } + + export type DiffManifestDescriptor = { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } } diff --git a/packages/core/src/v1/session.ts b/packages/core/src/v1/session.ts index 9594ee8b2..6c1bb2772 100644 --- a/packages/core/src/v1/session.ts +++ b/packages/core/src/v1/session.ts @@ -355,6 +355,30 @@ const FileDiff = Schema.Struct({ status: Schema.optional(Schema.Literals(["added", "deleted", "modified"])), }).annotate({ identifier: "SnapshotFileDiff" }) +const DiffManifestDescriptor = Schema.Struct({ + completeness: Schema.Literals(["complete", "truncated"]), + truncationReasons: Schema.Array( + Schema.Literals([ + "candidate_file_limit", + "discovery_output_limit", + "discovery_failed", + "manifest_bytes_limit", + "source_file_limit", + "source_total_limit", + "patch_file_limit", + "patch_total_limit", + "materialization_failed", + "time_limit", + ]), + ), + manifestHash: Schema.String, + totalFiles: NonNegativeInt, + totalFilesExact: Schema.Boolean, + statisticsExact: optionalOmitUndefined(Schema.Boolean), + includedFiles: NonNegativeInt, + truncatedFiles: NonNegativeInt, +}).annotate({ identifier: "SnapshotDiffManifestDescriptor" }) + export const User = Schema.Struct({ ...messageBase, role: Schema.Literal("user"), @@ -367,6 +391,7 @@ export const User = Schema.Struct({ title: Schema.optional(Schema.String), body: Schema.optional(Schema.String), diffs: Schema.Array(FileDiff), + diffManifest: optionalOmitUndefined(DiffManifestDescriptor), }), ), agent: Schema.String, @@ -558,6 +583,7 @@ const SessionSummary = Schema.Struct({ deletions: Schema.Finite, files: Schema.Finite, diffs: optionalOmitUndefined(Schema.Array(FileDiff)), + diffManifest: optionalOmitUndefined(DiffManifestDescriptor), }) const SessionTokens = Schema.Struct({ diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index f64e502e9..d005ce4e6 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -83,6 +83,26 @@ const SyncTimestamp = EventV2.define({ }, }) +const SyncPayload = EventV2.define({ + type: "test.payload", + sync: { + version: 1, + aggregate: "sessionID", + }, + schema: { + sessionID: Schema.String, + kind: Schema.Literals(["session", "message"]), + body: Schema.String, + }, +}) + +function payloadAtEncodedBytes(sessionID: string, kind: "session" | "message", bytes: number, suffix = "") { + const input = { sessionID, kind, body: suffix } + const remaining = bytes - Buffer.byteLength(JSON.stringify(input)) + if (remaining < 0) throw new Error(`Payload envelope exceeds requested size ${bytes}`) + return { ...input, body: "x".repeat(remaining) + suffix } +} + describe("EventV2", () => { it.effect("derives stable namespaced external IDs", () => Effect.sync(() => { @@ -207,6 +227,156 @@ describe("EventV2", () => { }), ) + it.effect("rejects oversized session-like publishes before every durable side effect", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const aggregateID = EventV2.ID.create() + const observed: string[] = [] + yield* events.beforeCommit(() => Effect.sync(() => observed.push("guard"))) + yield* events.project(SyncPayload, () => Effect.sync(() => observed.push("projector"))) + + yield* events.publish(SyncPayload, { sessionID: aggregateID, kind: "session", body: "baseline" }) + observed.length = 0 + const sequenceBefore = yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, aggregateID)) + .all() + const eventsBefore = yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, aggregateID)).all() + + const defect = yield* events + .publish( + SyncPayload, + payloadAtEncodedBytes(aggregateID, "session", EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1), + { commit: () => Effect.sync(() => observed.push("commit")) }, + ) + .pipe(Effect.catchDefect(Effect.succeed)) + + expect(defect).toBeInstanceOf(EventV2.EncodedPayloadTooLargeError) + expect(defect).toMatchObject({ + _tag: "EventV2.EncodedPayloadTooLarge", + type: SyncPayload.type, + encodedBytes: EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1, + limitBytes: EventV2.MAX_ENCODED_PAYLOAD_BYTES, + }) + expect(observed).toEqual([]) + expect( + yield* db.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, aggregateID)).all(), + ).toEqual(sequenceBefore) + expect(yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, aggregateID)).all()).toEqual( + eventsBefore, + ) + }), + ) + + it.effect("rejects oversized message-like replay before every durable side effect", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const aggregateID = EventV2.ID.create() + const observed: string[] = [] + yield* events.beforeCommit(() => Effect.sync(() => observed.push("guard"))) + yield* events.project(SyncPayload, () => Effect.sync(() => observed.push("projector"))) + + const defect = yield* events + .replay({ + id: EventV2.ID.create(), + type: EventV2.versionedType(SyncPayload.type, 1), + seq: 0, + aggregateID, + data: payloadAtEncodedBytes(aggregateID, "message", EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1), + }) + .pipe(Effect.catchDefect(Effect.succeed)) + + expect(defect).toBeInstanceOf(EventV2.EncodedPayloadTooLargeError) + expect(defect).toMatchObject({ + _tag: "EventV2.EncodedPayloadTooLarge", + type: SyncPayload.type, + encodedBytes: EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1, + }) + expect(observed).toEqual([]) + expect( + yield* db.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, aggregateID)).all(), + ).toEqual([]) + expect(yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, aggregateID)).all()).toEqual([]) + }), + ) + + it.effect("admits an exact UTF-8 byte boundary and preserves idempotent publish and replay", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const aggregateID = EventV2.ID.create() + const eventID = EventV2.ID.create() + const data = payloadAtEncodedBytes(aggregateID, "message", EventV2.MAX_ENCODED_PAYLOAD_BYTES, "\u{1F680}") + expect(Buffer.byteLength(JSON.stringify(data))).toBe(EventV2.MAX_ENCODED_PAYLOAD_BYTES) + let commits = 0 + + const first = yield* events.publish(SyncPayload, data, { + id: eventID, + idempotent: true, + commit: () => Effect.sync(() => commits++), + }) + const retry = yield* events.publish(SyncPayload, data, { + id: eventID, + idempotent: true, + commit: () => Effect.sync(() => commits++), + }) + yield* events.replay({ + id: eventID, + type: EventV2.versionedType(SyncPayload.type, 1), + seq: first.seq!, + aggregateID, + data, + }) + + const rows = yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, aggregateID)).all() + expect(first.seq).toBe(0) + expect(retry.seq).toBe(0) + expect(commits).toBe(2) + expect(rows).toHaveLength(1) + expect(rows[0]?.data).toEqual(data) + }), + ) + + it.effect("preflights a replay batch before committing a valid prefix", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const aggregateID = EventV2.ID.create() + const observed: string[] = [] + yield* events.beforeCommit(() => Effect.sync(() => observed.push("guard"))) + yield* events.project(SyncPayload, () => Effect.sync(() => observed.push("projector"))) + + const error = yield* events + .replayAll([ + { + id: EventV2.ID.create(), + type: EventV2.versionedType(SyncPayload.type, 1), + seq: 0, + aggregateID, + data: { sessionID: aggregateID, kind: "message", body: "valid prefix" }, + }, + { + id: EventV2.ID.create(), + type: EventV2.versionedType(SyncPayload.type, 1), + seq: 1, + aggregateID, + data: payloadAtEncodedBytes(aggregateID, "message", EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1), + }, + ]) + .pipe(Effect.catchDefect(Effect.succeed)) + + expect(error).toBeInstanceOf(EventV2.EncodedPayloadTooLargeError) + expect(observed).toEqual([]) + expect( + yield* db.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, aggregateID)).all(), + ).toEqual([]) + expect(yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, aggregateID)).all()).toEqual([]) + }), + ) + it.effect("rolls back the synchronized event and projector when the local commit fails", () => Effect.gen(function* () { const events = yield* EventV2.Service @@ -568,6 +738,124 @@ describe("EventV2", () => { }), ) + it.effect( + "pages aggregate history without gaps beyond the read batch limit", + () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const aggregateID = EventV2.ID.create() + const count = EventV2.AGGREGATE_READ_BATCH_EVENTS + 28 + for (let index = 0; index < count; index++) { + yield* events.publish(SyncMessage, { id: aggregateID, text: String(index) }) + } + + expect( + Array.from( + yield* events.aggregateEvents({ aggregateID }).pipe(Stream.take(count), Stream.runCollect), + ).map((event) => event.cursor), + ).toEqual(Array.from({ length: count }, (_, index) => EventV2.Cursor.make(index))) + }), + 15_000, + ) + + it.effect( + "drains aggregate history across byte-bounded pages without a live wake", + () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const aggregateID = EventV2.ID.create() + const bytes = Math.floor(EventV2.AGGREGATE_READ_BATCH_BYTES * 0.625) + yield* events.publish(SyncPayload, payloadAtEncodedBytes(aggregateID, "message", bytes, "first")) + yield* events.publish(SyncPayload, payloadAtEncodedBytes(aggregateID, "message", bytes, "second")) + + const received = Array.from( + yield* events.aggregateEvents({ aggregateID }).pipe(Stream.take(2), Stream.runCollect), + ) + expect(received.map((event) => event.cursor)).toEqual([ + EventV2.Cursor.make(0), + EventV2.Cursor.make(1), + ]) + expect(received.map((event) => (event.event.data as { body: string }).body.slice(-6))).toEqual([ + "xfirst", + "second", + ]) + }), + 15_000, + ) + + it.effect( + "keeps byte-bounded cursors contiguous when an event commits before the next page read", + () => + Effect.gen(function* () { + const secondReadStarted = yield* Deferred.make() + const continueSecondRead = yield* Deferred.make() + let reads = 0 + const database = Database.layerFromPath(":memory:") + const eventLayer = EventV2.layerWith({ + beforeAggregateRead: () => { + reads++ + if (reads !== 2) return Effect.void + return Deferred.succeed(secondReadStarted, undefined).pipe( + Effect.andThen(Deferred.await(continueSecondRead)), + ) + }, + }).pipe(Layer.provide(database)) + + yield* Effect.gen(function* () { + const events = yield* EventV2.Service + const aggregateID = EventV2.ID.create() + const bytes = Math.floor(EventV2.AGGREGATE_READ_BATCH_BYTES * 0.625) + yield* events.publish(SyncPayload, payloadAtEncodedBytes(aggregateID, "message", bytes, "first")) + yield* events.publish(SyncPayload, payloadAtEncodedBytes(aggregateID, "message", bytes, "second")) + const fiber = yield* events + .aggregateEvents({ aggregateID }) + .pipe(Stream.take(3), Stream.runCollect, Effect.forkScoped) + yield* Deferred.await(secondReadStarted) + + yield* events.publish(SyncPayload, { + sessionID: aggregateID, + kind: "message", + body: "committed-before-second-read", + }) + yield* Deferred.succeed(continueSecondRead, undefined) + + expect(Array.from(yield* Fiber.join(fiber)).map((event) => event.cursor)).toEqual([ + EventV2.Cursor.make(0), + EventV2.Cursor.make(1), + EventV2.Cursor.make(2), + ]) + }).pipe(Effect.provide(Layer.mergeAll(database, eventLayer))) + }), + 15_000, + ) + + it.effect( + "fails with a typed defect when aggregate history is removed between metadata and body reads", + () => { + let events: EventV2.Interface + const database = Database.layerFromPath(":memory:") + const eventLayer = EventV2.layerWith({ + afterAggregateReadMetadata: (aggregateID) => events.remove(aggregateID), + }).pipe(Layer.provide(database)) + + return Effect.gen(function* () { + events = yield* EventV2.Service + const aggregateID = EventV2.ID.create() + yield* events.publish(SyncMessage, { id: aggregateID, text: "removed during read" }) + const defect = yield* events.aggregateEvents({ aggregateID }).pipe( + Stream.take(1), + Stream.runCollect, + Effect.catchDefect(Effect.succeed), + ) + expect(defect).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect(defect).toMatchObject({ + _tag: "EventV2.InvalidSyncEvent", + type: EventV2.versionedType(SyncMessage.type, 1), + }) + }).pipe(Effect.provide(Layer.mergeAll(database, eventLayer))) + }, + ) + it.effect("omits live-only events from durable aggregate streams", () => Effect.gen(function* () { const events = yield* EventV2.Service diff --git a/packages/core/test/legacy-provider-recovery-delete-cascade.test.ts b/packages/core/test/legacy-provider-recovery-delete-cascade.test.ts new file mode 100644 index 000000000..a98bb7efd --- /dev/null +++ b/packages/core/test/legacy-provider-recovery-delete-cascade.test.ts @@ -0,0 +1,495 @@ +import { describe, expect, test } from "bun:test" +import { SqliteClient } from "@effect/sql-sqlite-bun" +import { Database } from "@deepagent-code/core/database/database" +import { DatabaseMigration } from "@deepagent-code/core/database/migration" +import { migrations } from "@deepagent-code/core/database/migration.gen" +import { EffectDrizzleSqlite } from "@deepagent-code/effect-drizzle-sqlite" +import { Effect, Exit } from "effect" +import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" +import { testEffect } from "./lib/effect" + +const it = testEffect(Database.layerFromPath(":memory:")) +const runMigration = (effect: Effect.Effect) => + Effect.runPromise( + effect.pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), Effect.scoped), + ) + +const authorityRows = Effect.gen(function* () { + const { db } = yield* Database.Service + return yield* db.all<{ table_name: string; row_count: number }>(` + SELECT 'session' AS table_name, count(*) AS row_count + FROM session WHERE id = 'ses_recovery_delete' + UNION ALL SELECT 'session_intent', count(*) + FROM session_intent WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_activity_admission', count(*) + FROM session_activity_admission WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_legacy_activity', count(*) + FROM session_legacy_activity WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_legacy_activity_admission', count(*) + FROM session_legacy_activity_admission WHERE activity_id = 'activity-recovery-delete' + UNION ALL SELECT 'session_activity_progress', count(*) + FROM session_activity_progress WHERE activity_id = 'activity-recovery-delete' + UNION ALL SELECT 'session_tool_request_receipt', count(*) + FROM session_tool_request_receipt WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_legacy_activity_terminal', count(*) + FROM session_legacy_activity_terminal WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_legacy_activity_migration_receipt', count(*) + FROM session_legacy_activity_migration_receipt WHERE activity_id = 'activity-recovery-delete' + UNION ALL SELECT 'session_prompt_epoch', count(*) + FROM session_prompt_epoch WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_prompt_epoch_message', count(*) + FROM session_prompt_epoch_message WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_world_state_baseline', count(*) + FROM session_world_state_baseline WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_tool_request_resolution_command', count(*) + FROM session_tool_request_resolution_command WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_tool_request_resolution', count(*) + FROM session_tool_request_resolution WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_prompt_epoch_recovery', count(*) + FROM session_prompt_epoch_recovery WHERE session_id = 'ses_recovery_delete' + UNION ALL SELECT 'session_history_state', count(*) + FROM session_history_state WHERE session_id = 'ses_recovery_delete' + `) +}) + +describe("legacy provider recovery delete cascade", () => { + test("rebuilds receipt ownership without losing valid argument or activity evidence", async () => { + await runMigration( + Effect.gen(function* () { + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + const recoveryIndex = migrations.findIndex((migration) => migration.id === "20260812120000_legacy_provider_recovery") + expect(recoveryIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, recoveryIndex)) + + yield* db.run(` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('project-receipt-upgrade', '/repo', '[]', 1, 1) + `) + yield* db.run(` + INSERT INTO session ( + id, project_id, slug, directory, title, version, time_created, time_updated + ) VALUES ( + 'ses_receipt_upgrade', 'project-receipt-upgrade', 'receipt-upgrade', '/repo', + 'Receipt upgrade', '1', 1, 1 + ) + `) + yield* db.run(` + INSERT INTO message (id, session_id, time_created, time_updated, data) + VALUES ('msg_receipt_upgrade', 'ses_receipt_upgrade', 1, 1, '{"role":"assistant"}') + `) + yield* db.run(` + INSERT INTO session_intent ( + intent_id, session_id, source, state, selected_variant, selected_payload_hash, + delivery, admitted_message_id, mutation_epoch, + time_created, time_selected, time_admitted, time_updated + ) VALUES ( + 'intent-receipt-upgrade', 'ses_receipt_upgrade', 'composer', 'admitted', 'original', + 'payload-receipt-upgrade', 'turn', 'msg_receipt_upgrade', 0, + 1, 1, 1, 1 + ) + `) + yield* db.run(` + INSERT INTO session_activity_admission ( + admission_id, session_id, source_kind, legacy_intent_id, admitted_message_id, + delivery, payload_fingerprint_kind, payload_fingerprint, created_at + ) VALUES ( + 'admission-receipt-upgrade', 'ses_receipt_upgrade', 'legacy_intent', + 'intent-receipt-upgrade', 'msg_receipt_upgrade', 'turn', 'payload_hash', + 'payload-receipt-upgrade', 1 + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity ( + activity_id, session_id, ordinal, trigger_admission_id, owner_token, + state, terminal_reason, created_at, settled_at + ) VALUES ( + 'activity-receipt-upgrade', 'ses_receipt_upgrade', 0, 'admission-receipt-upgrade', + 'owner-receipt-upgrade', 'active', NULL, 1, NULL + ) + `) + yield* db.run(` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, assistant_message_id, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, request_state, request_error_code, created_at, + prompt_epoch, prompt_window_id, effective_history_hash, request_input_hash, + final_request_hash, provider_state, adapter_prepared_at, dispatching_at, + terminal_at, owner_token + ) VALUES ( + 'receipt-upgrade-valid', 1, 'ses_receipt_upgrade', 'msg_receipt_upgrade', + 'msg_receipt_upgrade', 'provider', 'model', '[]', '[]', '[]', '[]', + 'dispatched', 'provider outcome unknown', 1, 0, 'window-upgrade', + 'history-upgrade', 'input-upgrade', 'final-upgrade', 'indeterminate_after_crash', + 1, 1, 2, 'owner-receipt-upgrade' + ) + `) + yield* db.run(` + INSERT INTO session_tool_argument_receipt ( + receipt_id, layer, ordinal, event_type, payload_keys, unavailable_reason, + created_at, validation_outcome + ) VALUES ( + 'receipt-upgrade-valid', 'raw_frame', 0, 'raw', '[]', + 'raw_receipt_gate_disabled', 1, 'schema_invalid' + ) + `) + yield* db.run(` + INSERT INTO session_activity_progress ( + activity_id, revision, assistant_message_id, provider_receipt_id, + state, created_at, settled_at + ) VALUES ( + 'activity-receipt-upgrade', 0, 'msg_receipt_upgrade', + 'receipt-upgrade-valid', 'progress', 1, 2 + ) + `) + yield* db.run(` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, provider_id, model_id, + registry_tool_ids, permission_filtered_tool_ids, final_offered_tool_ids, call_ids, + request_state, request_error_code, created_at, provider_state, terminal_at + ) VALUES ( + 'receipt-upgrade-orphan', 1, 'ses_missing', 'msg_missing', 'provider', 'model', + '[]', '[]', '[]', '[]', 'dispatched', 'historical orphan', 1, + 'indeterminate_after_crash', 1 + ) + `) + yield* db.run(` + INSERT INTO session_tool_argument_receipt ( + receipt_id, layer, ordinal, event_type, payload_keys, unavailable_reason, + created_at, validation_outcome + ) VALUES ( + 'receipt-upgrade-orphan', 'raw_frame', 0, 'raw', '[]', + 'raw_receipt_gate_disabled', 1, 'not_evaluated' + ) + `) + + const receiptColumns = yield* db.all<{ name: string; type: string; notnull: number; dflt_value: string | null }>(` + SELECT name, type, "notnull", dflt_value + FROM pragma_table_info('session_tool_request_receipt') ORDER BY cid + `) + yield* DatabaseMigration.applyOnly(db, [migrations[recoveryIndex]!]) + + expect( + yield* db.all(` + SELECT name, type, "notnull", dflt_value + FROM pragma_table_info('session_tool_request_receipt') ORDER BY cid + `), + ).toEqual(receiptColumns) + expect( + yield* db.get(` + SELECT receipt_id, session_id, provider_state, final_request_hash, owner_token + FROM session_tool_request_receipt WHERE receipt_id = 'receipt-upgrade-valid' + `), + ).toEqual({ + receipt_id: "receipt-upgrade-valid", + session_id: "ses_receipt_upgrade", + provider_state: "indeterminate_after_crash", + final_request_hash: "final-upgrade", + owner_token: "owner-receipt-upgrade", + }) + expect( + yield* db.get(` + SELECT receipt_id, validation_outcome + FROM session_tool_argument_receipt WHERE receipt_id = 'receipt-upgrade-valid' + `), + ).toEqual({ receipt_id: "receipt-upgrade-valid", validation_outcome: "schema_invalid" }) + expect( + yield* db.get(` + SELECT activity_id, provider_receipt_id, state + FROM session_activity_progress WHERE activity_id = 'activity-receipt-upgrade' + `), + ).toEqual({ + activity_id: "activity-receipt-upgrade", + provider_receipt_id: "receipt-upgrade-valid", + state: "progress", + }) + expect( + yield* db.get(`SELECT receipt_id FROM session_tool_request_receipt WHERE receipt_id = 'receipt-upgrade-orphan'`), + ).toBeUndefined() + expect( + yield* db.get(`SELECT receipt_id FROM session_tool_argument_receipt WHERE receipt_id = 'receipt-upgrade-orphan'`), + ).toBeUndefined() + expect( + yield* db.all(` + SELECT name FROM sqlite_master + WHERE type = 'index' AND name IN ( + 'session_tool_request_receipt_session_ordinal_idx', + 'session_tool_request_receipt_session_idx', + 'session_tool_request_receipt_msg_idx', + 'session_tool_request_receipt_prompt_window_idx', + 'session_tool_request_receipt_provider_state_idx' + ) ORDER BY name + `), + ).toEqual([ + { name: "session_tool_request_receipt_msg_idx" }, + { name: "session_tool_request_receipt_prompt_window_idx" }, + { name: "session_tool_request_receipt_provider_state_idx" }, + { name: "session_tool_request_receipt_session_idx" }, + { name: "session_tool_request_receipt_session_ordinal_idx" }, + ]) + expect( + yield* db.all(` + SELECT name FROM sqlite_master + WHERE type = 'trigger' AND name LIKE 'session_tool_request_receipt_%' + ORDER BY name + `), + ).toEqual([ + { name: "session_tool_request_receipt_binding_immutable" }, + { name: "session_tool_request_receipt_dispatch_guard" }, + { name: "session_tool_request_receipt_parent_cleanup" }, + { name: "session_tool_request_receipt_provider_transition" }, + { name: "session_tool_request_receipt_response_guard" }, + ]) + expect( + yield* db.all(` + SELECT "table", "from", "to", on_delete + FROM pragma_foreign_key_list('session_tool_request_receipt') + `), + ).toEqual([{ table: "session", from: "session_id", to: "id", on_delete: "CASCADE" }]) + expect( + Exit.isFailure( + yield* db + .run(` + UPDATE session_tool_request_receipt + SET provider_state = 'streaming' + WHERE receipt_id = 'receipt-upgrade-valid' + `) + .pipe(Effect.exit), + ), + ).toBe(true) + expect(yield* db.all("PRAGMA foreign_key_check")).toEqual([]) + }), + ) + }) + + it.live("deletes recovery and terminal activity authority with its parent Session", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + + yield* db.run(` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('project-recovery-delete', '/repo', '[]', 1, 1) + `) + yield* db.run(` + INSERT INTO session ( + id, project_id, slug, directory, title, version, mutation_epoch, time_created, time_updated + ) VALUES ( + 'ses_recovery_delete', 'project-recovery-delete', 'recovery-delete', '/repo', + 'Recovery delete', '1', 1, 1, 1 + ) + `) + yield* db.run(` + INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES + ('msg_recovery_delete_user', 'ses_recovery_delete', 1, 1, '{"role":"user"}'), + ('msg_recovery_delete_assistant', 'ses_recovery_delete', 2, 2, '{"role":"assistant"}'), + ('msg_recovery_delete_high_water', 'ses_recovery_delete', 3, 3, '{"role":"user"}') + `) + yield* db.run(` + INSERT INTO session_intent ( + intent_id, session_id, source, state, selected_variant, selected_payload_hash, + delivery, admitted_message_id, execution_mode, execution_state, mutation_epoch, + time_created, time_selected, time_admitted, time_updated + ) VALUES ( + 'intent-recovery-delete', 'ses_recovery_delete', 'composer', 'admitted', 'original', + 'payload-recovery-delete', 'turn', 'msg_recovery_delete_user', 'legacy', 'legacy', 0, + 1, 1, 1, 1 + ) + `) + yield* db.run(` + INSERT INTO session_activity_admission ( + admission_id, session_id, source_kind, legacy_intent_id, admitted_message_id, + delivery, payload_fingerprint_kind, payload_fingerprint, execution_mode, created_at + ) VALUES ( + 'admission-recovery-delete', 'ses_recovery_delete', 'legacy_intent', + 'intent-recovery-delete', 'msg_recovery_delete_user', 'turn', 'payload_hash', + 'payload-recovery-delete', 'legacy', 1 + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity ( + activity_id, session_id, ordinal, trigger_admission_id, owner_token, + state, terminal_reason, created_at, settled_at + ) VALUES ( + 'activity-recovery-delete', 'ses_recovery_delete', 0, 'admission-recovery-delete', + 'owner-recovery-delete', 'active', NULL, 1, NULL + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity_admission ( + activity_id, admission_id, ordinal, role, attached_at + ) VALUES ( + 'activity-recovery-delete', 'admission-recovery-delete', 0, 'trigger', 1 + ) + `) + yield* db.run(` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, assistant_message_id, + provider_attempt_id, provider_id, model_id, protocol, registry_tool_ids, + permission_filtered_tool_ids, final_offered_tool_ids, call_ids, prompt_epoch, + prompt_window_id, effective_history_hash, request_input_hash, final_request_hash, + provider_state, adapter_prepared_at, dispatching_at, terminal_at, owner_token, + request_state, request_error_code, created_at + ) VALUES ( + 'receipt-recovery-delete', 1, 'ses_recovery_delete', 'msg_recovery_delete_user', + 'msg_recovery_delete_assistant', NULL, 'provider', 'model', 'chat', '[]', '[]', + '[]', '[]', 0, 'window-recovery-source', 'history-recovery-source', + 'request-input-recovery-delete', 'request-final-recovery-delete', + 'indeterminate_after_crash', 1, 1, 2, 'owner-recovery-delete', 'dispatched', + 'provider_started_outcome_unknown_after_process_restart', 1 + ) + `) + yield* db.run(` + INSERT INTO session_activity_progress ( + activity_id, revision, assistant_message_id, provider_receipt_id, + input_membership_ordinal, state, created_at, settled_at + ) VALUES ( + 'activity-recovery-delete', 0, 'msg_recovery_delete_assistant', + 'receipt-recovery-delete', 0, 'recovery_required', 1, 2 + ) + `) + yield* db.run(` + UPDATE session_legacy_activity + SET state = 'recovery_required', terminal_reason = 'provider outcome unknown', settled_at = 2 + WHERE activity_id = 'activity-recovery-delete' + `) + yield* db.run(` + INSERT INTO session_legacy_activity_terminal ( + activity_id, session_id, mutation_epoch, state, reason_code, source, + operation_id, run_id, assistant_message_id, progress_revision, + membership_ordinal, owner_token, created_at + ) VALUES ( + 'activity-recovery-delete', 'ses_recovery_delete', 0, 'recovery_required', + 'provider outcome unknown', 'migration_backfill', 'terminal-recovery-delete', NULL, + 'msg_recovery_delete_assistant', 0, 0, 'owner-recovery-delete', 2 + ) + `) + yield* db.run(` + INSERT INTO session_legacy_activity_migration_receipt ( + receipt_id, batch_id, activity_id, classifier_version, before_state, after_state, + evidence_hash, terminal_operation_id, error_code, created_at + ) VALUES ( + 'migration-receipt-recovery-delete', 'migration-batch-recovery-delete', + 'activity-recovery-delete', 'legacy-terminal-state-v1', 'recovery_required', + 'recovery_required', 'evidence-recovery-delete', 'terminal-recovery-delete', NULL, 2 + ) + `) + yield* db.run(` + INSERT INTO session_prompt_epoch ( + session_id, epoch, state, source_end_message_id, reason, created_at, retired_at, + projection_version, canonicalization_version, base_message_count, + effective_history_hash, first_window_id, window_id, world_state_baseline_hash, + authority_state, recovery_reason + ) VALUES ( + 'ses_recovery_delete', 0, 'retired', 'msg_recovery_delete_assistant', 'bootstrap', + 1, 2, 1, 1, 2, 'history-recovery-source', 'window-recovery-source', + 'window-recovery-source', 'baseline-recovery-delete', 'recovery_required', + 'provider outcome unknown' + ) + `) + yield* db.run(` + INSERT INTO session_world_state_baseline ( + session_id, prompt_epoch, section_id, snapshot, fragment, fragment_hash, + provenance, created_at + ) VALUES ( + 'ses_recovery_delete', 0, 'world_state:env', + '{"kind":"env","version":1,"updatedAt":1,"value":"source"}', + '## Environment\nsource', 'fragment-recovery-source', 'legacy_migration', 1 + ) + `) + yield* db.run(` + INSERT INTO session_tool_request_resolution ( + resolution_id, receipt_id, session_id, legacy_activity_id, assistant_message_id, + source_prompt_epoch, source_window_id, source_effective_history_hash, + source_request_hash, source_mutation_epoch, expected_provider_state, decision, + actor_type, actor_id, reason, risk_acknowledged, safe_end_message_id, + safe_history_hash, safe_message_ids, ambiguity_message_id, physical_message_high_water, + successor_prompt_epoch, successor_window_id, successor_history_hash, + successor_mutation_epoch, created_at + ) VALUES ( + 'resolution-recovery-delete', 'receipt-recovery-delete', 'ses_recovery_delete', + 'activity-recovery-delete', 'msg_recovery_delete_assistant', 0, + 'window-recovery-source', 'history-recovery-source', 'request-final-recovery-delete', + 0, 'indeterminate_after_crash', 'abandoned', 'user', 'test-user', + 'abandon unknown provider result', 0, 'msg_recovery_delete_user', + 'history-recovery-successor', '["msg_recovery_delete_user"]', + 'msg_recovery_delete_assistant', 'msg_recovery_delete_high_water', 1, + 'window-recovery-successor', 'history-recovery-successor', 1, 3 + ) + `) + yield* db.run(` + INSERT INTO session_prompt_epoch ( + session_id, epoch, state, source_end_message_id, checkpoint_hash, reason, created_at, + projection_version, canonicalization_version, base_message_count, + effective_history_hash, first_window_id, previous_window_id, window_id, + world_state_baseline_hash, authority_state, recovery_resolution_id + ) VALUES ( + 'ses_recovery_delete', 1, 'active', 'msg_recovery_delete_high_water', + 'history-recovery-successor', 'recovery', 3, 1, 1, 1, + 'history-recovery-successor', 'window-recovery-source', 'window-recovery-source', + 'window-recovery-successor', 'baseline-recovery-delete', 'ready', + 'resolution-recovery-delete' + ) + `) + yield* db.run(` + INSERT INTO session_prompt_epoch_message ( + session_id, prompt_epoch, ordinal, message_id + ) VALUES ('ses_recovery_delete', 1, 0, 'msg_recovery_delete_user') + `) + yield* db.run(` + INSERT INTO session_world_state_baseline ( + session_id, prompt_epoch, section_id, snapshot, fragment, fragment_hash, + provenance, created_at + ) VALUES ( + 'ses_recovery_delete', 1, 'world_state:env', + '{"kind":"env","version":1,"updatedAt":1,"value":"source"}', + '## Environment\nsource', 'fragment-recovery-source', 'recovery_copied', 3 + ) + `) + yield* db.run(` + INSERT INTO session_prompt_epoch_recovery ( + session_id, prompt_epoch, resolution_id, source_prompt_epoch, + source_mutation_epoch, successor_mutation_epoch, ambiguity_message_id, + physical_message_high_water, created_at + ) VALUES ( + 'ses_recovery_delete', 1, 'resolution-recovery-delete', 0, 0, 1, + 'msg_recovery_delete_assistant', 'msg_recovery_delete_high_water', 3 + ) + `) + yield* db.run(` + INSERT INTO session_tool_request_resolution_command ( + command_id, request_hash, session_id, receipt_id, result_resolution_id, created_at + ) VALUES ( + 'command-recovery-delete', 'command-hash-recovery-delete', 'ses_recovery_delete', + 'receipt-recovery-delete', 'resolution-recovery-delete', 3 + ) + `) + yield* db.run(` + INSERT INTO session_history_state (session_id, state, reason, time_created, time_updated) + VALUES ('ses_recovery_delete', 'ready', NULL, 1, 3) + `) + + expect(yield* authorityRows).toEqual([ + { table_name: "session", row_count: 1 }, + { table_name: "session_intent", row_count: 1 }, + { table_name: "session_activity_admission", row_count: 1 }, + { table_name: "session_legacy_activity", row_count: 1 }, + { table_name: "session_legacy_activity_admission", row_count: 1 }, + { table_name: "session_activity_progress", row_count: 1 }, + { table_name: "session_tool_request_receipt", row_count: 1 }, + { table_name: "session_legacy_activity_terminal", row_count: 1 }, + { table_name: "session_legacy_activity_migration_receipt", row_count: 1 }, + { table_name: "session_prompt_epoch", row_count: 2 }, + { table_name: "session_prompt_epoch_message", row_count: 1 }, + { table_name: "session_world_state_baseline", row_count: 2 }, + { table_name: "session_tool_request_resolution_command", row_count: 1 }, + { table_name: "session_tool_request_resolution", row_count: 1 }, + { table_name: "session_prompt_epoch_recovery", row_count: 1 }, + { table_name: "session_history_state", row_count: 1 }, + ]) + + yield* db.run("DELETE FROM session WHERE id = 'ses_recovery_delete'") + + expect((yield* authorityRows).every((row) => row.row_count === 0)).toBe(true) + expect(yield* db.all("PRAGMA foreign_key_check")).toEqual([]) + }), + ) +}) diff --git a/packages/core/test/session-projector.test.ts b/packages/core/test/session-projector.test.ts index 651f1aad8..14adf2b2a 100644 --- a/packages/core/test/session-projector.test.ts +++ b/packages/core/test/session-projector.test.ts @@ -45,6 +45,33 @@ const assistantRow = ( } describe("SessionProjector", () => { + it.effect("decodes legacy diff manifest descriptors without statisticsExact", () => + Effect.sync(() => { + const decoded = Schema.decodeUnknownSync(SessionV1.User)({ + id: SessionV1.MessageID.make("msg_legacy_diff_manifest"), + sessionID, + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: ProviderV2.ID.make("provider"), modelID: ModelV2.ID.make("model") }, + summary: { + diffs: [], + diffManifest: { + completeness: "complete", + truncationReasons: [], + manifestHash: "sha256:legacy", + totalFiles: 0, + totalFilesExact: true, + includedFiles: 0, + truncatedFiles: 0, + }, + }, + }) + + expect(decoded.summary?.diffManifest?.statisticsExact).toBeUndefined() + }), + ) + it.effect("never persists computed V1 activity progress from a full message update", () => Effect.gen(function* () { const { db } = yield* Database.Service diff --git a/packages/deepagent-code/src/control-plane/workspace.ts b/packages/deepagent-code/src/control-plane/workspace.ts index e9e0cc241..1f1da2038 100644 --- a/packages/deepagent-code/src/control-plane/workspace.ts +++ b/packages/deepagent-code/src/control-plane/workspace.ts @@ -1,10 +1,13 @@ -import { Context, Effect, FiberMap, Iterable, Layer, Schema, Stream } from "effect" +import { Context, Effect, FiberMap, Layer, Schema, Stream } from "effect" import { serviceUse } from "@deepagent-code/core/effect/service-use" import { FetchHttpClient, HttpBody, HttpClient, HttpClientError, HttpClientRequest } from "effect/unstable/http" import { Database } from "@deepagent-code/core/database/database" import { asc } from "drizzle-orm" +import { and } from "drizzle-orm" import { eq } from "drizzle-orm" +import { gt } from "drizzle-orm" import { inArray } from "drizzle-orm" +import { sql } from "drizzle-orm" import { Project } from "@/project/project" import { GlobalBus } from "@/bus/global" import { Auth } from "@/auth" @@ -32,6 +35,8 @@ import { Vcs } from "@/project/vcs" import { InstanceStore } from "@/project/instance-store" import { InstanceBootstrap } from "@/project/bootstrap" import { WorkspaceAdapterRuntime } from "./workspace-adapter-runtime" +import { SessionRecoveryTransferGuard } from "@/session/recovery-transfer-guard" +import { encodeReplayRequestPrefix, SyncReplayLimits } from "@/sync/replay-protocol" export const Info = Schema.Struct({ ...WorkspaceInfoSchema.fields, @@ -125,6 +130,25 @@ export class SessionWarpHttpError extends Schema.TaggedErrorClass()( + "WorkspaceSessionWarpRecoveryProjectionError", + { + message: Schema.String, + sessionID: SessionID, + recoveryAuthorityID: Schema.String, + }, +) {} + +export class SessionWarpHistoryLimitError extends Schema.TaggedErrorClass()( + "WorkspaceSessionWarpHistoryLimitError", + { + message: Schema.String, + sessionID: SessionID, + eventID: EventV2.ID, + bytes: Schema.Number, + }, +) {} + export class SyncTimeoutError extends Schema.TaggedErrorClass()("WorkspaceSyncTimeoutError", { message: Schema.String, state: Schema.Record(Schema.String, Schema.Number), @@ -140,6 +164,9 @@ type SessionWarpError = | WorkspaceNotFoundError | SessionEventsNotFoundError | SessionWarpHttpError + | SessionWarpRecoveryProjectionError + | SessionWarpHistoryLimitError + | Vcs.RawDiffError | Vcs.PatchApplyError | HttpClientError.HttpClientError type WaitForSyncError = SyncTimeoutError | SyncAbortedError @@ -326,6 +353,89 @@ export const layer = Layer.effect( ) }) + const loadRawPatch = Effect.fn("Workspace.loadRawPatch")(function* (workspaceID: WorkspaceV2.ID) { + const workspace = yield* get(workspaceID) + if (!workspace) + return yield* new Vcs.RawDiffError({ + message: `Raw patch source workspace not found: ${workspaceID}`, + reason: "remote-status", + }) + + const target = yield* WorkspaceAdapterRuntime.target(workspace) + if (target.type === "local") { + const store = yield* InstanceStore.Service + return yield* store.provide({ directory: target.directory }, vcs.diffRaw()) + } + + const response = yield* http + .execute( + HttpClientRequest.get(route(target.url, "/vcs/diff/raw"), { + headers: new Headers(target.headers), + }), + ) + .pipe( + Effect.mapError( + () => + new Vcs.RawDiffError({ + message: `Raw patch source request failed: ${workspaceID}`, + reason: "remote-status", + }), + ), + ) + if (response.status < 200 || response.status >= 300) + return yield* new Vcs.RawDiffError({ + message: `Raw patch source returned HTTP ${response.status}`, + reason: "remote-status", + actual: response.status, + }) + + const contentLength = Number(response.headers["content-length"]) + if (Number.isFinite(contentLength) && contentLength > Vcs.RawDiffLimits.patchBytes) + return yield* new Vcs.RawDiffError({ + message: "Remote raw patch exceeded the declared response budget", + reason: "remote-output", + limit: Vcs.RawDiffLimits.patchBytes, + actual: contentLength, + }) + + const result = yield* Stream.runFoldEffect( + response.stream, + () => ({ chunks: [] as Uint8Array[], bytes: 0 }), + (acc, chunk) => { + const bytes = acc.bytes + chunk.byteLength + if (bytes <= Vcs.RawDiffLimits.patchBytes) { + acc.chunks.push(chunk) + return Effect.succeed({ chunks: acc.chunks, bytes }) + } + return Effect.fail( + new Vcs.RawDiffError({ + message: "Remote raw patch exceeded the streamed response budget", + reason: "remote-output", + limit: Vcs.RawDiffLimits.patchBytes, + actual: bytes, + }), + ) + }, + ).pipe( + Effect.mapError((error) => + error instanceof Vcs.RawDiffError + ? error + : new Vcs.RawDiffError({ + message: "Remote raw patch response failed while streaming", + reason: "remote-output", + }), + ), + ) + return yield* Effect.try({ + try: () => new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(result.chunks, result.bytes)), + catch: () => + new Vcs.RawDiffError({ + message: "Remote raw patch response is not valid UTF-8", + reason: "remote-decode", + }), + }) + }) + const syncHistory = Effect.fn("Workspace.syncHistory")(function* ( space: Info, url: URL | string, @@ -354,46 +464,53 @@ export const layer = Layer.effect( known: Object.keys(state).length, }) - const response = yield* http.execute( - HttpClientRequest.post(route(url, "/sync/history"), { - headers: new Headers(headers), - body: HttpBody.jsonUnsafe(state), - }), - ) + let total = 0 + while (true) { + const response = yield* http.execute( + HttpClientRequest.post(route(url, "/sync/history"), { + headers: new Headers(headers), + body: HttpBody.jsonUnsafe(state), + }), + ) - if (response.status < 200 || response.status >= 300) { - const body = yield* response.text - return yield* new SyncHttpError({ - message: `Workspace history HTTP failure: ${response.status} ${body}`, - status: response.status, - body, - }) - } + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text + return yield* new SyncHttpError({ + message: `Workspace history HTTP failure: ${response.status} ${body}`, + status: response.status, + body, + }) + } - const history = (yield* response.json) as HistoryEvent[] + const history = (yield* response.json) as HistoryEvent[] + if (history.length === 0) break + yield* Effect.forEach( + history, + (event) => + events + .replay( + { + id: EventV2.ID.make(event.id), + aggregateID: event.aggregate_id, + seq: event.seq, + type: event.type, + data: event.data, + }, + { publish: true, ownerID: space.id }, + ) + .pipe( + Effect.provideService(WorkspaceRef, space.id), + Effect.tap(() => Effect.sync(() => (state[event.aggregate_id] = event.seq))), + ), + { discard: true }, + ) + total += history.length + } log.info("workspace history synced", { workspaceID: space.id, - events: history.length, + events: total, }) - - yield* Effect.forEach( - history, - (event) => - events - .replay( - { - id: EventV2.ID.make(event.id), - aggregateID: event.aggregate_id, - seq: event.seq, - type: event.type, - data: event.data, - }, - { publish: true, ownerID: space.id }, - ) - .pipe(Effect.provideService(WorkspaceRef, space.id)), - { discard: true }, - ) }) const syncWorkspaceLoop = Effect.fn("Workspace.syncWorkspaceLoop")(function* (space: Info) { @@ -609,6 +726,43 @@ export const layer = Layer.effect( .get() .pipe(Effect.orDie) + const recoveryAuthorityID = yield* SessionRecoveryTransferGuard.authorityID(db, input.sessionID) + if ( + recoveryAuthorityID && + (input.workspaceID ?? undefined) !== (current?.workspaceID ?? undefined) + ) + return yield* new SessionWarpRecoveryProjectionError({ + message: + "Provider recovery projection cannot move between workspaces until canonical recovery snapshot import is available", + sessionID: input.sessionID, + recoveryAuthorityID, + }) + + const sourcePatch = + input.copyChanges && current?.workspaceID + ? yield* loadRawPatch(current.workspaceID).pipe( + Effect.provide(InstanceStore.defaultLayer.pipe(Layer.provide(InstanceBootstrap.defaultLayer))), + ) + : "" + + if (sourcePatch) { + const applied = yield* runInWorkspace({ + workspaceID: input.workspaceID ?? undefined, + local: () => vcs.apply({ patch: sourcePatch }), + remote: ({ target }) => + HttpClientRequest.post(route(target.url, "/vcs/apply"), { + headers: new Headers(target.headers), + body: HttpBody.jsonUnsafe({ patch: sourcePatch }), + }), + fallback: { applied: false }, + }).pipe(Effect.provide(InstanceStore.defaultLayer.pipe(Layer.provide(InstanceBootstrap.defaultLayer)))) + if (!applied.applied) + return yield* new Vcs.PatchApplyError({ + message: "Patch can't be applied because the target workspace was unavailable", + reason: "not-clean", + }) + } + if (current?.workspaceID) { const previous = yield* get(current.workspaceID) if (previous) { @@ -636,36 +790,6 @@ export const layer = Layer.effect( } } - const sourcePatch = - input.copyChanges && current?.workspaceID - ? yield* runInWorkspace({ - workspaceID: current?.workspaceID ?? undefined, - local: () => vcs.diffRaw(), - remote: ({ target }) => - HttpClientRequest.get(route(target.url, "/vcs/diff/raw"), { - headers: new Headers(target.headers), - }), - fallback: "", - response: "text", - }).pipe(Effect.provide(InstanceStore.defaultLayer.pipe(Layer.provide(InstanceBootstrap.defaultLayer)))) - : "" - - if (sourcePatch) { - // Attempt to apply the file changes to the new workspace. - // We intentionally do first so if it fails we don't warp - // the session. - yield* runInWorkspace({ - workspaceID: input.workspaceID ?? undefined, - local: () => vcs.apply({ patch: sourcePatch }), - remote: ({ target }) => - HttpClientRequest.post(route(target.url, "/vcs/apply"), { - headers: new Headers(target.headers), - body: HttpBody.jsonUnsafe({ patch: sourcePatch }), - }), - fallback: { applied: false }, - }).pipe(Effect.provide(InstanceStore.defaultLayer.pipe(Layer.provide(InstanceBootstrap.defaultLayer)))) - } - if (input.workspaceID === null) { yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: undefined }) @@ -698,82 +822,101 @@ export const layer = Layer.effect( return } - const rows = yield* db - .select({ - id: EventTable.id, - aggregateID: EventTable.aggregate_id, - seq: EventTable.seq, - type: EventTable.type, - data: EventTable.data, - }) - .from(EventTable) - .where(eq(EventTable.aggregate_id, input.sessionID)) - .orderBy(asc(EventTable.seq)) - .all() - .pipe(Effect.orDie) - if (rows.length === 0) + const pageEvents = Math.min(10, SyncReplayLimits.events) + const pageBytes = SyncReplayLimits.eventDataBytes + let cursor = -1 + let total = 0 + while (true) { + const candidates = yield* db + .select({ + id: EventTable.id, + seq: EventTable.seq, + bytes: sql`length(CAST(${EventTable.data} AS BLOB))`, + }) + .from(EventTable) + .where(and(eq(EventTable.aggregate_id, input.sessionID), gt(EventTable.seq, cursor))) + .orderBy(asc(EventTable.seq)) + .limit(pageEvents) + .all() + .pipe(Effect.orDie) + if (candidates.length === 0) break + const page = candidates.reduce<{ + ids: EventV2.ID[] + bytes: number + closed: boolean + oversized?: (typeof candidates)[number] + }>( + (result, row) => { + if (result.closed) return result + if (row.bytes > pageBytes) + return result.ids.length === 0 + ? { ...result, closed: true, oversized: row } + : { ...result, closed: true } + if (result.bytes > 0 && result.bytes + row.bytes > pageBytes) return { ...result, closed: true } + return { ids: [...result.ids, row.id], bytes: result.bytes + row.bytes, closed: false } + }, + { ids: [], bytes: 0, closed: false }, + ) + if (page.oversized) + return yield* new SessionWarpHistoryLimitError({ + message: `Legacy sync event ${page.oversized.id} exceeds the bounded warp page and requires artifact migration`, + sessionID: input.sessionID, + eventID: page.oversized.id, + bytes: page.oversized.bytes, + }) + if (page.ids.length === 0) break + const events = yield* db + .select({ + id: EventTable.id, + aggregateID: EventTable.aggregate_id, + seq: EventTable.seq, + type: EventTable.type, + data: EventTable.data, + }) + .from(EventTable) + .where(inArray(EventTable.id, page.ids)) + .orderBy(asc(EventTable.seq)) + .all() + .pipe(Effect.orDie) + const encoded = encodeReplayRequestPrefix(space.directory ?? "", events) + if (encoded.events.length === 0) + return yield* new SessionWarpHistoryLimitError({ + message: `Sync event ${events[0]!.id} cannot fit the bounded replay request`, + sessionID: input.sessionID, + eventID: events[0]!.id, + bytes: Buffer.byteLength( + JSON.stringify({ + directory: space.directory ?? "", + events: [events[0]], + }), + ), + }) + const response = yield* http.execute( + HttpClientRequest.post(route(target.url, "/sync/replay"), { + headers: new Headers(target.headers), + body: HttpBody.text(encoded.json, "application/json"), + }), + ) + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text + return yield* new SessionWarpHttpError({ + message: `Failed to warp session ${input.sessionID} into workspace ${workspaceID}: HTTP ${response.status} ${body}`, + workspaceID, + sessionID: input.sessionID, + status: response.status, + body, + }) + } + cursor = encoded.events.at(-1)!.seq + total += encoded.events.length + if (encoded.complete && page.ids.length === candidates.length && candidates.length < pageEvents) break + } + if (total === 0) return yield* new SessionEventsNotFoundError({ message: `No events found for session: ${input.sessionID}`, sessionID: input.sessionID, }) - const batches = Iterable.chunksOf(rows, 10) - const total = Iterable.size(batches) - - log.info("session warp prepared", { - workspaceID: input.workspaceID, - sessionID: input.sessionID, - target: String(route(target.url, "/sync/replay")), - events: rows.length, - batches: total, - first: rows[0]?.seq, - last: rows.at(-1)?.seq, - }) - - yield* Effect.forEach( - batches, - (events, i) => - Effect.gen(function* () { - const response = yield* http.execute( - HttpClientRequest.post(route(target.url, "/sync/replay"), { - headers: new Headers(target.headers), - body: HttpBody.jsonUnsafe({ - directory: space.directory ?? "", - events, - }), - }), - ) - - if (response.status < 200 || response.status >= 300) { - const body = yield* response.text - log.error("session warp batch failed", { - workspaceID: input.workspaceID, - sessionID: input.sessionID, - step: i + 1, - total, - status: response.status, - body, - }) - return yield* new SessionWarpHttpError({ - message: `Failed to warp session ${input.sessionID} into workspace ${workspaceID}: HTTP ${response.status} ${body}`, - workspaceID, - sessionID: input.sessionID, - status: response.status, - body, - }) - } - - log.info("session warp batch posted", { - workspaceID: input.workspaceID, - sessionID: input.sessionID, - step: i + 1, - total, - status: response.status, - }) - }), - { discard: true }, - ) - const response = yield* http.execute( HttpClientRequest.post(route(target.url, "/sync/steal"), { headers: new Headers(target.headers), @@ -802,7 +945,7 @@ export const layer = Layer.effect( log.info("session warp complete", { workspaceID: input.workspaceID, sessionID: input.sessionID, - batches: total, + events: total, }) }).pipe( Effect.tapError((err) => diff --git a/packages/deepagent-code/src/git/index.ts b/packages/deepagent-code/src/git/index.ts index e2af539e7..783f51a85 100644 --- a/packages/deepagent-code/src/git/index.ts +++ b/packages/deepagent-code/src/git/index.ts @@ -1,5 +1,5 @@ import { AppProcess } from "@deepagent-code/core/process" -import { Effect, Layer, Context, Stream } from "effect" +import { Duration, Effect, Layer, Context, Stream } from "effect" import { ChildProcess } from "effect/unstable/process" const cfg = [ @@ -123,8 +123,10 @@ export interface Result { export interface Options { readonly cwd: string readonly env?: Record + readonly maxErrorBytes?: number readonly maxOutputBytes?: number readonly stdin?: ChildProcess.CommandInput + readonly timeout?: Duration.Input } export interface Interface { @@ -192,7 +194,11 @@ export const layer = Layer.effect( stdout: "pipe", stderr: "pipe", }), - { maxOutputBytes: opts.maxOutputBytes }, + { + maxErrorBytes: opts.maxErrorBytes, + maxOutputBytes: opts.maxOutputBytes, + timeout: opts.timeout, + }, ) return { exitCode: result.exitCode, diff --git a/packages/deepagent-code/src/project/vcs.ts b/packages/deepagent-code/src/project/vcs.ts index 82b882fcf..fa6174db1 100644 --- a/packages/deepagent-code/src/project/vcs.ts +++ b/packages/deepagent-code/src/project/vcs.ts @@ -1,4 +1,4 @@ -import { Effect, Layer, Context, Schema, Stream, Scope } from "effect" +import { Duration, Effect, Layer, Context, Schema, Stream, Scope } from "effect" import { formatPatch, structuredPatch } from "diff" import { InstanceState } from "@/effect/instance-state" import { Watcher } from "@deepagent-code/core/filesystem/watcher" @@ -11,6 +11,12 @@ const log = Log.create({ service: "vcs" }) const PATCH_CONTEXT_LINES = 2_147_483_647 const MAX_PATCH_BYTES = 10_000_000 const MAX_TOTAL_PATCH_BYTES = 10_000_000 +export const RawDiffLimits = { + candidateFiles: 1_000, + discoveryBytes: 4 * 1024 * 1024, + wallTimeMs: 15_000, + patchBytes: 10 * 1024 * 1024, +} as const type DiffOptions = { readonly context?: number } @@ -288,13 +294,34 @@ export class PatchApplyError extends Schema.TaggedErrorClass()( reason: Schema.Literals(["non-git", "not-clean"]), }) {} +export class RawDiffError extends Schema.TaggedErrorClass()("VcsRawDiffError", { + message: Schema.String, + reason: Schema.Literals([ + "candidate-files", + "status-output", + "status-failed", + "tracked-output", + "tracked-failed", + "untracked-output", + "untracked-failed", + "total-output", + "time-limit", + "remote-status", + "remote-output", + "remote-decode", + ]), + limit: Schema.optional(Schema.Number), + actual: Schema.optional(Schema.Number), + file: Schema.optional(Schema.String), +}) {} + export interface Interface { readonly init: () => Effect.Effect readonly branch: () => Effect.Effect readonly defaultBranch: () => Effect.Effect readonly status: () => Effect.Effect readonly diff: (mode: Mode, options?: DiffOptions) => Effect.Effect - readonly diffRaw: () => Effect.Effect + readonly diffRaw: () => Effect.Effect readonly apply: (input: ApplyInput) => Effect.Effect } @@ -399,15 +426,149 @@ export const layer: Layer.Layer item.code === "??"), - (item) => git.patchUntracked(ctx.directory, item.file).pipe(Effect.map((patch) => patch.text)), + const deadline = Date.now() + RawDiffLimits.wallTimeMs + const deadlineRemaining = () => Duration.millis(Math.max(1, deadline - Date.now())) + const [hasHead, status] = yield* Effect.all( + [ + git + .run(["rev-parse", "--verify", "HEAD"], { + cwd: ctx.directory, + maxErrorBytes: RawDiffLimits.discoveryBytes, + maxOutputBytes: RawDiffLimits.discoveryBytes, + timeout: deadlineRemaining(), + }) + .pipe(Effect.map((result) => result.exitCode === 0)), + git.run(["status", "--porcelain=v1", "--untracked-files=all", "--no-renames", "-z", "--", "."], { + cwd: ctx.directory, + maxErrorBytes: RawDiffLimits.discoveryBytes, + maxOutputBytes: RawDiffLimits.discoveryBytes, + timeout: deadlineRemaining(), + }), + ], + { concurrency: 2 }, ) - return [tracked, ...untracked].filter(Boolean).join("\n") + if (Date.now() >= deadline) + return yield* new RawDiffError({ message: "Raw diff discovery exceeded its deadline", reason: "time-limit" }) + if (status.truncated) + return yield* new RawDiffError({ + message: "Raw diff status discovery exceeded its output budget", + reason: "status-output", + limit: RawDiffLimits.discoveryBytes, + }) + if (status.exitCode !== 0) + return yield* new RawDiffError({ + message: "Raw diff status discovery failed", + reason: "status-failed", + }) + + const candidates = status + .text() + .split("\0") + .filter(Boolean) + .flatMap((item) => { + const file = item.slice(3) + if (!file) return [] + return [{ code: item.slice(0, 2), file }] + }) + if (candidates.length > RawDiffLimits.candidateFiles) + return yield* new RawDiffError({ + message: "Raw diff candidate file count exceeded its limit", + reason: "candidate-files", + limit: RawDiffLimits.candidateFiles, + actual: candidates.length, + }) + + const patches: string[] = [] + let bytes = 0 + if (hasHead) { + const tracked = yield* git.run( + ["diff", "--patch", "--no-ext-diff", "--no-renames", "--unified=3", "HEAD", "--", "."], + { + cwd: ctx.directory, + maxErrorBytes: RawDiffLimits.discoveryBytes, + maxOutputBytes: RawDiffLimits.patchBytes + 1, + timeout: deadlineRemaining(), + }, + ) + if (Date.now() >= deadline) + return yield* new RawDiffError({ message: "Tracked raw diff exceeded its deadline", reason: "time-limit" }) + if (tracked.truncated || tracked.stdout.byteLength > RawDiffLimits.patchBytes) + return yield* new RawDiffError({ + message: "Tracked raw patch exceeded the total output budget", + reason: "tracked-output", + limit: RawDiffLimits.patchBytes, + }) + if (tracked.exitCode !== 0) + return yield* new RawDiffError({ + message: "Tracked raw patch generation failed", + reason: "tracked-failed", + }) + if (tracked.stdout.byteLength > 0) { + patches.push(tracked.text()) + bytes = tracked.stdout.byteLength + } + } + + for (const item of candidates.filter((item) => item.code === "??" || (!hasHead && item.code[1] !== "D"))) { + if (Date.now() >= deadline) + return yield* new RawDiffError({ message: "Raw diff exceeded its deadline", reason: "time-limit" }) + const separator = patches.length > 0 ? 1 : 0 + const patchRemaining = RawDiffLimits.patchBytes - bytes - separator + if (patchRemaining <= 0) + return yield* new RawDiffError({ + message: "Raw patch exceeded the total output budget before all untracked files were encoded", + reason: "total-output", + limit: RawDiffLimits.patchBytes, + actual: bytes + separator, + file: item.file, + }) + const patch = yield* git.run( + [ + "diff", + "--no-index", + "--patch", + "--no-ext-diff", + "--no-renames", + "--unified=3", + "--", + "/dev/null", + item.file, + ], + { + cwd: ctx.directory, + maxErrorBytes: RawDiffLimits.discoveryBytes, + maxOutputBytes: patchRemaining + 1, + timeout: deadlineRemaining(), + }, + ) + if (Date.now() >= deadline) + return yield* new RawDiffError({ message: "Untracked raw diff exceeded its deadline", reason: "time-limit" }) + if (patch.truncated || patch.stdout.byteLength > patchRemaining) + return yield* new RawDiffError({ + message: `Untracked raw patch exceeded the remaining output budget: ${item.file}`, + reason: "untracked-output", + limit: patchRemaining, + file: item.file, + }) + if (patch.exitCode !== 0 && patch.exitCode !== 1) + return yield* new RawDiffError({ + message: `Untracked raw patch generation failed: ${item.file}`, + reason: "untracked-failed", + file: item.file, + }) + if (patch.stdout.byteLength === 0) continue + patches.push(patch.text()) + bytes += separator + patch.stdout.byteLength + } + const result = patches.join("\n") + const actual = Buffer.byteLength(result) + if (actual <= RawDiffLimits.patchBytes) return result + return yield* new RawDiffError({ + message: "Raw patch exceeded the total UTF-8 output budget", + reason: "total-output", + limit: RawDiffLimits.patchBytes, + actual, + }) }), apply: Effect.fn("Vcs.apply")(function* (input: ApplyInput) { const ctx = yield* InstanceState.context diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/instance.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/instance.ts index 23d674986..067c12d5b 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/instance.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/instance.ts @@ -72,6 +72,20 @@ export class ApiVcsApplyError extends Schema.ErrorClass("VcsAp { httpApiStatus: 400 }, ) {} +export class ApiVcsRawDiffError extends Schema.ErrorClass("VcsRawDiffError")( + { + name: Schema.Literal("VcsRawDiffError"), + data: Schema.Struct({ + message: Schema.String, + reason: Vcs.RawDiffError.fields.reason, + limit: Vcs.RawDiffError.fields.limit, + actual: Vcs.RawDiffError.fields.actual, + file: Vcs.RawDiffError.fields.file, + }), + }, + { httpApiStatus: 503 }, +) {} + export const InstancePaths = { dispose: "/instance/dispose", path: "/path", @@ -149,6 +163,7 @@ export const InstanceApi = HttpApi.make("instance") Schema.String.pipe(HttpApiSchema.asText({ contentType: "text/x-diff; charset=utf-8" })), "Raw VCS diff", ), + error: ApiVcsRawDiffError, }).annotateMerge( OpenApi.annotations({ identifier: "vcs.diff.raw", diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts index 72fc3a4b6..387aa60ee 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts @@ -25,6 +25,7 @@ import { ConflictError, InvalidRequestError, PermissionNotFoundError, + ServiceUnavailableError, SessionBusyError, } from "../errors" import { described } from "./metadata" @@ -34,6 +35,7 @@ import { ModelV2 } from "@deepagent-code/core/model" import { GraphKind } from "@deepagent-code/core/context-federation/contract" import { GraphQueryStatus } from "@deepagent-code/core/context-federation/federation" import { Sensitivity } from "@deepagent-code/core/context-federation/authorization" +import { SessionLegacyProviderResolution } from "@/session/legacy-provider-resolution" const root = "/session" export const ListQuery = Schema.Struct({ @@ -51,7 +53,13 @@ export const DiffQuery = Schema.Struct({ }) export const MessagesQuery = Schema.Struct({ ...WorkspaceRoutingQueryFields, - limit: Schema.optional(Schema.NumberFromString.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0))), + limit: Schema.optional( + Schema.NumberFromString.check( + Schema.isInt(), + Schema.isGreaterThanOrEqualTo(1), + Schema.isLessThanOrEqualTo(MessageV2.ClientMessageLimits.page), + ), + ), before: Schema.optional(Schema.String), }) export const StatusMap = Schema.Record(Schema.String, SessionStatus.Info) @@ -273,6 +281,9 @@ export const ContextAttemptResolvePayload = Schema.Struct({ reason: Schema.String, riskAcknowledged: Schema.optional(Schema.Boolean), }) +export const ProviderResolutionPayload = Schema.Struct( + Struct.omit(SessionLegacyProviderResolution.ResolveInput.fields, ["sessionID"]), +) export const SessionPaths = { list: root, @@ -307,6 +318,7 @@ export const SessionPaths = { updatePart: `${root}/:sessionID/message/:messageID/part/:partID`, contextDiagnostics: `${root}/:sessionID/context`, contextAttemptResolve: `${root}/:sessionID/context/attempt/:attemptID/resolve`, + providerResolution: `${root}/:sessionID/provider-resolution`, } as const export const SessionApi = HttpApi.make("session") @@ -630,7 +642,7 @@ export const SessionApi = HttpApi.make("session") query: WorkspaceRoutingQuery, payload: RevertPayload, success: described(Session.Info, "Updated session"), - error: [HttpApiError.BadRequest, ApiNotFoundError, SessionBusyError], + error: [HttpApiError.BadRequest, ApiNotFoundError, SessionBusyError, ServiceUnavailableError], }).annotateMerge( OpenApi.annotations({ identifier: "session.revert", @@ -726,6 +738,34 @@ export const SessionApi = HttpApi.make("session") description: "Apply an audited abandon, verified-settle, or risk-acknowledged replay decision.", }), ), + HttpApiEndpoint.get("providerResolutionList", SessionPaths.providerResolution, { + params: { sessionID: SessionID }, + query: WorkspaceRoutingQuery, + success: described( + Schema.Array(SessionLegacyProviderResolution.Descriptor), + "Pending provider recovery decisions", + ), + error: [HttpApiError.BadRequest, ApiNotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "session.providerResolutionList", + summary: "List provider recovery decisions", + description: "List unresolved legacy provider outcomes that require explicit recovery.", + }), + ), + HttpApiEndpoint.post("providerResolutionResolve", SessionPaths.providerResolution, { + params: { sessionID: SessionID }, + query: WorkspaceRoutingQuery, + payload: ProviderResolutionPayload, + success: described(SessionLegacyProviderResolution.Resolution, "Resolved provider outcome"), + error: [HttpApiError.BadRequest, ApiNotFoundError, ConflictError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "session.providerResolutionResolve", + summary: "Resolve a provider outcome", + description: "Append an audited abandoned resolution and activate a safe successor history epoch.", + }), + ), ) .annotateMerge( OpenApi.annotations({ diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts index 68e046108..50a00a734 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts @@ -7,18 +7,24 @@ import { Authorization } from "../middleware/authorization" import { InstanceContextMiddleware } from "../middleware/instance-context" import { WorkspaceRoutingMiddleware, WorkspaceRoutingQuery } from "../middleware/workspace-routing" import { described } from "./metadata" +import { ConflictError, ServiceUnavailableError } from "../errors" +import { SyncReplayLimits } from "@/sync/replay-protocol" const root = "/sync" +export { SyncReplayLimits } from "@/sync/replay-protocol" export const ReplayEvent = Schema.Struct({ - id: EventV2.ID, - aggregateID: Schema.String, + id: EventV2.ID.check(Schema.isMaxLength(SyncReplayLimits.eventIDCharacters)), + aggregateID: Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(SyncReplayLimits.aggregateIDCharacters), + ), seq: NonNegativeInt, - type: Schema.String, + type: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(SyncReplayLimits.typeCharacters)), data: Schema.Record(Schema.String, Schema.Unknown), }) export const ReplayPayload = Schema.Struct({ - directory: Schema.String, - events: Schema.NonEmptyArray(ReplayEvent), + directory: Schema.String.check(Schema.isMaxLength(SyncReplayLimits.directoryCharacters)), + events: Schema.NonEmptyArray(ReplayEvent).check(Schema.isMaxLength(SyncReplayLimits.events)), }) export const ReplayResponse = Schema.Struct({ sessionID: Schema.String, @@ -72,7 +78,7 @@ export const SyncApi = HttpApi.make("sync") query: WorkspaceRoutingQuery, payload: SessionPayload, success: described(SessionPayload, "Session stolen into workspace"), - error: HttpApiError.BadRequest, + error: [HttpApiError.BadRequest, ConflictError], }).annotateMerge( OpenApi.annotations({ identifier: "sync.steal", @@ -84,13 +90,13 @@ export const SyncApi = HttpApi.make("sync") query: WorkspaceRoutingQuery, payload: HistoryPayload, success: described(Schema.Array(HistoryEvent), "Sync events"), - error: HttpApiError.BadRequest, + error: [HttpApiError.BadRequest, ServiceUnavailableError], }).annotateMerge( OpenApi.annotations({ identifier: "sync.history.list", summary: "List sync events", description: - "List sync events for all aggregates. Keys are aggregate IDs the client already knows about, values are the last known sequence ID. Events with seq > value are returned for those aggregates. Aggregates not listed in the input get their full history.", + "List a bounded page of sync events. Keys are aggregate IDs the client already knows about, values are the last known sequence ID. Oversized legacy events fail closed until maintenance migration externalizes their inline payload.", }), ), ) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/workspace.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/workspace.ts index 795ed421f..c509af99c 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/workspace.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/workspace.ts @@ -2,7 +2,7 @@ import { Workspace } from "@/control-plane/workspace" import { WorkspaceAdapterEntry } from "@/control-plane/types" import { Schema, Struct } from "effect" import { HttpApi, HttpApiEndpoint, HttpApiError, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi" -import { ApiVcsApplyError } from "./instance" +import { ApiVcsApplyError, ApiVcsRawDiffError } from "./instance" import { ApiNotFoundError } from "../errors" import { Authorization } from "../middleware/authorization" import { InstanceContextMiddleware } from "../middleware/instance-context" @@ -118,7 +118,7 @@ export const WorkspaceApi = HttpApi.make("workspace") query: WorkspaceRoutingQuery, payload: WarpPayload, success: described(HttpApiSchema.NoContent, "Session warped"), - error: [ApiWorkspaceWarpError, ApiVcsApplyError, ApiNotFoundError], + error: [ApiWorkspaceWarpError, ApiVcsApplyError, ApiVcsRawDiffError, ApiNotFoundError], }).annotateMerge( OpenApi.annotations({ identifier: "experimental.workspace.warp", diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/instance.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/instance.ts index 354239e73..118334560 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/instance.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/instance.ts @@ -9,7 +9,7 @@ import { Skill } from "@/skill" import { Effect } from "effect" import { HttpApiBuilder } from "effect/unstable/httpapi" import { InstanceHttpApi } from "../api" -import { ApiVcsApplyError } from "../groups/instance" +import { ApiVcsApplyError, ApiVcsRawDiffError } from "../groups/instance" import { markInstanceForDisposal } from "../lifecycle" import { Config } from "@/config/config" import { snapshotGateway } from "@/deepagent/config" @@ -95,7 +95,21 @@ export const instanceHandlers = HttpApiBuilder.group(InstanceHttpApi, "instance" }) const getVcsDiffRaw = Effect.fn("InstanceHttpApi.vcsDiffRaw")(function* () { - return yield* vcs.diffRaw() + return yield* vcs.diffRaw().pipe( + Effect.mapError( + (error) => + new ApiVcsRawDiffError({ + name: "VcsRawDiffError", + data: { + message: error.message, + reason: error.reason, + limit: error.limit, + actual: error.actual, + file: error.file, + }, + }), + ), + ) }) const applyVcs = Effect.fn("InstanceHttpApi.vcsApply")(function* (ctx: { payload: Vcs.ApplyInput }) { diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session-errors.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session-errors.ts index eed6542a6..672501b0c 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session-errors.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session-errors.ts @@ -1,5 +1,6 @@ import type { NotFoundError as StorageNotFoundError } from "@/storage/storage" import { Session } from "@/session/session" +import { SessionRevert } from "@/session/revert" import { Effect } from "effect" import * as ApiError from "../errors" @@ -29,3 +30,24 @@ export function mapBusy(self: Effect.Effect) { ), ) } + +export function mapRevert(self: Effect.Effect) { + return self.pipe( + Effect.catchTag("SessionBusyError", (error) => + Effect.fail( + new ApiError.SessionBusyError({ + sessionID: error.sessionID, + message: `Session is busy: ${error.sessionID}`, + }), + ), + ), + Effect.catchTag("SessionRevertLimitError", (error) => + Effect.fail( + new ApiError.ServiceUnavailableError({ + service: "session.revert", + message: `Session revert exceeds the bounded ${error.maxFiles}-file safety limit`, + }), + ), + ), + ) +} diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts index 63dbd13bb..b338ce211 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts @@ -18,6 +18,7 @@ import { SessionRevert } from "@/session/revert" import { SessionRunState } from "@/session/run-state" import { SessionStatus } from "@/session/status" import { SessionSummary } from "@/session/summary" +import { SessionLegacyProviderResolution } from "@/session/legacy-provider-resolution" import { Todo } from "@/session/todo" import { MessageID, PartID, SessionID } from "@/session/schema" import { NamedError } from "@deepagent-code/core/util/error" @@ -39,6 +40,7 @@ import { PermissionResponsePayload, PromptPreparePayload, PromptPayload, + ProviderResolutionPayload, RevertPayload, ShellPayload, SummarizePayload, @@ -47,6 +49,7 @@ import { import { ConflictError, PermissionNotFoundError, notFound } from "../errors" import * as SessionError from "./session-errors" import { randomUUID } from "node:crypto" +import { getWorkspaceContext } from "../utils/workspace-context" const tryParseJson = (text: string) => Effect.try({ @@ -85,6 +88,7 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", const summary = yield* SessionSummary.Service const events = yield* EventV2Bridge.Service const contextDiagnosticsSvc = yield* ContextFederationDiagnostics.Service + const providerResolutionSvc = yield* SessionLegacyProviderResolution.Service const scope = yield* Scope.Scope const list = Effect.fn("SessionHttpApi.list")(function* (ctx: { query: typeof ListQuery.Type }) { @@ -154,7 +158,6 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", params: { sessionID: SessionID } query: typeof MessagesQuery.Type }) { - if (ctx.query.before && ctx.query.limit === undefined) return yield* new HttpApiError.BadRequest({}) if (ctx.query.before) { const before = ctx.query.before yield* Effect.try({ @@ -163,26 +166,23 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", }) } yield* requireSession(ctx.params.sessionID) - if (ctx.query.limit === undefined || ctx.query.limit === 0) { - return yield* SessionError.mapStorageNotFound(session.messages({ sessionID: ctx.params.sessionID })) - } - const page = yield* SessionError.mapStorageNotFound( - MessageV2.page({ + MessageV2.clientPage({ sessionID: ctx.params.sessionID, - limit: ctx.query.limit, + limit: ctx.query.limit ?? MessageV2.ClientMessageLimits.page, before: ctx.query.before, }), ) - if (!page.cursor) return page.items + const items = page.items.map((item) => ({ ...item, info: MessageV2.clientProjection(item.info) })) + if (!page.cursor) return items const request = yield* HttpServerRequest.HttpServerRequest // toURL() honors the Host + x-forwarded-proto headers, so the Link // header echoes the real origin instead of a hard-coded localhost. const url = Option.getOrElse(HttpServerRequest.toURL(request), () => new URL(request.url, "http://localhost")) - url.searchParams.set("limit", ctx.query.limit.toString()) + url.searchParams.set("limit", (ctx.query.limit ?? MessageV2.ClientMessageLimits.page).toString()) url.searchParams.set("before", page.cursor) - return HttpServerResponse.jsonUnsafe(page.items, { + return HttpServerResponse.jsonUnsafe(items, { headers: { "Access-Control-Expose-Headers": "Link, X-Next-Cursor", Link: `<${url.toString()}>; rel="next"`, @@ -194,9 +194,10 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", const message = Effect.fn("SessionHttpApi.message")(function* (ctx: { params: { sessionID: SessionID; messageID: MessageID } }) { - return yield* SessionError.mapStorageNotFound( - MessageV2.get({ sessionID: ctx.params.sessionID, messageID: ctx.params.messageID }), + const result = yield* SessionError.mapStorageNotFound( + MessageV2.clientGet({ sessionID: ctx.params.sessionID, messageID: ctx.params.messageID }), ) + return result }) const create = Effect.fn("SessionHttpApi.create")(function* (ctx: { payload?: Session.CreateInput }) { @@ -563,7 +564,7 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", payload: typeof RevertPayload.Type }) { yield* requireSession(ctx.params.sessionID) - return yield* SessionError.mapBusy(revertSvc.revert({ sessionID: ctx.params.sessionID, ...ctx.payload })) + return yield* SessionError.mapRevert(revertSvc.revert({ sessionID: ctx.params.sessionID, ...ctx.payload })) }) const unrevert = Effect.fn("SessionHttpApi.unrevert")(function* (ctx: { params: { sessionID: SessionID } }) { @@ -668,6 +669,32 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", return resolved }) + const providerResolutionList = Effect.fn("SessionHttpApi.providerResolutionList")(function* (ctx: { + params: { sessionID: SessionID } + }) { + yield* requireSession(ctx.params.sessionID) + return yield* providerResolutionSvc + .describe(ctx.params.sessionID) + .pipe(Effect.mapError((error) => notFound(error.reason))) + }) + + const providerResolutionResolve = Effect.fn("SessionHttpApi.providerResolutionResolve")(function* (ctx: { + params: { sessionID: SessionID } + payload: typeof ProviderResolutionPayload.Type + }) { + yield* requireSession(ctx.params.sessionID) + const actor = yield* getWorkspaceContext() + return yield* providerResolutionSvc + .resolve({ ...ctx.payload, sessionID: ctx.params.sessionID, actorID: actor.userID }) + .pipe( + Effect.mapError((error) => + error instanceof SessionLegacyProviderResolution.NotFound + ? notFound(error.reason) + : new ConflictError({ message: error.reason, resource: error.code }), + ), + ) + }) + return handlers .handle("list", list) .handle("status", status) @@ -702,5 +729,7 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", .handle("updatePart", updatePart) .handle("contextDiagnostics", contextDiagnostics) .handle("contextAttemptResolve", contextAttemptResolve) + .handle("providerResolutionList", providerResolutionList) + .handle("providerResolutionResolve", providerResolutionResolve) }), ) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts index 5e580a116..d9dfe43f1 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts @@ -1,24 +1,61 @@ import { Workspace } from "@/control-plane/workspace" import * as InstanceState from "@/effect/instance-state" import { Session } from "@/session/session" +import { MessageV2 } from "@/session/message-v2" +import { SessionV1 } from "@deepagent-code/core/v1/session" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" import { EventV2Bridge } from "@/event-v2-bridge" import { EventTable } from "@deepagent-code/core/event/sql" +import { SessionRecoveryTransferGuard } from "@/session/recovery-transfer-guard" import { asc } from "drizzle-orm" import { and } from "drizzle-orm" import { eq } from "drizzle-orm" +import { inArray } from "drizzle-orm" import { lte } from "drizzle-orm" import { not } from "drizzle-orm" import { or } from "drizzle-orm" -import { Effect, Scope } from "effect" +import { sql } from "drizzle-orm" +import { Effect, Option, Schema, Scope } from "effect" import { HttpApiBuilder, HttpApiError } from "effect/unstable/httpapi" import { InstanceHttpApi } from "../api" -import { HistoryPayload, ReplayPayload, SessionPayload } from "../groups/sync" +import { HistoryPayload, ReplayPayload, SessionPayload, SyncReplayLimits } from "../groups/sync" +import { encodeReplayRequestPrefix } from "@/sync/replay-protocol" import * as Log from "@deepagent-code/core/util/log" +import { ServiceUnavailableError } from "../errors" +import { ConflictError } from "../errors" const log = Log.create({ service: "server.sync" }) - +export const SyncHistoryLimits = { + events: 100, + bytes: 4 * 1024 * 1024, +} as const +const decodeJson = Schema.decodeUnknownOption(Schema.UnknownFromJsonString) +const record = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) +const projectedEventData = sql`CASE + WHEN ${EventTable.type} = ${EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1)} + AND json_valid(${EventTable.data}) + AND json_type(${EventTable.data}, '$.info.summary.diffs') = 'array' + AND length(${EventTable.data}) > ${SyncHistoryLimits.bytes} + THEN json_set(${EventTable.data}, '$.info.summary.diffs', json('[]')) + WHEN ${EventTable.type} = ${EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1)} + AND json_valid(${EventTable.data}) + AND json_type(${EventTable.data}, '$.info.summary.diffs') = 'array' + THEN json_set( + ${EventTable.data}, + '$.info.summary.diffs', + json(COALESCE(( + SELECT json_group_array(json_remove(value, '$.patch')) + FROM ( + SELECT value + FROM json_each(json_extract(${EventTable.data}, '$.info.summary.diffs')) + LIMIT ${MessageV2.ClientDiffLimits.files} + ) + ), '[]')) + ) + ELSE ${EventTable.data} +END` export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handlers) => Effect.gen(function* () { const workspace = yield* Workspace.Service @@ -35,13 +72,9 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl }) const replay = Effect.fn("SyncHttpApi.replay")(function* (ctx: { payload: typeof ReplayPayload.Type }) { - const payload: EventV2.SerializedEvent[] = ctx.payload.events.map((event) => ({ - id: event.id, - aggregateID: event.aggregateID, - seq: event.seq, - type: event.type, - data: { ...event.data }, - })) + const payload: EventV2.SerializedEvent[] = [...ctx.payload.events] + const encoded = encodeReplayRequestPrefix(ctx.payload.directory, payload) + if (!encoded.complete || encoded.events.length !== payload.length) return yield* new HttpApiError.BadRequest({}) const source = payload[0].aggregateID log.info("sync replay requested", { sessionID: source, @@ -65,6 +98,14 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl const workspaceID = yield* InstanceState.workspaceID if (!workspaceID) return yield* new HttpApiError.BadRequest({}) + const recoveryAuthorityID = yield* SessionRecoveryTransferGuard.authorityID(db, ctx.payload.sessionID) + if (recoveryAuthorityID) + return yield* new ConflictError({ + message: + "Provider recovery projection cannot move between workspaces until canonical snapshot import is available", + resource: recoveryAuthorityID, + }) + yield* session.setWorkspace({ sessionID: ctx.payload.sessionID, workspaceID }) log.info("sync session stolen", { @@ -77,17 +118,83 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl const history = Effect.fn("SyncHttpApi.history")(function* (ctx: { payload: typeof HistoryPayload.Type }) { const exclude = Object.entries(ctx.payload) - return yield* db - .select() + const where = + exclude.length > 0 + ? not(or(...exclude.map(([id, seq]) => and(eq(EventTable.aggregate_id, id), lte(EventTable.seq, seq))))!) + : undefined + const candidates = yield* db + .select({ + id: EventTable.id, + bytes: sql`length(CAST(${EventTable.data} AS BLOB))`, + }) + .from(EventTable) + .where(where) + .orderBy(asc(EventTable.seq), asc(EventTable.aggregate_id)) + .limit(SyncHistoryLimits.events) + .all() + .pipe(Effect.orDie) + const page = candidates.reduce<{ + ids: EventV2.ID[] + bytes: number + oversized: string | undefined + closed: boolean + }>( + (result, row) => { + if (result.closed) return result + if (row.bytes > SyncHistoryLimits.bytes) + return { + ...result, + oversized: result.ids.length === 0 ? row.id : undefined, + closed: true, + } + if (result.bytes > 0 && result.bytes + row.bytes > SyncHistoryLimits.bytes) return { ...result, closed: true } + return { + ids: [...result.ids, row.id], + bytes: result.bytes + row.bytes, + oversized: undefined, + closed: false, + } + }, + { ids: [], bytes: 0, oversized: undefined, closed: false }, + ) + if (page.oversized) + return yield* new ServiceUnavailableError({ + service: "sync.history", + message: `Legacy sync event ${page.oversized} exceeds the bounded history page and requires artifact migration`, + }) + if (page.ids.length === 0) return [] + const rows = yield* db + .select({ + id: EventTable.id, + aggregate_id: EventTable.aggregate_id, + seq: EventTable.seq, + type: EventTable.type, + data: projectedEventData, + }) .from(EventTable) - .where( - exclude.length > 0 - ? not(or(...exclude.map(([id, seq]) => and(eq(EventTable.aggregate_id, id), lte(EventTable.seq, seq))))!) - : undefined, - ) - .orderBy(asc(EventTable.seq)) + .where(inArray(EventTable.id, page.ids)) + .orderBy(asc(EventTable.seq), asc(EventTable.aggregate_id)) .all() .pipe(Effect.orDie) + const result = rows.map((row) => { + const data = Option.getOrUndefined(decodeJson(row.data)) + return { ...row, data: record(data) ? data : {} } + }) + const bounded = result.reduce<{ items: typeof result; bytes: number; closed: boolean }>( + (page, item) => { + if (page.closed) return page + const bytes = Buffer.byteLength(JSON.stringify(item)) + (page.items.length > 0 ? 1 : 0) + if (page.bytes + bytes + 2 > SyncHistoryLimits.bytes) return { ...page, closed: true } + return { items: [...page.items, item], bytes: page.bytes + bytes, closed: false } + }, + { items: [], bytes: 0, closed: false }, + ) + if (bounded.items.length === 0 && result.length > 0) + return yield* new ServiceUnavailableError({ + service: "sync.history", + message: `Projected sync event ${result[0]!.id} exceeds the bounded history page`, + }) + return bounded.items }) return handlers.handle("start", start).handle("replay", replay).handle("steal", steal).handle("history", history) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/workspace.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/workspace.ts index 7f5c437d9..abacf5632 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/workspace.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/workspace.ts @@ -6,7 +6,7 @@ import { Cause, Effect } from "effect" import { HttpApiBuilder } from "effect/unstable/httpapi" import { InstanceHttpApi } from "../api" import { notFound } from "../errors" -import { ApiVcsApplyError } from "../groups/instance" +import { ApiVcsApplyError, ApiVcsRawDiffError } from "../groups/instance" import { ApiWorkspaceCreateError, ApiWorkspaceWarpError, CreatePayload, WarpPayload } from "../groups/workspace" export const workspaceHandlers = HttpApiBuilder.group(InstanceHttpApi, "workspace", (handlers) => @@ -80,6 +80,18 @@ export const workspaceHandlers = HttpApiBuilder.group(InstanceHttpApi, "workspac }, }) } + if (error instanceof Vcs.RawDiffError) { + return new ApiVcsRawDiffError({ + name: "VcsRawDiffError", + data: { + message: error.message, + reason: error.reason, + limit: error.limit, + actual: error.actual, + file: error.file, + }, + }) + } return new ApiWorkspaceWarpError({ name: "WorkspaceWarpError", data: { diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts new file mode 100644 index 000000000..efea76dca --- /dev/null +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts @@ -0,0 +1,64 @@ +import { Effect, FileSystem, Stream } from "effect" +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import { SyncPaths, SyncReplayLimits } from "../groups/sync" + +const bodyLimitExceeded = Symbol("sync-replay-body-limit-exceeded") + +export const syncReplayBodyLimitLayer = HttpRouter.middleware<{ handles: unknown }>()((effect) => + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest + if (request.method !== "POST" || pathOf(request.url) !== SyncPaths.replay) return yield* effect + + const contentLength = Number(request.headers["content-length"]) + if (Number.isFinite(contentLength) && contentLength > SyncReplayLimits.requestBytes) + return HttpServerResponse.empty({ status: 413 }) + + if (!(request.source instanceof Request)) + return yield* effect.pipe( + Effect.provideService(HttpServerRequest.MaxBodySize, FileSystem.Size(SyncReplayLimits.requestBytes)), + ) + + const buffered = yield* bufferWebBody(request.source) + if (!buffered) return HttpServerResponse.empty({ status: 413 }) + return yield* effect.pipe( + Effect.provideService( + HttpServerRequest.HttpServerRequest, + HttpServerRequest.fromWeb( + new Request(request.source.url, { + method: request.source.method, + headers: request.source.headers, + body: buffered, + signal: request.source.signal, + }), + ), + ), + ) + }), +).layer + +function pathOf(url: string) { + const queryIndex = url.indexOf("?") + const hashIndex = url.indexOf("#") + const end = queryIndex === -1 ? hashIndex : hashIndex === -1 ? queryIndex : Math.min(queryIndex, hashIndex) + return end === -1 ? url : url.slice(0, end) +} + +function bufferWebBody(request: Request) { + if (!request.body) return Effect.succeed(new Uint8Array()) + return Stream.fromReadableStream({ + evaluate: () => request.body!, + onError: (cause) => cause, + }).pipe( + Stream.runFoldEffect( + () => ({ chunks: [] as Uint8Array[], bytes: 0 }), + (result, chunk) => { + const bytes = result.bytes + chunk.byteLength + if (bytes > SyncReplayLimits.requestBytes) return Effect.fail(bodyLimitExceeded) + result.chunks.push(chunk) + return Effect.succeed({ chunks: result.chunks, bytes }) + }, + ), + Effect.map((result) => Buffer.concat(result.chunks, result.bytes)), + Effect.catch((error) => (error === bodyLimitExceeded ? Effect.succeed(undefined) : Effect.die(error))), + ) +} diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/server.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/server.ts index 638e9614b..a012ec1b5 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/server.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/server.ts @@ -49,6 +49,7 @@ import { SessionSteer } from "@/session/steer" import { SessionRunState } from "@/session/run-state" import { SessionStatus } from "@/session/status" import { SessionSummary } from "@/session/summary" +import { SessionLegacyProviderResolution } from "@/session/legacy-provider-resolution" import { Todo } from "@/session/todo" import { SessionShare } from "@/share/session" import { ShareNext } from "@/share/share-next" @@ -135,6 +136,7 @@ import { corsVaryFix } from "./middleware/cors-vary" import { errorLayer } from "./middleware/error" import { fenceLayer } from "./middleware/fence" import { schemaErrorLayer } from "./middleware/schema-error" +import { syncReplayBodyLimitLayer } from "./middleware/sync-replay-body-limit" export const context = Context.makeUnsafe(new Map()) @@ -330,6 +332,7 @@ export function createRoutes( Layer.provide([ errorLayer, compressionLayer, + syncReplayBodyLimitLayer, corsVaryFix, fenceLayer.pipe(Layer.provide(Database.defaultLayer)), cors(corsOptions), @@ -373,6 +376,7 @@ export function createRoutes( SessionRunState.defaultLayer, SessionStatus.defaultLayer, SessionSummary.defaultLayer, + SessionLegacyProviderResolution.defaultLayer, ShareNext.defaultLayer, Snapshot.defaultLayer, EventV2Bridge.defaultLayer, diff --git a/packages/deepagent-code/src/session/legacy-provider-resolution.ts b/packages/deepagent-code/src/session/legacy-provider-resolution.ts new file mode 100644 index 000000000..3b2236815 --- /dev/null +++ b/packages/deepagent-code/src/session/legacy-provider-resolution.ts @@ -0,0 +1,765 @@ +import { Database } from "@deepagent-code/core/database/database" +import { EventV2 } from "@deepagent-code/core/event" +import { NonNegativeInt } from "@deepagent-code/core/schema" +import { + SessionHistoryStateTable, + SessionIntentTable, + SessionPromptEpochMessageTable, + SessionPromptEpochRecoveryTable, + SessionSteerTable, + SessionTable, + SessionToolRequestResolutionCommandTable, + SessionToolRequestResolutionTable, + SessionWorldStateBaselineTable, +} from "@deepagent-code/core/session/sql" +import { CanonicalJson } from "@deepagent-code/core/util/canonical-json" +import { Hash } from "@deepagent-code/core/util/hash" +import { and, eq, inArray, isNull, sql } from "drizzle-orm" +import { Cause, Context, Data, Effect, Exit, Layer, Option, Schema } from "effect" +import { SessionActivityProgressTable, SessionLegacyActivityTable } from "./activity-sql" +import { CompactionRunTable } from "./compaction-sql" +import { HistoryAuthority } from "./history-authority" +import { MessageV2 } from "./message-v2" +import { SessionPromptEpochTable } from "./prompt-epoch.sql" +import { MessageID, SessionID } from "./schema" +import { SessionToolRequestReceiptTable } from "./tool-request-receipt.sql" +import { EventV2Bridge } from "../event-v2-bridge" +import { WorldStateSlot } from "@deepagent-code/core/deepagent/context/world-state" +import { + renderSessionWorldStateBaseline, + sessionWorldStateBaselineHash, + type SessionWorldStateBaselineSection, +} from "./context-ledger" + +const REQUEST_VERSION = 1 +const DEFAULT_REASON = "User chose to abandon the provider request whose outcome is unknown after process restart" + +export const Expected = Schema.Struct({ + providerState: Schema.Literal("indeterminate_after_crash"), + promptEpoch: NonNegativeInt, + sessionMutationEpoch: NonNegativeInt, + requestHash: Schema.String, + historyHash: Schema.String, + worldStateBaselineHash: Schema.String, +}) + +export const ResolveInput = Schema.Struct({ + sessionID: SessionID, + commandID: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(200)), + receiptID: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(200)), + decision: Schema.Literal("abandoned"), + expected: Expected, + reason: Schema.optional(Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(2000))), + riskAcknowledged: Schema.optional(Schema.Boolean), +}) + +export const Descriptor = Schema.Struct({ + receiptID: Schema.String, + sessionID: SessionID, + assistantMessageID: MessageID, + providerID: Schema.String, + modelID: Schema.String, + providerState: Schema.Literal("indeterminate_after_crash"), + promptEpoch: NonNegativeInt, + promptWindowID: Schema.String, + historyHash: Schema.String, + requestHash: Schema.String, + sessionMutationEpoch: NonNegativeInt, + continuationRecoverySupported: Schema.Boolean, + workspaceRecoverySupported: Schema.Boolean, + sourceWorldStateBaselineStatus: Schema.Literals(["available", "missing", "invalid"]), + worldStateBaselineHash: Schema.optional(Schema.String), +}) + +export const Resolution = Schema.Struct({ + resolutionID: Schema.String, + commandID: Schema.String, + receiptID: Schema.String, + sessionID: SessionID, + decision: Schema.Literal("abandoned"), + sourcePromptEpoch: NonNegativeInt, + successorPromptEpoch: NonNegativeInt, + sourceMutationEpoch: NonNegativeInt, + successorMutationEpoch: NonNegativeInt, + safeEndMessageID: Schema.optional(MessageID), + safeHistoryHash: Schema.String, + successorWindowID: Schema.String, + successorHistoryHash: Schema.String, + createdAt: NonNegativeInt, +}) + +export const Event = { + Completed: EventV2.define({ + type: "session.provider-resolution.completed", + schema: { + sessionID: SessionID, + resolutionID: Schema.String, + commandID: Schema.String, + receiptID: Schema.String, + decision: Schema.Literal("abandoned"), + sourcePromptEpoch: NonNegativeInt, + successorPromptEpoch: NonNegativeInt, + sourceMutationEpoch: NonNegativeInt, + successorMutationEpoch: NonNegativeInt, + createdAt: NonNegativeInt, + }, + }), +} + +export class Conflict extends Data.TaggedError("SessionLegacyProviderResolution.Conflict")<{ + readonly code: + | "command_id_conflict" + | "receipt_already_resolved" + | "continuation_recovery_not_supported" + | "workspace_recovery_not_supported" + | "source_world_state_baseline_missing" + | "source_world_state_baseline_invalid" + | "stale_recovery_state" + | "stale_session_mutation" + | "unsafe_history_boundary" + | "decision_not_supported" + readonly reason: string +}> {} + +export class NotFound extends Data.TaggedError("SessionLegacyProviderResolution.NotFound")<{ + readonly reason: string +}> {} + +export interface Interface { + readonly describe: (sessionID: SessionID) => Effect.Effect<(typeof Descriptor.Type)[], NotFound> + readonly resolve: ( + input: typeof ResolveInput.Type & { actorID: string }, + ) => Effect.Effect +} + +export class Service extends Context.Service()("@deepagent-code/SessionLegacyProviderResolution") {} + +const requestHash = (input: typeof ResolveInput.Type) => + Hash.sha256( + CanonicalJson.stringify({ + version: REQUEST_VERSION, + sessionID: input.sessionID, + commandID: input.commandID, + receiptID: input.receiptID, + decision: input.decision, + expected: input.expected, + reason: input.reason ?? DEFAULT_REASON, + riskAcknowledged: input.riskAcknowledged ?? false, + }), + ) + +const resolutionID = (input: Pick) => + `res_${Hash.sha256(`${input.sessionID}:${input.receiptID}:${input.commandID}`).slice(0, 40)}` + +const fromResolution = ( + commandID: string, + row: typeof SessionToolRequestResolutionTable.$inferSelect, +): typeof Resolution.Type => ({ + resolutionID: row.resolution_id, + commandID, + receiptID: row.receipt_id, + sessionID: row.session_id, + decision: row.decision, + sourcePromptEpoch: row.source_prompt_epoch, + successorPromptEpoch: row.successor_prompt_epoch, + sourceMutationEpoch: row.source_mutation_epoch, + successorMutationEpoch: row.successor_mutation_epoch, + ...(row.safe_end_message_id ? { safeEndMessageID: row.safe_end_message_id } : {}), + safeHistoryHash: row.safe_history_hash, + successorWindowID: row.successor_window_id, + successorHistoryHash: row.successor_history_hash, + createdAt: row.created_at, +}) + +const service = Effect.gen(function* () { + const { db } = yield* Database.Service + const events = yield* EventV2Bridge.Service + + const describe: Interface["describe"] = Effect.fn("SessionLegacyProviderResolution.describe")(function* (sessionID) { + const session = yield* db + .select({ mutationEpoch: SessionTable.mutation_epoch, workspaceID: SessionTable.workspace_id }) + .from(SessionTable) + .where(eq(SessionTable.id, sessionID)) + .get() + .pipe(Effect.orDie) + if (!session) return yield* new NotFound({ reason: "session was not found" }) + const rows = yield* db + .select({ receipt: SessionToolRequestReceiptTable }) + .from(SessionToolRequestReceiptTable) + .leftJoin( + SessionToolRequestResolutionTable, + eq(SessionToolRequestResolutionTable.receipt_id, SessionToolRequestReceiptTable.receipt_id), + ) + .where( + and( + eq(SessionToolRequestReceiptTable.session_id, sessionID), + eq(SessionToolRequestReceiptTable.provider_state, "indeterminate_after_crash"), + isNull(SessionToolRequestReceiptTable.provider_attempt_id), + isNull(SessionToolRequestResolutionTable.resolution_id), + ), + ) + .all() + .pipe(Effect.orDie) + return yield* Effect.forEach(rows, (item) => + Effect.gen(function* () { + const receipt = item.receipt + const request = receipt.final_request_hash ?? receipt.provider_request_hash ?? receipt.request_input_hash + if ( + !receipt.assistant_message_id || + receipt.prompt_epoch === null || + !receipt.prompt_window_id || + !receipt.effective_history_hash || + !request + ) + return + const continuation = yield* db + .select({ runID: CompactionRunTable.run_id }) + .from(CompactionRunTable) + .where( + and( + eq(CompactionRunTable.continuation_receipt_id, receipt.receipt_id), + eq(CompactionRunTable.continuation_state, "indeterminate"), + ), + ) + .get() + .pipe(Effect.orDie) + const epoch = yield* db + .select({ baselineHash: SessionPromptEpochTable.world_state_baseline_hash }) + .from(SessionPromptEpochTable) + .where( + and( + eq(SessionPromptEpochTable.session_id, sessionID), + eq(SessionPromptEpochTable.epoch, receipt.prompt_epoch), + ), + ) + .get() + .pipe(Effect.orDie) + const baseline = yield* db + .select() + .from(SessionWorldStateBaselineTable) + .where( + and( + eq(SessionWorldStateBaselineTable.session_id, sessionID), + eq(SessionWorldStateBaselineTable.prompt_epoch, receipt.prompt_epoch), + ), + ) + .all() + .pipe(Effect.orDie) + const sections = baseline.flatMap((row) => { + const snapshot = Option.getOrUndefined(Schema.decodeUnknownOption(WorldStateSlot)(row.snapshot)) + if (!snapshot) return [] + return [{ sectionID: row.section_id, snapshot, fragment: row.fragment, fragmentHash: row.fragment_hash }] + }) satisfies SessionWorldStateBaselineSection[] + const computed = + sections.length === baseline.length && sections.length > 0 + ? sessionWorldStateBaselineHash({ sections, rendered: renderSessionWorldStateBaseline(sections) }) + : undefined + const baselineStatus = + !epoch?.baselineHash || baseline.length === 0 + ? ("missing" as const) + : computed !== epoch.baselineHash + ? ("invalid" as const) + : ("available" as const) + return { + receiptID: receipt.receipt_id, + sessionID, + assistantMessageID: MessageID.make(receipt.assistant_message_id), + providerID: receipt.provider_id, + modelID: receipt.model_id, + providerState: "indeterminate_after_crash" as const, + promptEpoch: receipt.prompt_epoch, + promptWindowID: receipt.prompt_window_id, + historyHash: receipt.effective_history_hash, + requestHash: request, + sessionMutationEpoch: session.mutationEpoch, + continuationRecoverySupported: !continuation, + workspaceRecoverySupported: session.workspaceID === null, + sourceWorldStateBaselineStatus: baselineStatus, + ...(baselineStatus === "available" && epoch?.baselineHash + ? { worldStateBaselineHash: epoch.baselineHash } + : {}), + } + }), + ).pipe(Effect.map((items) => items.filter((item): item is NonNullable => item !== undefined))) + }) + + const resolve: Interface["resolve"] = Effect.fn("SessionLegacyProviderResolution.resolve")(function* (input) { + if (input.decision !== "abandoned" || input.riskAcknowledged) + return yield* new Conflict({ + code: "decision_not_supported", + reason: "the Incident Hotfix only supports abandoned", + }) + const hash = requestHash(input) + const proposed = { + resolutionID: resolutionID(input), + successorPromptEpoch: input.expected.promptEpoch + 1, + successorMutationEpoch: input.expected.sessionMutationEpoch + 1, + createdAt: Date.now(), + } + const transaction = yield* Effect.exit( + db.transaction( + (tx) => + Effect.gen(function* () { + const inserted = yield* tx + .insert(SessionToolRequestResolutionCommandTable) + .values({ + command_id: input.commandID, + request_hash: hash, + session_id: input.sessionID, + receipt_id: input.receiptID, + result_resolution_id: null, + created_at: proposed.createdAt, + }) + .onConflictDoNothing() + .returning() + .get() + const command = + inserted ?? + (yield* tx + .select() + .from(SessionToolRequestResolutionCommandTable) + .where(eq(SessionToolRequestResolutionCommandTable.command_id, input.commandID)) + .get()) + if ( + !command || + command.request_hash !== hash || + command.session_id !== input.sessionID || + command.receipt_id !== input.receiptID + ) + return yield* new Conflict({ + code: "command_id_conflict", + reason: "command ID was reused with different recovery input", + }) + const known = command.result_resolution_id + ? yield* tx + .select() + .from(SessionToolRequestResolutionTable) + .where(eq(SessionToolRequestResolutionTable.resolution_id, command.result_resolution_id)) + .get() + : undefined + if (command.result_resolution_id && !known) + return yield* Effect.die(new Error(`resolution result disappeared: ${command.result_resolution_id}`)) + if (known) return known + + const exact = yield* tx + .select() + .from(SessionToolRequestResolutionTable) + .where(eq(SessionToolRequestResolutionTable.resolution_id, proposed.resolutionID)) + .get() + if (exact) { + const repaired = yield* tx + .update(SessionToolRequestResolutionCommandTable) + .set({ result_resolution_id: exact.resolution_id }) + .where( + and( + eq(SessionToolRequestResolutionCommandTable.command_id, input.commandID), + eq(SessionToolRequestResolutionCommandTable.request_hash, hash), + isNull(SessionToolRequestResolutionCommandTable.result_resolution_id), + ), + ) + .returning({ commandID: SessionToolRequestResolutionCommandTable.command_id }) + .get() + if (!repaired) { + const current = yield* tx + .select({ result: SessionToolRequestResolutionCommandTable.result_resolution_id }) + .from(SessionToolRequestResolutionCommandTable) + .where(eq(SessionToolRequestResolutionCommandTable.command_id, input.commandID)) + .get() + if (current?.result !== exact.resolution_id) + return yield* Effect.die(new Error("provider recovery command result CAS failed")) + } + return exact + } + + const session = yield* tx + .select({ mutationEpoch: SessionTable.mutation_epoch, workspaceID: SessionTable.workspace_id }) + .from(SessionTable) + .where(eq(SessionTable.id, input.sessionID)) + .get() + if (!session) return yield* new NotFound({ reason: "session was not found" }) + if (session.workspaceID !== null) + return yield* new Conflict({ + code: "workspace_recovery_not_supported", + reason: "workspace ownership changed before provider recovery commit", + }) + if (session.mutationEpoch !== input.expected.sessionMutationEpoch) + return yield* new Conflict({ code: "stale_session_mutation", reason: "session mutation epoch changed" }) + const receipt = yield* tx + .select() + .from(SessionToolRequestReceiptTable) + .where( + and( + eq(SessionToolRequestReceiptTable.receipt_id, input.receiptID), + eq(SessionToolRequestReceiptTable.session_id, input.sessionID), + ), + ) + .get() + if (!receipt) return yield* new NotFound({ reason: "provider receipt was not found" }) + const resolved = yield* tx + .select() + .from(SessionToolRequestResolutionTable) + .where(eq(SessionToolRequestResolutionTable.receipt_id, input.receiptID)) + .get() + if (resolved) + return yield* new Conflict({ + code: "receipt_already_resolved", + reason: `provider receipt was already resolved as ${resolved.decision}`, + }) + const continuation = yield* tx + .select({ runID: CompactionRunTable.run_id }) + .from(CompactionRunTable) + .where( + and( + eq(CompactionRunTable.continuation_receipt_id, input.receiptID), + eq(CompactionRunTable.continuation_state, "indeterminate"), + ), + ) + .get() + if (continuation) + return yield* new Conflict({ + code: "continuation_recovery_not_supported", + reason: `compaction continuation ${continuation.runID} requires the Maintenance recovery protocol`, + }) + const source = yield* tx + .select() + .from(SessionPromptEpochTable) + .where( + and( + eq(SessionPromptEpochTable.session_id, input.sessionID), + eq(SessionPromptEpochTable.epoch, input.expected.promptEpoch), + eq(SessionPromptEpochTable.state, "active"), + ), + ) + .get() + const receiptRequestHash = + receipt.final_request_hash ?? receipt.provider_request_hash ?? receipt.request_input_hash + if ( + !source || + source.authority_state !== "recovery_required" || + receipt.provider_attempt_id !== null || + receipt.provider_state !== input.expected.providerState || + receipt.prompt_epoch !== input.expected.promptEpoch || + receipt.prompt_window_id !== source.window_id || + receipt.effective_history_hash !== input.expected.historyHash || + receiptRequestHash !== input.expected.requestHash + ) + return yield* new Conflict({ + code: "stale_recovery_state", + reason: "provider recovery authority changed", + }) + if (!receipt.assistant_message_id) + return yield* new Conflict({ + code: "unsafe_history_boundary", + reason: "ambiguous assistant binding is missing", + }) + + // The physical message snapshot and successor high-water are read under the same + // IMMEDIATE transaction that retires the source authority and installs its successor. + const transactionDatabase = tx as unknown as Database.Interface["db"] + const cutoffAuthority = yield* MessageV2.promptHistoryCutoffAuthorityInTransaction(transactionDatabase, { + sessionID: input.sessionID, + cutoffMessageID: MessageID.make(receipt.user_message_id), + }) + if (!cutoffAuthority) + return yield* new Conflict({ + code: "unsafe_history_boundary", + reason: "safe history prefix is not provable", + }) + const chronological = cutoffAuthority.physical + const ambiguityIndex = chronological.findIndex( + (message) => message.info.id === receipt.assistant_message_id, + ) + const triggerIndex = chronological.findIndex((message) => message.info.id === receipt.user_message_id) + if (ambiguityIndex < 0 || triggerIndex < 0 || triggerIndex >= ambiguityIndex) + return yield* new Conflict({ + code: "unsafe_history_boundary", + reason: "ambiguous provider turn is not contiguous", + }) + const safeMessages = cutoffAuthority.projection.messages + if (safeMessages.some((message) => message.info.role === "assistant" && !message.info.finish)) + return yield* new Conflict({ + code: "unsafe_history_boundary", + reason: "safe prefix contains unfinished assistant output", + }) + + const sourceBaseline = yield* tx + .select() + .from(SessionWorldStateBaselineTable) + .where( + and( + eq(SessionWorldStateBaselineTable.session_id, input.sessionID), + eq(SessionWorldStateBaselineTable.prompt_epoch, source.epoch), + ), + ) + .all() + if (sourceBaseline.length === 0 || !source.world_state_baseline_hash) + return yield* new Conflict({ + code: "source_world_state_baseline_missing", + reason: "source World State baseline requires Maintenance recovery", + }) + const sections = yield* Effect.try({ + try: () => + sourceBaseline.map((row) => ({ + sectionID: row.section_id, + snapshot: Schema.decodeUnknownSync(WorldStateSlot)(row.snapshot), + fragment: row.fragment, + fragmentHash: row.fragment_hash, + })) satisfies SessionWorldStateBaselineSection[], + catch: () => + new Conflict({ + code: "source_world_state_baseline_invalid", + reason: "source World State baseline is not decodable", + }), + }) + const baselineHash = sessionWorldStateBaselineHash({ + sections, + rendered: renderSessionWorldStateBaseline(sections), + }) + if ( + baselineHash !== source.world_state_baseline_hash || + baselineHash !== input.expected.worldStateBaselineHash + ) + return yield* new Conflict({ + code: "source_world_state_baseline_invalid", + reason: "source World State baseline hash changed or no longer matches the admitted command", + }) + + const safeHistoryHash = HistoryAuthority.hash(safeMessages) + const successorWindowID = HistoryAuthority.windowID() + const highWater = MessageID.make(chronological.at(-1)?.info.id ?? receipt.assistant_message_id) + const activity = yield* tx + .select({ activityID: SessionActivityProgressTable.activity_id }) + .from(SessionActivityProgressTable) + .innerJoin( + SessionLegacyActivityTable, + eq(SessionLegacyActivityTable.activity_id, SessionActivityProgressTable.activity_id), + ) + .where( + and( + eq(SessionActivityProgressTable.provider_receipt_id, input.receiptID), + eq(SessionActivityProgressTable.state, "recovery_required"), + eq(SessionLegacyActivityTable.state, "recovery_required"), + eq(SessionLegacyActivityTable.session_id, input.sessionID), + ), + ) + .get() + const retired = yield* tx + .update(SessionPromptEpochTable) + .set({ state: "retired", retired_at: proposed.createdAt }) + .where( + and( + eq(SessionPromptEpochTable.session_id, input.sessionID), + eq(SessionPromptEpochTable.epoch, source.epoch), + eq(SessionPromptEpochTable.state, "active"), + eq(SessionPromptEpochTable.authority_state, "recovery_required"), + ), + ) + .returning({ epoch: SessionPromptEpochTable.epoch }) + .get() + if (!retired) return yield* Effect.die(new Error("provider recovery epoch retirement CAS failed")) + const resolution = yield* tx + .insert(SessionToolRequestResolutionTable) + .values({ + resolution_id: proposed.resolutionID, + receipt_id: input.receiptID, + session_id: input.sessionID, + legacy_activity_id: activity?.activityID ?? null, + assistant_message_id: MessageID.make(receipt.assistant_message_id), + source_prompt_epoch: source.epoch, + source_window_id: source.window_id!, + source_effective_history_hash: receipt.effective_history_hash!, + source_request_hash: receiptRequestHash!, + source_mutation_epoch: session.mutationEpoch, + expected_provider_state: "indeterminate_after_crash", + decision: "abandoned", + actor_type: "user", + actor_id: input.actorID, + reason: input.reason ?? DEFAULT_REASON, + risk_acknowledged: false, + safe_end_message_id: safeMessages.at(-1)?.info.id ?? null, + safe_history_hash: safeHistoryHash, + safe_message_ids: safeMessages.map((message) => message.info.id), + ambiguity_message_id: MessageID.make(receipt.assistant_message_id), + physical_message_high_water: highWater, + successor_prompt_epoch: proposed.successorPromptEpoch, + successor_window_id: successorWindowID, + successor_history_hash: safeHistoryHash, + successor_mutation_epoch: proposed.successorMutationEpoch, + created_at: proposed.createdAt, + }) + .returning() + .get() + if (!resolution) return yield* Effect.die(new Error("provider recovery resolution insert failed")) + yield* tx + .insert(SessionPromptEpochTable) + .values({ + session_id: input.sessionID, + epoch: proposed.successorPromptEpoch, + state: "active", + checkpoint_user_id: null, + checkpoint_assistant_id: null, + retained_tail_start_id: null, + source_end_message_id: highWater, + checkpoint_hash: safeHistoryHash, + projection_version: HistoryAuthority.PROJECTION_VERSION, + canonicalization_version: HistoryAuthority.CANONICALIZATION_VERSION, + base_message_count: safeMessages.length, + effective_history_hash: safeHistoryHash, + first_window_id: source.first_window_id ?? source.window_id, + previous_window_id: source.window_id, + window_id: successorWindowID, + world_state_baseline_hash: baselineHash, + authority_state: "ready", + recovery_reason: null, + recovery_resolution_id: proposed.resolutionID, + reason: "recovery", + created_at: proposed.createdAt, + retired_at: null, + }) + .run() + if (safeMessages.length > 0) + yield* tx + .insert(SessionPromptEpochMessageTable) + .values( + safeMessages.map((message, ordinal) => ({ + session_id: input.sessionID, + prompt_epoch: proposed.successorPromptEpoch, + ordinal, + message_id: message.info.id, + })), + ) + .run() + yield* tx + .insert(SessionWorldStateBaselineTable) + .values( + sections.map((section) => ({ + session_id: input.sessionID, + prompt_epoch: proposed.successorPromptEpoch, + section_id: section.sectionID, + snapshot: section.snapshot, + fragment: section.fragment, + fragment_hash: section.fragmentHash, + provenance: "recovery_copied" as const, + created_at: proposed.createdAt, + })), + ) + .run() + yield* tx + .insert(SessionPromptEpochRecoveryTable) + .values({ + session_id: input.sessionID, + prompt_epoch: proposed.successorPromptEpoch, + resolution_id: proposed.resolutionID, + source_prompt_epoch: source.epoch, + source_mutation_epoch: session.mutationEpoch, + successor_mutation_epoch: proposed.successorMutationEpoch, + ambiguity_message_id: MessageID.make(receipt.assistant_message_id), + physical_message_high_water: highWater, + created_at: proposed.createdAt, + }) + .run() + const mutated = yield* tx + .update(SessionTable) + .set({ mutation_epoch: proposed.successorMutationEpoch, time_updated: proposed.createdAt }) + .where(and(eq(SessionTable.id, input.sessionID), eq(SessionTable.mutation_epoch, session.mutationEpoch))) + .returning({ epoch: SessionTable.mutation_epoch }) + .get() + if (mutated?.epoch !== proposed.successorMutationEpoch) + return yield* Effect.die(new Error("provider recovery mutation epoch CAS failed")) + yield* tx + .update(SessionIntentTable) + .set({ + state: "superseded", + owner_token: null, + lease_expires_at: null, + time_updated: proposed.createdAt, + version: sql`${SessionIntentTable.version} + 1`, + }) + .where( + and( + eq(SessionIntentTable.session_id, input.sessionID), + inArray(SessionIntentTable.state, ["preparing", "admitting", "failed"]), + sql`${SessionIntentTable.mutation_epoch} < ${proposed.successorMutationEpoch}`, + ), + ) + .run() + yield* tx + .update(SessionSteerTable) + .set({ superseded_at: proposed.createdAt }) + .where( + and( + eq(SessionSteerTable.session_id, input.sessionID), + isNull(SessionSteerTable.consumed_seq), + isNull(SessionSteerTable.superseded_at), + sql`${SessionSteerTable.mutation_epoch} < ${proposed.successorMutationEpoch}`, + ), + ) + .run() + const history = yield* tx + .update(SessionHistoryStateTable) + .set({ state: "ready", reason: null, time_updated: proposed.createdAt }) + .where( + and( + eq(SessionHistoryStateTable.session_id, input.sessionID), + eq(SessionHistoryStateTable.state, "recovery_required"), + ), + ) + .returning({ sessionID: SessionHistoryStateTable.session_id }) + .get() + if (!history) return yield* Effect.die(new Error("provider recovery history state CAS failed")) + const completed = yield* tx + .update(SessionToolRequestResolutionCommandTable) + .set({ result_resolution_id: proposed.resolutionID }) + .where( + and( + eq(SessionToolRequestResolutionCommandTable.command_id, input.commandID), + eq(SessionToolRequestResolutionCommandTable.request_hash, hash), + isNull(SessionToolRequestResolutionCommandTable.result_resolution_id), + ), + ) + .returning({ commandID: SessionToolRequestResolutionCommandTable.command_id }) + .get() + if (!completed) return yield* Effect.die(new Error("provider recovery command result CAS failed")) + }), + { behavior: "immediate" }, + ), + ) + if (Exit.isFailure(transaction)) { + const failure = Cause.squash(transaction.cause) + if (failure instanceof Conflict || failure instanceof NotFound) return yield* failure + return yield* Effect.die(failure) + } + const committed = transaction.value + const result = + committed ?? + (yield* db + .select() + .from(SessionToolRequestResolutionTable) + .where(eq(SessionToolRequestResolutionTable.resolution_id, proposed.resolutionID)) + .get() + .pipe(Effect.orDie)) + if (!result) return yield* Effect.die(new Error(`resolution result disappeared: ${proposed.resolutionID}`)) + yield* events + .publish(Event.Completed, { + sessionID: input.sessionID, + resolutionID: result.resolution_id, + commandID: input.commandID, + receiptID: result.receipt_id, + decision: "abandoned", + sourcePromptEpoch: result.source_prompt_epoch, + successorPromptEpoch: result.successor_prompt_epoch, + sourceMutationEpoch: result.source_mutation_epoch, + successorMutationEpoch: result.successor_mutation_epoch, + createdAt: result.created_at, + }) + .pipe(Effect.catchCause(() => Effect.void)) + return fromResolution(input.commandID, result) + }) + + return Service.of({ describe, resolve }) +}) + +export const layer = Layer.effect(Service, service) +export const defaultLayer = layer.pipe(Layer.provide(Database.defaultLayer), Layer.provide(EventV2Bridge.defaultLayer)) + +export * as SessionLegacyProviderResolution from "./legacy-provider-resolution" diff --git a/packages/deepagent-code/src/session/message-v2.ts b/packages/deepagent-code/src/session/message-v2.ts index dab9ed1e9..6989c851d 100644 --- a/packages/deepagent-code/src/session/message-v2.ts +++ b/packages/deepagent-code/src/session/message-v2.ts @@ -29,9 +29,12 @@ import { Database } from "@deepagent-code/core/database/database" import { NotFoundError } from "@/storage/storage" import { and } from "drizzle-orm" import { desc } from "drizzle-orm" +import { asc } from "drizzle-orm" import { eq } from "drizzle-orm" import { inArray } from "drizzle-orm" +import { isNull } from "drizzle-orm" import { lt } from "drizzle-orm" +import { gt } from "drizzle-orm" import { or } from "drizzle-orm" import { sql } from "drizzle-orm" import { @@ -39,8 +42,11 @@ import { PartTable, SessionForkAdmissionTable, SessionForkIntentTable, + SessionIntentTable, SessionPromptEpochMessageTable, + SessionPromptEpochRecoveryTable, SessionTable, + SessionToolRequestResolutionTable, SessionWorldStateBaselineTable, } from "@deepagent-code/core/session/sql" import { SessionPromptEpochTable } from "./prompt-epoch.sql" @@ -54,6 +60,7 @@ import { Data, Effect, Exit, Option, Schema } from "effect" import * as EffectLogger from "@deepagent-code/core/effect/logger" import { SessionHistoryStateTable } from "@deepagent-code/core/session/sql" import { CompactionArtifactTable, CompactionRunTable } from "./compaction-sql" +import { SessionToolRequestReceiptTable } from "./tool-request-receipt.sql" import { HistoryAuthority } from "./history-authority" import { collectSessionWorldStateBaseline, @@ -110,6 +117,7 @@ export type PromptHistoryProjection = { readonly windowID: string } readonly worldStateBaselineHash?: string + readonly recoveryResolutionID?: string } export type PromptWorldStateProjection = { @@ -275,12 +283,59 @@ export function stripActivityProgress(message: T): T { return rest as T } -export function stripActivityProgressPart(messagePart: T): T { - if ( - messagePart.type !== "text" || - !messagePart.metadata || - !("deepagent_activity_progress" in messagePart.metadata) +export const ClientDiffLimits = { + files: 200, +} as const + +export const ClientMessageLimits = { + page: 100, +} as const + +const decodeClientMessageData = Schema.decodeUnknownOption(Schema.UnknownFromJsonString) + +const clientMessageData = sql`CASE + WHEN json_valid(${MessageTable.data}) AND json_type(${MessageTable.data}, '$.summary.diffs') = 'array' + THEN json_set( + ${MessageTable.data}, + '$.summary.diffs', + json(COALESCE(( + SELECT json_group_array(json_remove(value, '$.patch')) + FROM ( + SELECT value + FROM json_each(json_extract(${MessageTable.data}, '$.summary.diffs')) + LIMIT ${ClientDiffLimits.files} + ) + ), '[]')) ) + ELSE ${MessageTable.data} +END` + +const promptMessageData = sql`CASE + WHEN json_valid(${MessageTable.data}) + AND json_type(${MessageTable.data}, '$.summary.diffs') = 'array' + THEN json_remove(${MessageTable.data}, '$.summary.diffs') + ELSE ${MessageTable.data} +END` + +export function clientProjection(message: T): T { + if (message.role !== "user" || !message.summary) return message + const diffs = message.summary.diffs.slice(0, ClientDiffLimits.files).map((item) => ({ + ...(item.file === undefined ? {} : { file: item.file }), + additions: item.additions, + deletions: item.deletions, + ...(item.status === undefined ? {} : { status: item.status }), + })) + return { + ...message, + summary: { + ...message.summary, + diffs, + }, + } as T +} + +export function stripActivityProgressPart(messagePart: T): T { + if (messagePart.type !== "text" || !messagePart.metadata || !("deepagent_activity_progress" in messagePart.metadata)) return messagePart const { deepagent_activity_progress: _, ...metadata } = messagePart.metadata return { @@ -299,6 +354,8 @@ const part = (row: typeof PartTable.$inferSelect) => const older = (row: Cursor) => or(lt(MessageTable.time_created, row.time), and(eq(MessageTable.time_created, row.time), lt(MessageTable.id, row.id))) +const newer = (row: Cursor) => + or(gt(MessageTable.time_created, row.time), and(eq(MessageTable.time_created, row.time), gt(MessageTable.id, row.id))) function hydrate(db: Database.Interface["db"], rows: (typeof MessageTable.$inferSelect)[]) { const ids = rows.map((row) => row.id) @@ -374,9 +431,7 @@ function hydrate(db: Database.Interface["db"], rows: (typeof MessageTable.$infer activityID: row.activityID, revision: row.revision, state, - ...(row.revision === row.latestRevision && row.terminalReason - ? { terminalReason: row.terminalReason } - : {}), + ...(row.revision === row.latestRevision && row.terminalReason ? { terminalReason: row.terminalReason } : {}), }) } } @@ -842,6 +897,97 @@ export const page = Effect.fn("MessageV2.page")(function* (input: { } }) +export const forwardPage = Effect.fn("MessageV2.forwardPage")(function* (input: { + sessionID: SessionID + limit: number + after: string +}) { + const { db } = yield* Database.Service + const after = cursor.decode(input.after) + const selected = yield* db + .select({ + id: MessageTable.id, + session_id: MessageTable.session_id, + time_created: MessageTable.time_created, + time_updated: MessageTable.time_updated, + data: clientMessageData, + }) + .from(MessageTable) + .where(and(eq(MessageTable.session_id, input.sessionID), newer(after))) + .orderBy(asc(MessageTable.time_created), asc(MessageTable.id)) + .limit(input.limit + 1) + .all() + .pipe(Effect.orDie) + const rows = selected.flatMap((row) => { + const data = Option.getOrUndefined(decodeClientMessageData(row.data)) + if (!data || typeof data !== "object" || Array.isArray(data)) return [] + return [{ ...row, data: data as typeof MessageTable.$inferSelect.data }] + }) + if (rows.length !== selected.length) return yield* Effect.die("stored message data is not valid JSON object") + const more = rows.length > input.limit + const slice = more ? rows.slice(0, input.limit) : rows + const items = yield* hydrate(db, slice) + const tail = slice.at(-1) + return { + items, + more, + cursor: more && tail ? cursor.encode({ id: tail.id, time: tail.time_created }) : undefined, + } +}) + +export const clientPage = Effect.fn("MessageV2.clientPage")(function* (input: { + sessionID: SessionID + limit: number + before?: string +}) { + const { db } = yield* Database.Service + const before = input.before ? cursor.decode(input.before) : undefined + const where = before + ? and(eq(MessageTable.session_id, input.sessionID), older(before)) + : eq(MessageTable.session_id, input.sessionID) + const selected = yield* db + .select({ + id: MessageTable.id, + session_id: MessageTable.session_id, + time_created: MessageTable.time_created, + time_updated: MessageTable.time_updated, + data: clientMessageData, + }) + .from(MessageTable) + .where(where) + .orderBy(desc(MessageTable.time_created), desc(MessageTable.id)) + .limit(input.limit + 1) + .all() + .pipe(Effect.orDie) + if (selected.length === 0) { + const row = yield* db + .select({ id: SessionTable.id }) + .from(SessionTable) + .where(eq(SessionTable.id, input.sessionID)) + .get() + .pipe(Effect.orDie) + if (!row) return yield* new NotFoundError({ message: `Session not found: ${input.sessionID}` }) + return { items: [] as WithParts[], more: false } + } + + const rows = selected.flatMap((row) => { + const data = Option.getOrUndefined(decodeClientMessageData(row.data)) + if (!data || typeof data !== "object" || Array.isArray(data)) return [] + return [{ ...row, data: data as typeof MessageTable.$inferSelect.data }] + }) + if (rows.length !== selected.length) return yield* Effect.die("stored message data is not valid JSON object") + const more = rows.length > input.limit + const slice = more ? rows.slice(0, input.limit) : rows + const items = yield* hydrate(db, slice) + items.reverse() + const tail = slice.at(-1) + return { + items, + more, + cursor: more && tail ? cursor.encode({ id: tail.id, time: tail.time_created }) : undefined, + } +}) + export function stream(sessionID: SessionID) { const size = 50 return Effect.gen(function* () { @@ -865,6 +1011,50 @@ export function stream(sessionID: SessionID) { }) } +export function promptStream(sessionID: SessionID) { + const size = 50 + return Effect.gen(function* () { + const { db } = yield* Database.Service + const result = [] as WithParts[] + let before: string | undefined + while (true) { + const decoded = before ? cursor.decode(before) : undefined + const where = decoded + ? and(eq(MessageTable.session_id, sessionID), older(decoded)) + : eq(MessageTable.session_id, sessionID) + const selected = yield* db + .select({ + id: MessageTable.id, + session_id: MessageTable.session_id, + time_created: MessageTable.time_created, + time_updated: MessageTable.time_updated, + data: promptMessageData, + }) + .from(MessageTable) + .where(where) + .orderBy(desc(MessageTable.time_created), desc(MessageTable.id)) + .limit(size + 1) + .all() + .pipe(Effect.orDie) + if (selected.length === 0) break + const rows = selected.flatMap((row) => { + const data = Option.getOrUndefined(decodeClientMessageData(row.data)) + if (!data || typeof data !== "object" || Array.isArray(data)) return [] + return [{ ...row, data: data as typeof MessageTable.$inferSelect.data }] + }) + if (rows.length !== selected.length) return yield* Effect.die("stored message data is not valid JSON object") + const more = rows.length > size + const slice = more ? rows.slice(0, size) : rows + const items = yield* hydrate(db, slice) + result.push(...items) + const tail = slice.at(-1) + if (!more || !tail) break + before = cursor.encode({ id: tail.id, time: tail.time_created }) + } + return result + }) +} + export function parts(messageID: MessageID) { return Effect.gen(function* () { const { db } = yield* Database.Service @@ -896,6 +1086,31 @@ export const get = Effect.fn("MessageV2.get")(function* (input: { sessionID: Ses return result[0]! }) +export const clientGet = Effect.fn("MessageV2.clientGet")(function* (input: { + sessionID: SessionID + messageID: MessageID +}) { + const { db } = yield* Database.Service + const selected = yield* db + .select({ + id: MessageTable.id, + session_id: MessageTable.session_id, + time_created: MessageTable.time_created, + time_updated: MessageTable.time_updated, + data: clientMessageData, + }) + .from(MessageTable) + .where(and(eq(MessageTable.id, input.messageID), eq(MessageTable.session_id, input.sessionID))) + .get() + .pipe(Effect.orDie) + if (!selected) return yield* new NotFoundError({ message: `Message not found: ${input.messageID}` }) + const data = Option.getOrUndefined(decodeClientMessageData(selected.data)) + if (!data || typeof data !== "object" || Array.isArray(data)) + return yield* Effect.die("stored message data is not valid JSON object") + const result = yield* hydrate(db, [{ ...selected, data: data as typeof MessageTable.$inferSelect.data }]) + return result[0]! +}) + export function filterCompacted(msgs: Iterable) { const result = [] as WithParts[] const completed = new Set() @@ -950,7 +1165,7 @@ export function filterCompacted(msgs: Iterable) { } export const filterCompactedEffect = Effect.fnUntraced(function* (sessionID: SessionID) { - return filterCompacted(yield* stream(sessionID)) + return filterCompacted(yield* promptStream(sessionID)) }) const projectPromptHistory = Effect.fn("MessageV2.projectPromptHistory")(function* ( @@ -991,7 +1206,8 @@ const projectPromptHistory = Effect.fn("MessageV2.projectPromptHistory")(functio epoch.base_message_count === null || !epoch.effective_history_hash || !epoch.first_window_id || - !epoch.window_id + !epoch.window_id || + !snapshot.recoveryBindingValid ) { return yield* failHistoryAuthority({ sessionID, @@ -1000,7 +1216,12 @@ const projectPromptHistory = Effect.fn("MessageV2.projectPromptHistory")(functio }) } - const selected = selectEpochHistory(chronological, epoch, snapshot.replacementMessageIDs) + const selected = selectEpochHistory( + chronological, + epoch, + snapshot.replacementMessageIDs, + snapshot.recoveryAdmittedMessageIDs, + ) if (!selected.ok) return yield* failHistoryAuthority({ sessionID, epoch: epoch.epoch, reason: selected.reason }) if (epoch.base_message_count < 0 || epoch.base_message_count > selected.messages.length) { return yield* failHistoryAuthority({ @@ -1032,6 +1253,7 @@ const projectPromptHistory = Effect.fn("MessageV2.projectPromptHistory")(functio windowID: epoch.window_id, }, ...(epoch.world_state_baseline_hash ? { worldStateBaselineHash: epoch.world_state_baseline_hash } : {}), + ...(epoch.recovery_resolution_id ? { recoveryResolutionID: epoch.recovery_resolution_id } : {}), } satisfies PromptHistoryProjection }) @@ -1041,6 +1263,136 @@ export const promptHistoryProjectionEffect = Effect.fn("MessageV2.promptHistoryP return yield* projectPromptHistory(sessionID) }) +type HistorySnapshot = { + physical: WithParts[] + chronological: WithParts[] + epoch: EpochRow | undefined + replacementMessageIDs: MessageID[] + recoveryBindingValid: boolean + recoveryAdmittedMessageIDs: Set +} + +function promptHistoryCutoffProjectionFromSnapshot( + sessionID: SessionID, + cutoffMessageID: MessageID, + snapshot: HistorySnapshot, +) { + return Effect.gen(function* () { + const epoch = snapshot.epoch + if ( + !epoch || + (epoch.authority_state !== "ready" && epoch.authority_state !== "recovery_required") || + epoch.projection_version !== HistoryAuthority.PROJECTION_VERSION || + epoch.canonicalization_version !== HistoryAuthority.CANONICALIZATION_VERSION || + epoch.base_message_count === null || + !epoch.effective_history_hash || + !epoch.first_window_id || + !epoch.window_id || + !snapshot.recoveryBindingValid + ) + return + const selected = selectEpochHistory( + snapshot.chronological, + epoch, + snapshot.replacementMessageIDs, + snapshot.recoveryAdmittedMessageIDs, + ) + if (!selected.ok || epoch.base_message_count < 0 || epoch.base_message_count > selected.messages.length) return + if (HistoryAuthority.hash(selected.messages.slice(0, epoch.base_message_count)) !== epoch.effective_history_hash) + return + const cutoffIndex = selected.messages.findIndex((message) => message.info.id === cutoffMessageID) + if (cutoffIndex < 0) return + const messages = selected.messages.slice(0, cutoffIndex) + return { + sessionID, + epoch: epoch.epoch, + messages, + orderedMessageIDs: messages.map((message) => message.info.id), + effectiveHistoryHash: HistoryAuthority.hash(messages), + projectionVersion: epoch.projection_version, + canonicalizationVersion: epoch.canonicalization_version, + baseMessageCount: messages.length, + window: { + firstWindowID: epoch.first_window_id, + ...(epoch.previous_window_id ? { previousWindowID: epoch.previous_window_id } : {}), + windowID: epoch.window_id, + }, + ...(epoch.world_state_baseline_hash ? { worldStateBaselineHash: epoch.world_state_baseline_hash } : {}), + ...(epoch.recovery_resolution_id ? { recoveryResolutionID: epoch.recovery_resolution_id } : {}), + } satisfies PromptHistoryProjection + }) +} + +export function promptHistoryCutoffAuthorityInTransaction( + tx: Pick, + input: { sessionID: SessionID; cutoffMessageID: MessageID }, +) { + return Effect.gen(function* () { + const snapshotExit = yield* Effect.exit(readHistorySnapshotInTransaction(tx, input.sessionID)) + if (Exit.isFailure(snapshotExit)) return + const projection = yield* promptHistoryCutoffProjectionFromSnapshot( + input.sessionID, + input.cutoffMessageID, + snapshotExit.value, + ) + if (!projection) return + if (snapshotExit.value.epoch?.authority_state !== "recovery_required") + return { projection, physical: snapshotExit.value.physical } + const unresolved = yield* tx + .select({ + userMessageID: SessionToolRequestReceiptTable.user_message_id, + assistantMessageID: SessionToolRequestReceiptTable.assistant_message_id, + }) + .from(SessionToolRequestReceiptTable) + .leftJoin( + SessionToolRequestResolutionTable, + eq(SessionToolRequestResolutionTable.receipt_id, SessionToolRequestReceiptTable.receipt_id), + ) + .where( + and( + eq(SessionToolRequestReceiptTable.session_id, input.sessionID), + eq(SessionToolRequestReceiptTable.provider_state, "indeterminate_after_crash"), + isNull(SessionToolRequestResolutionTable.resolution_id), + ), + ) + .all() + const cutoffIndex = snapshotExit.value.chronological.findIndex( + (message) => message.info.id === input.cutoffMessageID, + ) + const unsafe = unresolved.some((receipt) => { + const userIndex = snapshotExit.value.chronological.findIndex( + (message) => message.info.id === receipt.userMessageID, + ) + return userIndex >= 0 && cutoffIndex > userIndex + }) + if (unsafe) return + return { projection, physical: snapshotExit.value.physical } + }) +} + +export function promptHistoryCutoffProjectionInTransaction( + tx: Pick, + input: { sessionID: SessionID; cutoffMessageID: MessageID }, +) { + return Effect.gen(function* () { + return (yield* promptHistoryCutoffAuthorityInTransaction(tx, input))?.projection + }) +} + +export const promptHistoryCutoffProjectionEffect = Effect.fn("MessageV2.promptHistoryCutoffProjection")( + function* (input: { sessionID: SessionID; cutoffMessageID: MessageID }) { + const { db } = yield* Database.Service + const projection = yield* db + .transaction((tx) => promptHistoryCutoffProjectionInTransaction(tx as unknown as Database.Interface["db"], input)) + .pipe(Effect.orDie) + if (projection) return projection + return yield* new HistoryAuthorityError({ + sessionID: input.sessionID, + reason: "fork cutoff is outside the recoverable history prefix", + }) + }, +) + export const promptHistoryBeforeCompactionEffect = Effect.fn("MessageV2.promptHistoryBeforeCompaction")( function* (input: { sessionID: SessionID; markerMessageID: MessageID }) { return yield* projectPromptHistory(input.sessionID, input.markerMessageID) @@ -1156,7 +1508,7 @@ export const promptControlHistoryEffect = Effect.fn("MessageV2.promptControlHist const known = new Set(history.map((message) => message.info.id)) return [ ...history, - ...(yield* stream(sessionID)) + ...(yield* promptStream(sessionID)) .reverse() .filter((message) => markers.has(message.info.id) && !known.has(message.info.id)), ] @@ -1168,7 +1520,7 @@ type EpochSelection = | { readonly ok: false; readonly reason: string } function readHistorySnapshotInTransaction( - tx: Database.Interface["db"], + tx: Pick, sessionID: SessionID, ignoredCompactionMarkerID?: MessageID, ) { @@ -1182,7 +1534,9 @@ function readHistorySnapshotInTransaction( .filter((artifact) => artifact.state !== "committed") .map((artifact) => artifact.message_id), ) - const physical = (yield* stream(sessionID).pipe(Effect.provideService(Database.Service, { db: tx }))).reverse() + const physical = (yield* promptStream(sessionID).pipe( + Effect.provideService(Database.Service, { db: tx }), + )).reverse() const visible = physical.filter( (message) => message.info.id === ignoredCompactionMarkerID || !hidden.has(message.info.id), ) @@ -1205,7 +1559,57 @@ function readHistorySnapshotInTransaction( .orderBy(SessionPromptEpochMessageTable.ordinal) .all()).map((row) => row.message_id) : [] - return { chronological, epoch, replacementMessageIDs } + const recoveryBinding = epoch?.recovery_resolution_id + ? yield* tx + .select({ + resolutionID: SessionPromptEpochRecoveryTable.resolution_id, + sourcePromptEpoch: SessionPromptEpochRecoveryTable.source_prompt_epoch, + successorPromptEpoch: SessionToolRequestResolutionTable.successor_prompt_epoch, + successorWindowID: SessionToolRequestResolutionTable.successor_window_id, + successorHistoryHash: SessionToolRequestResolutionTable.successor_history_hash, + successorMutationEpoch: SessionPromptEpochRecoveryTable.successor_mutation_epoch, + }) + .from(SessionPromptEpochRecoveryTable) + .innerJoin( + SessionToolRequestResolutionTable, + eq(SessionToolRequestResolutionTable.resolution_id, SessionPromptEpochRecoveryTable.resolution_id), + ) + .where( + and( + eq(SessionPromptEpochRecoveryTable.session_id, sessionID), + eq(SessionPromptEpochRecoveryTable.prompt_epoch, epoch.epoch), + eq(SessionPromptEpochRecoveryTable.resolution_id, epoch.recovery_resolution_id), + ), + ) + .get() + : undefined + const recoveryBindingValid = + !epoch?.recovery_resolution_id || + !!( + recoveryBinding && + recoveryBinding.sourcePromptEpoch < epoch.epoch && + recoveryBinding.successorPromptEpoch === epoch.epoch && + recoveryBinding.successorWindowID === epoch.window_id && + recoveryBinding.successorHistoryHash === epoch.effective_history_hash && + recoveryBinding.successorMutationEpoch >= 0 + ) + const recoveryAdmittedMessageIDs = + recoveryBinding && epoch?.recovery_resolution_id + ? new Set( + (yield* tx + .select({ messageID: SessionIntentTable.admitted_message_id }) + .from(SessionIntentTable) + .where( + and( + eq(SessionIntentTable.session_id, sessionID), + eq(SessionIntentTable.state, "admitted"), + eq(SessionIntentTable.mutation_epoch, recoveryBinding.successorMutationEpoch), + ), + ) + .all()).flatMap((row) => (row.messageID ? [MessageID.make(row.messageID)] : [])), + ) + : new Set() + return { physical, chronological, epoch, replacementMessageIDs, recoveryBindingValid, recoveryAdmittedMessageIDs } }) } @@ -1225,6 +1629,7 @@ function selectEpochHistory( chronological: WithParts[], epoch: EpochRow, replacementMessageIDs: readonly MessageID[] = [], + recoveryAdmittedMessageIDs = new Set(), ): EpochSelection { if (epoch.base_message_count !== null && epoch.base_message_count !== replacementMessageIDs.length) { return { ok: false, reason: `epoch ${epoch.epoch} replacement membership is incomplete` } @@ -1238,7 +1643,7 @@ function selectEpochHistory( if (replacement.length !== replacementMessageIDs.length) { return { ok: false, reason: `epoch ${epoch.epoch} replacement message is missing` } } - if (epoch.epoch > 0) { + if (epoch.epoch > 0 && !epoch.recovery_resolution_id) { const user = replacement[0] const assistant = replacement[1] if ( @@ -1263,16 +1668,44 @@ function selectEpochHistory( return { ok: false, reason: `epoch ${epoch.epoch} append boundary is missing` } } const replacementIDs = new Set(replacementMessageIDs) + const future = boundaryIndex < 0 ? [] : chronological.slice(boundaryIndex + 1) + const appended = epoch.recovery_resolution_id + ? future.filter((message) => { + if (message.info.role === "user") { + if (!recoveryAdmittedMessageIDs.has(message.info.id)) return false + replacementIDs.add(message.info.id) + return true + } + if (!replacementIDs.has(message.info.parentID)) return false + replacementIDs.add(message.info.id) + return true + }) + : future.filter((message) => !replacementIDs.has(message.info.id)) return { ok: true, - messages: [ - ...replacement, - ...(boundaryIndex < 0 - ? [] - : chronological.slice(boundaryIndex + 1).filter((message) => !replacementIDs.has(message.info.id))), - ], + messages: [...replacement, ...appended], } } + if (epoch.recovery_resolution_id) { + const boundaryIndex = epoch.source_end_message_id + ? chronological.findIndex((message) => message.info.id === epoch.source_end_message_id) + : -1 + if (epoch.source_end_message_id && boundaryIndex < 0) { + return { ok: false, reason: `epoch ${epoch.epoch} append boundary is missing` } + } + const admitted = new Set() + const appended = (boundaryIndex < 0 ? [] : chronological.slice(boundaryIndex + 1)).filter((message) => { + if (message.info.role === "user") { + if (!recoveryAdmittedMessageIDs.has(message.info.id)) return false + admitted.add(message.info.id) + return true + } + if (!admitted.has(message.info.parentID)) return false + admitted.add(message.info.id) + return true + }) + return { ok: true, messages: appended } + } if (epoch.epoch === 0) return { ok: true, messages: chronological } if (!epoch.checkpoint_user_id || !epoch.checkpoint_assistant_id) { return { ok: false, reason: `epoch ${epoch.epoch} is missing checkpoint authority` } @@ -1318,7 +1751,7 @@ function selectEpochHistory( // missing/incomplete authority simply fails the caller's CAS so no replacement window is committed // against a different physical history. export function promptHistoryProjectionInTransaction( - tx: Database.Interface["db"], + tx: Pick, sessionID: SessionID, ignoredCompactionMarkerID?: MessageID, ) { @@ -1335,10 +1768,16 @@ export function promptHistoryProjectionInTransaction( epoch.base_message_count === null || !epoch.effective_history_hash || !epoch.first_window_id || - !epoch.window_id + !epoch.window_id || + !snapshot.recoveryBindingValid ) return - const selected = selectEpochHistory(snapshot.chronological, epoch, snapshot.replacementMessageIDs) + const selected = selectEpochHistory( + snapshot.chronological, + epoch, + snapshot.replacementMessageIDs, + snapshot.recoveryAdmittedMessageIDs, + ) if (!selected.ok || epoch.base_message_count < 0 || epoch.base_message_count > selected.messages.length) return if (HistoryAuthority.hash(selected.messages.slice(0, epoch.base_message_count)) !== epoch.effective_history_hash) return @@ -1357,6 +1796,7 @@ export function promptHistoryProjectionInTransaction( windowID: epoch.window_id, }, ...(epoch.world_state_baseline_hash ? { worldStateBaselineHash: epoch.world_state_baseline_hash } : {}), + ...(epoch.recovery_resolution_id ? { recoveryResolutionID: epoch.recovery_resolution_id } : {}), } satisfies PromptHistoryProjection }) } @@ -1491,7 +1931,7 @@ const migrateHistoryAuthority = Effect.fn("MessageV2.migrateHistoryAuthority")(f const hidden = new Set( artifacts.filter((artifact) => artifact.state !== "committed").map((artifact) => artifact.message_id), ) - const physical = (yield* stream(input.sessionID).pipe( + const physical = (yield* promptStream(input.sessionID).pipe( Effect.provideService(Database.Service, { db: tx as unknown as Database.Interface["db"], }), @@ -1692,6 +2132,7 @@ function legacyEpochCandidate( world_state_baseline_hash: null, authority_state: "legacy_pending", recovery_reason: null, + recovery_resolution_id: null, reason: "bootstrap", created_at: Date.now(), retired_at: null, @@ -1738,6 +2179,7 @@ function legacyEpochCandidate( world_state_baseline_hash: null, authority_state: "legacy_pending", recovery_reason: null, + recovery_resolution_id: null, reason: "compaction", created_at: Date.now(), retired_at: null, diff --git a/packages/deepagent-code/src/session/prompt-epoch.sql.ts b/packages/deepagent-code/src/session/prompt-epoch.sql.ts index b4452ae6e..03ec565a6 100644 --- a/packages/deepagent-code/src/session/prompt-epoch.sql.ts +++ b/packages/deepagent-code/src/session/prompt-epoch.sql.ts @@ -9,6 +9,7 @@ export type PromptEpochState = "active" | "retired" export type PromptEpochReason = | "bootstrap" | "compaction" + | "recovery" | "model" | "agent" | "directory" @@ -36,6 +37,7 @@ export const SessionPromptEpochTable = sqliteTable("session_prompt_epoch", { world_state_baseline_hash: text(), authority_state: text().$type<"legacy_pending" | "ready" | "recovery_required">(), recovery_reason: text(), + recovery_resolution_id: text(), reason: text().$type().notNull(), created_at: integer().notNull(), retired_at: integer(), diff --git a/packages/deepagent-code/src/session/prompt-epoch.ts b/packages/deepagent-code/src/session/prompt-epoch.ts index c89a0ffff..3a8daf62d 100644 --- a/packages/deepagent-code/src/session/prompt-epoch.ts +++ b/packages/deepagent-code/src/session/prompt-epoch.ts @@ -38,6 +38,7 @@ export interface PromptEpochRow { world_state_baseline_hash: string | null authority_state: "legacy_pending" | "ready" | "recovery_required" | null recovery_reason: string | null + recovery_resolution_id: string | null reason: PromptEpochReason created_at: number retired_at: number | null @@ -140,6 +141,7 @@ export function activateInTransaction(tx: Transaction, input: Parameters() +const pageSize = 50 + +export class LimitError extends Schema.TaggedErrorClass()("SessionRevertLimitError", { + sessionID: SessionID, + maxFiles: Schema.Number, +}) {} export const RevertInput = Schema.Struct({ sessionID: SessionID, @@ -22,7 +28,7 @@ export const RevertInput = Schema.Struct({ export type RevertInput = Schema.Schema.Type export interface Interface { - readonly revert: (input: RevertInput) => Effect.Effect + readonly revert: (input: RevertInput) => Effect.Effect readonly unrevert: (input: { sessionID: SessionID }) => Effect.Effect readonly cleanup: (session: Session.Info, mutationEpoch?: number) => Effect.Effect } @@ -36,56 +42,100 @@ export const layer = Layer.effect( const snap = yield* Snapshot.Service const storage = yield* Storage.Service const events = yield* EventV2Bridge.Service - const summary = yield* SessionSummary.Service const state = yield* SessionRunState.Service const revertUnlocked = Effect.fn("SessionRevert.revertUnlocked")(function* (input: RevertInput) { yield* state.assertNotBusy(input.sessionID) - const all = yield* sessions.messages({ sessionID: input.sessionID }).pipe(Effect.orDie) - let lastUser: SessionV1.User | undefined const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie) - - let rev: Session.Info["revert"] + const target = yield* sessions + .getClientMessage({ sessionID: input.sessionID, messageID: input.messageID }) + .pipe(Effect.orDie) + const lastUser = Option.getOrUndefined( + yield* sessions + .findMessage(input.sessionID, (message) => message.info.role === "user" && message.info.id <= input.messageID) + .pipe(Effect.orDie), + )?.info as SessionV1.User | undefined + const remaining = [] as SessionV1.Part[] + let rev: Session.Info["revert"] | undefined const patches: Snapshot.Patch[] = [] - for (const msg of all) { - if (msg.info.role === "user") lastUser = msg.info - const remaining = [] - for (const part of msg.parts) { - if (rev) { - if (part.type === "patch") patches.push(part) - continue - } - - if (!rev) { - if ((msg.info.id === input.messageID && !input.partID) || part.id === input.partID) { - const partID = remaining.some((item) => ["text", "tool"].includes(item.type)) ? input.partID : undefined - rev = { - messageID: !partID && lastUser ? lastUser.id : msg.info.id, - partID, - } - } - remaining.push(part) + const files = new Set() + const collect = (part: SessionV1.Part) => { + if (part.type !== "patch") return Effect.void + const novel = part.files.filter((file) => !files.has(file)) + if (files.size + novel.length > Snapshot.DiffLimits.captureCandidateFiles) + return Effect.fail( + new LimitError({ sessionID: input.sessionID, maxFiles: Snapshot.DiffLimits.captureCandidateFiles }), + ) + novel.forEach((file) => files.add(file)) + if (novel.length > 0) patches.push({ ...part, files: novel }) + return Effect.void + } + for (const part of target.parts) { + if (rev) { + yield* collect(part) + continue + } + if (!input.partID || part.id === input.partID) { + const partID = remaining.some((item) => ["text", "tool"].includes(item.type)) ? input.partID : undefined + rev = { + messageID: !partID && lastUser?.role === "user" ? lastUser.id : target.info.id, + partID, } } + remaining.push(part) } - if (!rev) return session + let after = MessageV2.cursor.encode({ id: target.info.id, time: target.info.time.created }) + while (true) { + const page = yield* sessions.messagesForwardPage({ sessionID: input.sessionID, limit: pageSize, after }) + for (const message of page.items) { + for (const part of message.parts) yield* collect(part) + } + if (!page.more || !page.cursor) break + after = page.cursor + } + rev.snapshot = session.revert?.snapshot ?? (yield* snap.track()) if (session.revert?.snapshot) yield* snap.restore(session.revert.snapshot) yield* snap.revert(patches) - if (rev.snapshot) rev.diff = yield* snap.diff(rev.snapshot) - const range = all.filter((msg) => msg.info.id >= rev.messageID) - const diffs = yield* summary.computeDiff({ messages: range }) + const range = yield* sessions + .snapshotRangeFromMessage({ + sessionID: input.sessionID, + messageID: rev.messageID, + }) + .pipe(Effect.orDie) + const manifest = + range.from && range.to ? yield* snap.diffManifest(range.from, range.to) : SessionSummary.emptyManifest() + const descriptor = { + completeness: manifest.completeness, + truncationReasons: [...manifest.truncationReasons], + manifestHash: manifest.manifestHash, + totalFiles: manifest.totalFiles, + totalFilesExact: manifest.totalFilesExact, + statisticsExact: manifest.statisticsExact, + includedFiles: manifest.includedFiles, + truncatedFiles: manifest.truncatedFiles, + } + const diffs = manifest.files.map( + (item) => + ({ + ...(item.file === undefined ? {} : { file: item.file }), + additions: item.additions, + deletions: item.deletions, + ...(item.status === undefined ? {} : { status: item.status }), + }) satisfies Snapshot.FileDiff, + ) yield* storage.write(["session_diff", input.sessionID], diffs).pipe(Effect.ignore) - yield* events.publish(Session.Event.Diff, { sessionID: input.sessionID, diff: diffs }) + yield* events.publish(Session.Event.Diff, { sessionID: input.sessionID, diff: diffs, manifest: descriptor }) yield* sessions.commitRevert({ sessionID: input.sessionID, revert: rev, summary: { - additions: diffs.reduce((sum, x) => sum + x.additions, 0), - deletions: diffs.reduce((sum, x) => sum + x.deletions, 0), - files: diffs.length, + additions: manifest.additions, + deletions: manifest.deletions, + files: manifest.totalFiles, + diffManifest: descriptor, }, }) return yield* sessions.get(input.sessionID).pipe(Effect.orDie) @@ -114,24 +164,22 @@ export const layer = Layer.effect( const current = yield* sessions.get(session.id).pipe(Effect.orDie) if (JSON.stringify(current.revert) !== JSON.stringify(session.revert)) return const sessionID = session.id - const msgs = yield* sessions.messages({ sessionID }).pipe(Effect.orDie) const messageID = session.revert.messageID - const remove = [] as SessionV1.WithParts[] let target: SessionV1.WithParts | undefined - for (const msg of msgs) { - if (msg.info.id < messageID) continue - if (msg.info.id > messageID) { - remove.push(msg) - continue - } - if (session.revert.partID) { - target = msg - continue + let before: string | undefined + while (true) { + const page = yield* sessions.messagesPage({ sessionID, limit: pageSize, before }).pipe(Effect.orDie) + const reachedTarget = page.items.some((message) => message.info.id <= messageID) + for (const message of page.items) { + if (message.info.id < messageID) continue + if (message.info.id === messageID && session.revert.partID) { + target = message + continue + } + yield* sessions.removeMessage({ sessionID, messageID: message.info.id }) } - remove.push(msg) - } - for (const msg of remove) { - yield* sessions.removeMessage({ sessionID, messageID: msg.info.id }) + if (reachedTarget || !page.more || !page.cursor) break + before = page.cursor } if (session.revert.partID && target) { const partID = session.revert.partID @@ -163,7 +211,6 @@ export const defaultLayer = Layer.suspend(() => Layer.provide(Snapshot.defaultLayer), Layer.provide(Storage.defaultLayer), Layer.provide(EventV2Bridge.defaultLayer), - Layer.provide(SessionSummary.defaultLayer), ), ) diff --git a/packages/deepagent-code/src/session/session.ts b/packages/deepagent-code/src/session/session.ts index 11554f649..50eeb89e1 100644 --- a/packages/deepagent-code/src/session/session.ts +++ b/packages/deepagent-code/src/session/session.ts @@ -18,9 +18,11 @@ import { NotFoundError } from "@/storage/storage" import { eq } from "drizzle-orm" import { and } from "drizzle-orm" import { gte } from "drizzle-orm" +import { gt } from "drizzle-orm" import { isNull } from "drizzle-orm" import { isNotNull } from "drizzle-orm" import { desc } from "drizzle-orm" +import { asc } from "drizzle-orm" import { like } from "drizzle-orm" import { sql } from "drizzle-orm" import { inArray } from "drizzle-orm" @@ -28,6 +30,7 @@ import { lt } from "drizzle-orm" import { or } from "drizzle-orm" import { notLike } from "drizzle-orm" import type { SQL } from "drizzle-orm" +import { getTableColumns } from "drizzle-orm" import { MessageTable, PartTable, @@ -125,7 +128,10 @@ export function isDefaultTitle(title: string) { ).test(title) } -type SessionRow = typeof SessionTable.$inferSelect +const { summary_diffs: _, ...sessionClientColumns } = getTableColumns(SessionTable) +type SessionRow = Omit & { + summary_diffs?: typeof SessionTable.$inferSelect.summary_diffs +} export function fromRow(row: SessionRow): Info { const summary = @@ -135,6 +141,7 @@ export function fromRow(row: SessionRow): Info { deletions: row.summary_deletions ?? 0, files: row.summary_files ?? 0, diffs: row.summary_diffs ?? undefined, + diffManifest: row.summary_diff_manifest ?? undefined, } : undefined const share = row.share_url ? { url: row.share_url } : undefined @@ -200,6 +207,7 @@ export function toRow(info: Info) { summary_deletions: info.summary?.deletions, summary_files: info.summary?.files, summary_diffs: info.summary?.diffs, + summary_diff_manifest: info.summary?.diffManifest, metadata: info.metadata, cost: info.cost ?? 0, tokens_input: (info.tokens ?? EmptyTokens).input, @@ -345,6 +353,18 @@ const Summary = Schema.Struct({ deletions: Schema.Finite, files: Schema.Finite, diffs: optionalOmitUndefined(Schema.Array(Snapshot.FileDiff)), + diffManifest: optionalOmitUndefined( + Schema.Struct({ + completeness: Snapshot.DiffManifest.fields.completeness, + truncationReasons: Snapshot.DiffManifest.fields.truncationReasons, + manifestHash: Snapshot.DiffManifest.fields.manifestHash, + totalFiles: Snapshot.DiffManifest.fields.totalFiles, + totalFilesExact: Snapshot.DiffManifest.fields.totalFilesExact, + statisticsExact: optionalOmitUndefined(Snapshot.DiffManifest.fields.statisticsExact), + includedFiles: Snapshot.DiffManifest.fields.includedFiles, + truncatedFiles: Snapshot.DiffManifest.fields.truncatedFiles, + }), + ), }) const Tokens = Schema.Struct({ @@ -557,6 +577,18 @@ export const Event = { schema: { sessionID: SessionID, diff: Schema.Array(Snapshot.FileDiff), + manifest: Schema.optional( + Schema.Struct({ + completeness: Snapshot.DiffManifest.fields.completeness, + truncationReasons: Snapshot.DiffManifest.fields.truncationReasons, + manifestHash: Snapshot.DiffManifest.fields.manifestHash, + totalFiles: Snapshot.DiffManifest.fields.totalFiles, + totalFilesExact: Snapshot.DiffManifest.fields.totalFilesExact, + statisticsExact: optionalOmitUndefined(Snapshot.DiffManifest.fields.statisticsExact), + includedFiles: Snapshot.DiffManifest.fields.includedFiles, + truncatedFiles: Snapshot.DiffManifest.fields.truncatedFiles, + }), + ), }, }), Error: EventV2.define({ @@ -693,6 +725,18 @@ export interface Interface { readonly assertRunnable: (sessionID: SessionID) => Effect.Effect readonly touch: (sessionID: SessionID) => Effect.Effect readonly get: (id: SessionID) => Effect.Effect + readonly getMessage: (input: { + sessionID: SessionID + messageID: MessageID + }) => Effect.Effect + readonly getClientMessage: (input: { + sessionID: SessionID + messageID: MessageID + }) => Effect.Effect + readonly turnSnapshotRange: (input: { + sessionID: SessionID + parentID: MessageID + }) => Effect.Effect<{ from?: string; to?: string }> readonly mutationEpoch: (sessionID: SessionID) => Effect.Effect readonly setTitle: (input: { sessionID: SessionID; title: string }) => Effect.Effect readonly setPreview: (input: { sessionID: SessionID; preview: string }) => Effect.Effect @@ -723,6 +767,15 @@ export interface Interface { limit: number before?: string }) => Effect.Effect<{ items: SessionV1.WithParts[]; more: boolean; cursor?: string }, NotFound> + readonly messagesForwardPage: (input: { + sessionID: SessionID + limit: number + after: string + }) => Effect.Effect<{ items: SessionV1.WithParts[]; more: boolean; cursor?: string }> + readonly snapshotRangeFromMessage: (input: { + sessionID: SessionID + messageID: MessageID + }) => Effect.Effect<{ from?: string; to?: string }, NotFound> readonly children: (parentID: SessionID) => Effect.Effect readonly remove: (sessionID: SessionID) => Effect.Effect readonly updateMessage: (msg: T) => Effect.Effect @@ -819,11 +872,66 @@ export const layer: Layer.Layer< }) const get = Effect.fn("Session.get")(function* (id: SessionID) { - const row = yield* db.select().from(SessionTable).where(eq(SessionTable.id, id)).get().pipe(Effect.orDie) + const row = yield* db + .select(sessionClientColumns) + .from(SessionTable) + .where(eq(SessionTable.id, id)) + .get() + .pipe(Effect.orDie) if (!row) return yield* Effect.fail(new NotFoundError({ message: `Session not found: ${id}` })) return fromRow(row) }) + const getMessage: Interface["getMessage"] = (input) => + MessageV2.get(input).pipe(Effect.provideService(Database.Service, database)) + const getClientMessage: Interface["getClientMessage"] = (input) => + MessageV2.clientGet(input).pipe(Effect.provideService(Database.Service, database)) + + const turnSnapshotRange: Interface["turnSnapshotRange"] = Effect.fn("Session.turnSnapshotRange")(function* (input) { + const message = or( + eq(MessageTable.id, input.parentID), + eq(sql`json_extract(${MessageTable.data}, '$.parentID')`, input.parentID), + ) + const start = yield* db + .select({ snapshot: sql`json_extract(${PartTable.data}, '$.snapshot')` }) + .from(PartTable) + .innerJoin( + MessageTable, + and(eq(MessageTable.id, PartTable.message_id), eq(MessageTable.session_id, PartTable.session_id)), + ) + .where( + and( + eq(MessageTable.session_id, input.sessionID), + message, + eq(sql`json_extract(${PartTable.data}, '$.type')`, "step-start"), + ), + ) + .orderBy(asc(MessageTable.time_created), asc(MessageTable.id), asc(PartTable.id)) + .get() + .pipe(Effect.orDie) + const finish = yield* db + .select({ snapshot: sql`json_extract(${PartTable.data}, '$.snapshot')` }) + .from(PartTable) + .innerJoin( + MessageTable, + and(eq(MessageTable.id, PartTable.message_id), eq(MessageTable.session_id, PartTable.session_id)), + ) + .where( + and( + eq(MessageTable.session_id, input.sessionID), + message, + eq(sql`json_extract(${PartTable.data}, '$.type')`, "step-finish"), + ), + ) + .orderBy(desc(MessageTable.time_created), desc(MessageTable.id), desc(PartTable.id)) + .get() + .pipe(Effect.orDie) + return { + ...(start?.snapshot ? { from: start.snapshot } : {}), + ...(finish?.snapshot ? { to: finish.snapshot } : {}), + } + }) + const mutationEpoch = Effect.fn("Session.mutationEpoch")(function* (sessionID: SessionID) { const row = yield* db .select({ mutationEpoch: SessionTable.mutation_epoch }) @@ -857,10 +965,10 @@ export const layer: Layer.Layer< const query = conditions.length > 0 ? db - .select() + .select(sessionClientColumns) .from(SessionTable) .where(and(...conditions)) - : db.select().from(SessionTable) + : db.select(sessionClientColumns).from(SessionTable) const rows = yield* query .orderBy(desc(SessionTable.time_updated), desc(SessionTable.id)) .limit(input?.limit ?? 100) @@ -888,7 +996,7 @@ export const layer: Layer.Layer< const children = Effect.fn("Session.children")(function* (parentID: SessionID) { const rows = yield* db - .select() + .select(sessionClientColumns) .from(SessionTable) .where(and(eq(SessionTable.parent_id, parentID))) .all() @@ -929,10 +1037,26 @@ export const layer: Layer.Layer< .pipe(Effect.orDie) if (existing && existing.session_id !== msg.sessionID) return yield* Effect.die(`Session.updateMessage: message ${msg.id} belongs to another Session`) - yield* events.publish(SessionV1.Event.MessageUpdated, { - sessionID: msg.sessionID, - info: MessageV2.stripActivityProgress(msg), - }) + yield* events.publish( + SessionV1.Event.MessageUpdated, + { + sessionID: msg.sessionID, + info: MessageV2.clientProjection(MessageV2.stripActivityProgress(msg)), + }, + { + commit: () => + db + .update(MessageTable) + .set({ + data: Object.fromEntries( + Object.entries(msg).filter(([key]) => key !== "id" && key !== "sessionID"), + ) as typeof MessageTable.$inferInsert.data, + }) + .where(and(eq(MessageTable.id, msg.id), eq(MessageTable.session_id, msg.sessionID))) + .run() + .pipe(Effect.orDie), + }, + ) return msg }).pipe(Effect.withSpan("Session.updateMessage")) @@ -949,7 +1073,10 @@ export const layer: Layer.Layer< ), ) if (!message) return - yield* events.publish(SessionV1.Event.MessageUpdated, { sessionID: input.sessionID, info: message.info }) + yield* events.publish(SessionV1.Event.MessageUpdated, { + sessionID: input.sessionID, + info: MessageV2.clientProjection(message.info), + }) return message.info }) @@ -1209,7 +1336,7 @@ export const layer: Layer.Layer< }) for (const row of rows) { - const message = MessageV2.stripActivityProgress(row.info) + const message = MessageV2.clientProjection(MessageV2.stripActivityProgress(row.info)) const messageEventID = EventV2.ID.make( `evt_${Hash.sha256(`fork-event:v1:${intentID}:message:${row.info.id}`).slice(0, 26)}`, ) @@ -1952,7 +2079,14 @@ export const layer: Layer.Layer< const ctx = yield* InstanceState.context const original = yield* get(input.sessionID) const title = getForkedTitle(original.title) - const sourceProjection = yield* MessageV2.promptHistoryProjectionEffect(input.sessionID).pipe( + const sourceProjection = yield* ( + input.messageID + ? MessageV2.promptHistoryCutoffProjectionEffect({ + sessionID: input.sessionID, + cutoffMessageID: input.messageID, + }) + : MessageV2.promptHistoryProjectionEffect(input.sessionID) + ).pipe( Effect.provideService(Database.Service, database), Effect.mapError( (error) => @@ -1971,29 +2105,15 @@ export const layer: Layer.Layer< .get() .pipe(Effect.orDie) if (!sourceSession) return yield* new NotFoundError({ message: `Session not found: ${input.sessionID}` }) - const cutoffIndex = input.messageID - ? sourceProjection.messages.findIndex((message) => message.info.id === input.messageID) - : sourceProjection.messages.length - if (cutoffIndex < 0) { - return yield* Effect.fail( - new ForkConflict({ intentID, reason: "fork cutoff is not part of the active effective history" }), - ) - } - if (sourceProjection.epoch > 0 && cutoffIndex < 2) { - return yield* Effect.fail( - new ForkConflict({ intentID, reason: "fork cutoff splits the active checkpoint pair" }), - ) - } const sourceMessages = - forkMode === "task" - ? sanitizeTaskHistory(sourceProjection.messages.slice(0, cutoffIndex)) - : sourceProjection.messages.slice(0, cutoffIndex) - if (sourceProjection.epoch > 0 && sourceMessages.length < 2) { + forkMode === "task" ? sanitizeTaskHistory(sourceProjection.messages) : sourceProjection.messages + const sourceUsesCheckpoint = sourceProjection.epoch > 0 && !sourceProjection.recoveryResolutionID + if (sourceUsesCheckpoint && sourceMessages.length < 2) { return yield* Effect.fail(new ForkConflict({ intentID, reason: "fork would omit the active checkpoint pair" })) } if ( forkMode === "task" && - sourceProjection.epoch > 0 && + sourceUsesCheckpoint && (sourceMessages[0]?.info.role !== "user" || sourceMessages[1]?.info.role !== "assistant" || !sourceMessages[1].info.summary) @@ -2228,7 +2348,7 @@ export const layer: Layer.Layer< // `parentID` would misclassify it as a background subagent. It mirrors the ForkOrigin marker // persisted below (context store), but travels with Session.Info so no extra IO/route is needed. const firstWindowID = HistoryAuthority.windowID() - const targetWindowID = sourceProjection.epoch === 0 ? firstWindowID : HistoryAuthority.windowID() + const targetWindowID = sourceUsesCheckpoint ? HistoryAuthority.windowID() : firstWindowID const messageIDMap = new Map( sourceMessages.map((message) => [ message.info.id, @@ -2286,7 +2406,7 @@ export const layer: Layer.Layer< projectionVersion: sourceProjection.projectionVersion, sanitationPolicyVersion: forkMode === "task" ? 3 : 1, ...(input.taskRequestHash ? { taskRequestHash: input.taskRequestHash } : {}), - targetPromptEpoch: sourceProjection.epoch === 0 ? 0 : 1, + targetPromptEpoch: sourceUsesCheckpoint ? 1 : 0, targetWindowID, targetEffectiveHistoryHash, targetWorldStateBaselineHash: worldStateBaseline.hash, @@ -2315,9 +2435,9 @@ export const layer: Layer.Layer< tokens: EmptyTokens, time: { created: Date.now(), updated: Date.now() }, } - const sourceCheckpointUserID = sourceProjection.epoch > 0 ? sourceMessages[0]?.info.id : undefined - const sourceCheckpointAssistantID = sourceProjection.epoch > 0 ? sourceMessages[1]?.info.id : undefined - const sourceMarker = sourceProjection.epoch > 0 ? sourceMessages[0] : undefined + const sourceCheckpointUserID = sourceUsesCheckpoint ? sourceMessages[0]?.info.id : undefined + const sourceCheckpointAssistantID = sourceUsesCheckpoint ? sourceMessages[1]?.info.id : undefined + const sourceMarker = sourceUsesCheckpoint ? sourceMessages[0] : undefined const sourceTailStartID = sourceMarker?.parts.find( (part): part is SessionV1.CompactionPart => part.type === "compaction", )?.tail_start_id @@ -2326,7 +2446,7 @@ export const layer: Layer.Layer< ? messageIDMap.get(sourceCheckpointAssistantID) : undefined const targetTailStartID = sourceTailStartID ? messageIDMap.get(sourceTailStartID) : undefined - if (sourceProjection.epoch > 0 && (!targetCheckpointUserID || !targetCheckpointAssistantID)) { + if (sourceUsesCheckpoint && (!targetCheckpointUserID || !targetCheckpointAssistantID)) { return yield* Effect.fail( new ForkConflict({ intentID, reason: "fork projection has an invalid checkpoint pair" }), ) @@ -2351,16 +2471,19 @@ export const layer: Layer.Layer< return yield* Effect.die( new ForkConflict({ intentID, reason: "fork admission is not ready for manifest commit" }), ) - const currentSource = yield* MessageV2.promptHistoryProjectionEffect(input.sessionID).pipe( - Effect.provideService(Database.Service, database), - Effect.orDie, - ) + const currentSource = input.messageID + ? yield* MessageV2.promptHistoryCutoffProjectionInTransaction(db, { + sessionID: input.sessionID, + cutoffMessageID: input.messageID, + }) + : yield* MessageV2.promptHistoryProjectionInTransaction(db, input.sessionID) const currentSession = yield* db .select({ mutation_epoch: SessionTable.mutation_epoch }) .from(SessionTable) .where(eq(SessionTable.id, input.sessionID)) .get() if ( + !currentSource || !currentSession || currentSession.mutation_epoch !== sourceSession.mutation_epoch || currentSource.epoch !== sourceProjection.epoch || @@ -2417,7 +2540,7 @@ export const layer: Layer.Layer< } const now = Date.now() - if (sourceProjection.epoch === 0) { + if (!sourceUsesCheckpoint) { yield* db .insert(SessionPromptEpochTable) .values({ @@ -2439,6 +2562,7 @@ export const layer: Layer.Layer< world_state_baseline_hash: worldStateBaseline.hash, authority_state: "ready", recovery_reason: null, + recovery_resolution_id: null, reason: "bootstrap", created_at: now, retired_at: null, @@ -2467,6 +2591,7 @@ export const layer: Layer.Layer< world_state_baseline_hash: null, authority_state: "ready", recovery_reason: null, + recovery_resolution_id: null, reason: "bootstrap", created_at: now, retired_at: now, @@ -2490,6 +2615,7 @@ export const layer: Layer.Layer< world_state_baseline_hash: worldStateBaseline.hash, authority_state: "ready", recovery_reason: null, + recovery_resolution_id: null, reason: "compaction", created_at: now, retired_at: null, @@ -2503,7 +2629,7 @@ export const layer: Layer.Layer< .values( cloned.map((message, ordinal) => ({ session_id: session.id, - prompt_epoch: sourceProjection.epoch === 0 ? 0 : 1, + prompt_epoch: sourceUsesCheckpoint ? 1 : 0, ordinal, message_id: message.info.id, })), @@ -2515,7 +2641,7 @@ export const layer: Layer.Layer< .values( worldStateBaseline.sections.map((section) => ({ session_id: session.id, - prompt_epoch: sourceProjection.epoch === 0 ? 0 : 1, + prompt_epoch: sourceUsesCheckpoint ? 1 : 0, section_id: section.sectionID, snapshot: section.snapshot, fragment: section.fragment, @@ -2551,7 +2677,7 @@ export const layer: Layer.Layer< projection_version: sourceProjection.projectionVersion, sanitation_policy_version: forkMode === "task" ? 3 : 1, target_session_id: session.id, - target_prompt_epoch: sourceProjection.epoch === 0 ? 0 : 1, + target_prompt_epoch: sourceUsesCheckpoint ? 1 : 0, target_window_id: targetWindowID, target_effective_history_hash: targetEffectiveHistoryHash, target_world_state_baseline_hash: worldStateBaseline.hash, @@ -2893,6 +3019,17 @@ export const layer: Layer.Layer< revert: info.revert === null ? undefined : (info.revert ?? current.revert), permission: info.permission === null ? undefined : (info.permission ?? current.permission), } as Info + if (next.summary?.diffs !== undefined) { + next.summary = { + ...next.summary, + diffs: next.summary.diffs.slice(0, MessageV2.ClientDiffLimits.files).map((item) => ({ + ...(item.file === undefined ? {} : { file: item.file }), + additions: item.additions, + deletions: item.deletions, + ...(item.status === undefined ? {} : { status: item.status }), + })), + } + } yield* events.publish(SessionV1.Event.Updated, { sessionID, info: next }) }) @@ -2991,6 +3128,7 @@ export const layer: Layer.Layer< summary_deletions: input.summary.deletions, summary_files: input.summary.files, summary_diffs: input.summary.diffs, + summary_diff_manifest: input.summary.diffManifest, } : {}), time_updated: now, @@ -2998,7 +3136,7 @@ export const layer: Layer.Layer< .where( and(eq(SessionTable.id, input.sessionID), eq(SessionTable.mutation_epoch, current.mutationEpoch)), ) - .returning() + .returning(sessionClientColumns) .get() .pipe(Effect.orDie) if (!row) return yield* Effect.die("Session mutation epoch changed inside an IMMEDIATE transaction") @@ -3083,7 +3221,64 @@ export const layer: Layer.Layer< }) const messagesPage: Interface["messagesPage"] = (input) => - MessageV2.page(input).pipe(Effect.provideService(Database.Service, database)) + MessageV2.clientPage(input).pipe(Effect.provideService(Database.Service, database)) + + const messagesForwardPage: Interface["messagesForwardPage"] = (input) => + MessageV2.forwardPage(input).pipe(Effect.provideService(Database.Service, database)) + + const snapshotRangeFromMessage: Interface["snapshotRangeFromMessage"] = Effect.fn( + "Session.snapshotRangeFromMessage", + )(function* (input) { + const target = yield* db + .select({ id: MessageTable.id, time: MessageTable.time_created }) + .from(MessageTable) + .where(and(eq(MessageTable.session_id, input.sessionID), eq(MessageTable.id, input.messageID))) + .get() + .pipe(Effect.orDie) + if (!target) return yield* new NotFoundError({ message: `Message not found: ${input.messageID}` }) + const atOrAfter = or( + gt(MessageTable.time_created, target.time), + and(eq(MessageTable.time_created, target.time), gte(MessageTable.id, target.id)), + ) + const start = yield* db + .select({ snapshot: sql`json_extract(${PartTable.data}, '$.snapshot')` }) + .from(PartTable) + .innerJoin( + MessageTable, + and(eq(MessageTable.id, PartTable.message_id), eq(MessageTable.session_id, PartTable.session_id)), + ) + .where( + and( + eq(MessageTable.session_id, input.sessionID), + atOrAfter, + eq(sql`json_extract(${PartTable.data}, '$.type')`, "step-start"), + ), + ) + .orderBy(asc(MessageTable.time_created), asc(MessageTable.id), asc(PartTable.id)) + .get() + .pipe(Effect.orDie) + const finish = yield* db + .select({ snapshot: sql`json_extract(${PartTable.data}, '$.snapshot')` }) + .from(PartTable) + .innerJoin( + MessageTable, + and(eq(MessageTable.id, PartTable.message_id), eq(MessageTable.session_id, PartTable.session_id)), + ) + .where( + and( + eq(MessageTable.session_id, input.sessionID), + atOrAfter, + eq(sql`json_extract(${PartTable.data}, '$.type')`, "step-finish"), + ), + ) + .orderBy(desc(MessageTable.time_created), desc(MessageTable.id), desc(PartTable.id)) + .get() + .pipe(Effect.orDie) + return { + ...(start?.snapshot ? { from: start.snapshot } : {}), + ...(finish?.snapshot ? { to: finish.snapshot } : {}), + } + }) const messages: Interface["messages"] = Effect.fn("Session.messages")(function* (input) { if (input.limit) { @@ -3148,7 +3343,7 @@ export const layer: Layer.Layer< const size = 50 let before: string | undefined while (true) { - const page = yield* MessageV2.page({ sessionID, limit: size, before }).pipe( + const page = yield* MessageV2.clientPage({ sessionID, limit: size, before }).pipe( Effect.provideService(Database.Service, database), ) if (page.items.length === 0) break @@ -3171,6 +3366,9 @@ export const layer: Layer.Layer< assertRunnable, touch, get, + getMessage, + getClientMessage, + turnSnapshotRange, mutationEpoch, setTitle, setPreview, @@ -3188,6 +3386,8 @@ export const layer: Layer.Layer< diff, messages, messagesPage, + messagesForwardPage, + snapshotRangeFromMessage, children, remove, updateMessage, @@ -3271,7 +3471,7 @@ function listByProject( const limit = input.limit ?? 100 return db - .select() + .select(sessionClientColumns) .from(SessionTable) .where(and(...conditions)) .orderBy(desc(SessionTable.time_updated)) @@ -3321,10 +3521,10 @@ export function* listGlobal(input?: { const query = conditions.length > 0 ? db - .select() + .select(sessionClientColumns) .from(SessionTable) .where(and(...conditions)) - : db.select().from(SessionTable) + : db.select(sessionClientColumns).from(SessionTable) return query.orderBy(desc(SessionTable.time_updated), desc(SessionTable.id)).limit(limit).all().pipe(Effect.orDie) }) diff --git a/packages/deepagent-code/src/session/summary.ts b/packages/deepagent-code/src/session/summary.ts index 9b838b2cb..3f6225ee8 100644 --- a/packages/deepagent-code/src/session/summary.ts +++ b/packages/deepagent-code/src/session/summary.ts @@ -62,10 +62,25 @@ function unquoteGitPath(input: string) { return Buffer.from(bytes).toString() } +export const emptyManifest = (): Snapshot.DiffManifest => ({ + files: [], + additions: 0, + deletions: 0, + totalFiles: 0, + totalFilesExact: true, + statisticsExact: true, + includedFiles: 0, + truncatedFiles: 0, + manifestHash: "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + completeness: "complete", + truncationReasons: [], +}) + export interface Interface { readonly summarize: (input: { sessionID: SessionID; messageID: MessageID }) => Effect.Effect readonly diff: (input: { sessionID: SessionID; messageID?: MessageID }) => Effect.Effect readonly computeDiff: (input: { messages: SessionV1.WithParts[] }) => Effect.Effect + readonly computeManifest: (input: { messages: SessionV1.WithParts[] }) => Effect.Effect } export class Service extends Context.Service()("@deepagent-code/SessionSummary") {} @@ -78,10 +93,10 @@ export const layer = Layer.effect( const events = yield* EventV2Bridge.Service const config = yield* Config.Service - const computeDiff = Effect.fn("SessionSummary.computeDiff")(function* (input: { messages: SessionV1.WithParts[] }) { + const snapshots = (messages: SessionV1.WithParts[]) => { let from: string | undefined let to: string | undefined - for (const item of input.messages) { + for (const item of messages) { if (!from) { for (const part of item.parts) { if (part.type === "step-start" && part.snapshot) { @@ -94,53 +109,116 @@ export const layer = Layer.effect( if (part.type === "step-finish" && part.snapshot) to = part.snapshot } } - if (from && to) return yield* snapshot.diffFull(from, to) - return [] + return { from, to } + } + + const descriptor = (manifest: Snapshot.DiffManifest) => ({ + completeness: manifest.completeness, + truncationReasons: [...manifest.truncationReasons], + manifestHash: manifest.manifestHash, + totalFiles: manifest.totalFiles, + totalFilesExact: manifest.totalFilesExact, + statisticsExact: manifest.statisticsExact, + includedFiles: manifest.includedFiles, + truncatedFiles: manifest.truncatedFiles, + }) + + const computeManifest = Effect.fn("SessionSummary.computeManifest")(function* (input: { + messages: SessionV1.WithParts[] + }) { + const range = snapshots(input.messages) + if (range.from && range.to) return yield* snapshot.diffManifest(range.from, range.to) + return emptyManifest() + }) + + const computeDiff = Effect.fn("SessionSummary.computeDiff")(function* (input: { messages: SessionV1.WithParts[] }) { + return (yield* computeManifest(input)).files.map((item) => { + if (item.patch !== "") return item + const { patch, ...metadata } = item + return metadata + }) + }) + + const turnManifest = Effect.fn("SessionSummary.turnManifest")(function* (input: { + sessionID: SessionID + parentID: MessageID + }) { + const range = yield* sessions.turnSnapshotRange(input) + if (range.from && range.to) return yield* snapshot.diffManifest(range.from, range.to) + return emptyManifest() }) const summarize = Effect.fn("SessionSummary.summarize")(function* (input: { sessionID: SessionID messageID: MessageID }) { + const empty = emptyManifest() yield* sessions.setSummary({ sessionID: input.sessionID, summary: { additions: 0, deletions: 0, files: 0, + diffManifest: descriptor(empty), }, }) - yield* events.publish(Session.Event.Diff, { sessionID: input.sessionID, diff: [] }) + yield* events.publish(Session.Event.Diff, { + sessionID: input.sessionID, + diff: [], + manifest: descriptor(empty), + }) if ((yield* config.get()).snapshot === false) return - const all = yield* sessions.messages({ sessionID: input.sessionID }).pipe(Effect.orDie) - if (!all.length) return - - const messages = all.filter( - (m) => m.info.id === input.messageID || (m.info.role === "assistant" && m.info.parentID === input.messageID), - ) - const target = messages.find((m) => m.info.id === input.messageID) + const target = yield* sessions + .getClientMessage({ sessionID: input.sessionID, messageID: input.messageID }) + .pipe(Effect.orDie) if (!target || target.info.role !== "user") return - const msgDiffs = yield* computeDiff({ messages }) - target.info.summary = { ...target.info.summary, diffs: msgDiffs } + const manifest = yield* turnManifest({ sessionID: input.sessionID, parentID: target.info.id }) + const diffs = manifest.files.map((item) => { + if (item.patch !== "") return item + const { patch, ...metadata } = item + return metadata + }) + target.info.summary = { ...target.info.summary, diffs, diffManifest: descriptor(manifest) } + yield* sessions.setSummary({ + sessionID: input.sessionID, + summary: { + additions: manifest.additions, + deletions: manifest.deletions, + files: manifest.totalFiles, + diffManifest: descriptor(manifest), + }, + }) + yield* events.publish(Session.Event.Diff, { + sessionID: input.sessionID, + diff: diffs, + manifest: descriptor(manifest), + }) yield* sessions.updateMessage(target.info) }) const diff = Effect.fn("SessionSummary.diff")(function* (input: { sessionID: SessionID; messageID?: MessageID }) { if (!input.messageID) return [] - const message = (yield* sessions.messages({ sessionID: input.sessionID }).pipe(Effect.orDie)).find( - (item) => item.info.id === input.messageID, + const message = yield* sessions.getClientMessage({ sessionID: input.sessionID, messageID: input.messageID }).pipe( + Effect.catchTag("NotFoundError", () => Effect.succeed(undefined)), + Effect.orDie, ) if (!message || message.info.role !== "user") return [] const diffs = message.info.summary?.diffs ?? [] return diffs.map((item) => { - if (item.file === undefined) return item + const metadata = { + ...(item.file === undefined ? {} : { file: item.file }), + additions: item.additions, + deletions: item.deletions, + ...(item.status === undefined ? {} : { status: item.status }), + } satisfies Snapshot.FileDiff + if (item.file === undefined) return metadata const file = unquoteGitPath(item.file) - if (file === item.file) return item - return { ...item, file } + if (file === item.file) return metadata + return { ...metadata, file } }) }) - return Service.of({ summarize, diff, computeDiff }) + return Service.of({ summarize, diff, computeDiff, computeManifest }) }), ) diff --git a/packages/deepagent-code/src/snapshot/index.ts b/packages/deepagent-code/src/snapshot/index.ts index 723d496ab..d931a565e 100644 --- a/packages/deepagent-code/src/snapshot/index.ts +++ b/packages/deepagent-code/src/snapshot/index.ts @@ -6,6 +6,7 @@ import { AppProcess } from "@deepagent-code/core/process" import { InstanceState } from "@/effect/instance-state" import { FSUtil } from "@deepagent-code/core/fs-util" import { Hash } from "@deepagent-code/core/util/hash" +import { NonNegativeInt } from "@deepagent-code/core/schema" import { Config } from "@/config/config" import { Global } from "@deepagent-code/core/global" import * as Log from "@deepagent-code/core/util/log" @@ -28,6 +29,51 @@ export const FileDiff = Schema.Struct({ }).annotate({ identifier: "SnapshotFileDiff" }) export type FileDiff = typeof FileDiff.Type +export const DiffLimits = { + candidateFiles: 1_000, + captureCandidateFiles: 10_000, + discoveryOutputBytes: 4 * 1024 * 1024, + manifestBytes: 256 * 1024, + captureFileBytes: 8 * 1024 * 1024, + captureTotalBytes: 64 * 1024 * 1024, + sourceFileBytes: 1 * 1024 * 1024, + sourceTotalBytes: 16 * 1024 * 1024, + patchFileBytes: 512 * 1024, + patchTotalBytes: 4 * 1024 * 1024, + fallbackFiles: 2, + fallbackWallTime: Duration.seconds(1), + wallTime: Duration.seconds(15), +} as const + +export const DiffTruncationReason = Schema.Literals([ + "candidate_file_limit", + "discovery_output_limit", + "discovery_failed", + "manifest_bytes_limit", + "source_file_limit", + "source_total_limit", + "patch_file_limit", + "patch_total_limit", + "materialization_failed", + "time_limit", +]) +export type DiffTruncationReason = typeof DiffTruncationReason.Type + +export const DiffManifest = Schema.Struct({ + files: Schema.Array(FileDiff), + additions: Schema.Finite, + deletions: Schema.Finite, + totalFiles: NonNegativeInt, + totalFilesExact: Schema.Boolean, + statisticsExact: Schema.Boolean, + includedFiles: NonNegativeInt, + truncatedFiles: NonNegativeInt, + manifestHash: Schema.String, + completeness: Schema.Literals(["complete", "truncated"]), + truncationReasons: Schema.Array(DiffTruncationReason), +}) +export type DiffManifest = typeof DiffManifest.Type + const log = Log.create({ service: "snapshot" }) const prune = "7.days" const limit = 2 * 1024 * 1024 @@ -38,6 +84,16 @@ interface GitResult { readonly code: ChildProcessSpawner.ExitCode readonly text: string readonly stderr: string + readonly stdoutTruncated: boolean + readonly timedOut: boolean +} + +type DiffRow = { + file: string + status: "added" | "deleted" | "modified" + binary: boolean + additions: number + deletions: number } type State = Omit @@ -50,6 +106,8 @@ export interface Interface { readonly restore: (snapshot: string) => Effect.Effect readonly revert: (patches: Patch[]) => Effect.Effect readonly diff: (hash: string) => Effect.Effect + readonly diffManifest: (from: string, to: string) => Effect.Effect + readonly diffFullManifest: (from: string, to: string) => Effect.Effect readonly diffFull: (from: string, to: string) => Effect.Effect } @@ -86,15 +144,32 @@ export const layer: Layer.Layer list.join("\0") + "\0" const git = Effect.fnUntraced( - function* (cmd: string[], opts?: { cwd?: string; env?: Record; stdin?: string }) { + function* ( + cmd: string[], + opts?: { + cwd?: string + env?: Record + stdin?: string + maxOutputBytes?: number + maxErrorBytes?: number + timeout?: Duration.Input + }, + ) { const result = yield* appProcess.run( ChildProcess.make("git", cmd, { cwd: opts?.cwd, env: opts?.env, extendEnv: true }), - { stdin: opts?.stdin }, + { + stdin: opts?.stdin, + maxOutputBytes: opts?.maxOutputBytes, + maxErrorBytes: opts?.maxErrorBytes ?? DiffLimits.discoveryOutputBytes, + timeout: opts?.timeout, + }, ) return { code: ChildProcessSpawner.ExitCode(result.exitCode), text: result.stdout.toString("utf8"), stderr: result.stderr.toString("utf8"), + stdoutTruncated: result.stdoutTruncated, + timedOut: false, } satisfies GitResult }, Effect.catch((err) => @@ -102,12 +177,14 @@ export const layer: Layer.Layer() + const ignore = Effect.fnUntraced(function* (files: string[], timeout = DiffLimits.wallTime) { + if (!files.length) return { ok: true as const, files: new Set() } const check = yield* git( [ ...quote, @@ -123,15 +200,22 @@ export const layer: Layer.Layer() - return new Set(check.text.split("\0").filter(Boolean)) + if ( + (check.code !== 0 && check.code !== 1) || + check.stdoutTruncated || + check.timedOut + ) + return { ok: false as const, timedOut: check.timedOut, outputLimited: check.stdoutTruncated } + return { ok: true as const, files: new Set(check.text.split("\0").filter(Boolean)) } }) const drop = Effect.fnUntraced(function* (files: string[]) { - if (!files.length) return - yield* git( + if (!files.length) return true + const result = yield* git( [ ...cfg, ...args(["rm", "--cached", "-f", "--ignore-unmatch", "--pathspec-from-file=-", "--pathspec-file-nul"]), @@ -139,24 +223,32 @@ export const layer: Layer.Layer fs.exists(file).pipe(Effect.orDie) @@ -198,65 +290,112 @@ export const layer: Layer.Layer DiffLimits.captureCandidateFiles) { + log.warn("snapshot capture exceeded candidate budget", { + files: all.length, + limit: DiffLimits.captureCandidateFiles, + }) + return false + } // Resolve source-repo ignore rules against the exact candidate set. // --no-index keeps this pattern-based even when a path is already tracked. const ignored = yield* ignore(all) + if (!ignored.ok) { + log.warn("failed to resolve snapshot ignore rules", { + timedOut: ignored.timedOut, + outputLimited: ignored.outputLimited, + }) + return false + } // Remove newly-ignored files from snapshot index to prevent re-adding - if (ignored.size > 0) { - const ignoredFiles = Array.from(ignored) + if (ignored.files.size > 0) { + const ignoredFiles = Array.from(ignored.files) log.info("removing gitignored files from snapshot", { count: ignoredFiles.length }) - yield* drop(ignoredFiles) + if (!(yield* drop(ignoredFiles))) return false } - const allow = all.filter((item) => !ignored.has(item)) - if (!allow.length) return - - const large = new Set( - (yield* Effect.all( - allow.map((item) => - fs - .stat(path.join(state.directory, item)) - .pipe(Effect.catch(() => Effect.void)) - .pipe( - Effect.map((stat) => { - if (!stat || stat.type !== "File") return - const size = typeof stat.size === "bigint" ? Number(stat.size) : stat.size - return size > limit ? item : undefined - }), - ), - ), - { concurrency: 8 }, - )).filter((item): item is string => Boolean(item)), - ) + const allow = all.filter((item) => !ignored.files.has(item)) + if (!allow.length) return true + + const sizes = (yield* Effect.all( + allow.map((item) => + fs + .stat(path.join(state.directory, item)) + .pipe(Effect.catch(() => Effect.void)) + .pipe( + Effect.map((stat) => { + if (!stat || stat.type !== "File") return + const size = typeof stat.size === "bigint" ? Number(stat.size) : stat.size + return { item, size } + }), + ), + ), + { concurrency: 8 }, + )).filter((item): item is { item: string; size: number } => Boolean(item)) + const large = new Set(sizes.filter((item) => item.size > limit).map((item) => item.item)) const block = new Set(untracked.filter((item) => large.has(item))) yield* sync(Array.from(block)) + const staged = allow.filter((item) => !block.has(item)) + const stagedSizes = sizes.filter((item) => !block.has(item.item)) + const oversized = stagedSizes.find((item) => item.size > DiffLimits.captureFileBytes) + const totalBytes = stagedSizes.reduce((total, item) => total + item.size, 0) + if (oversized || totalBytes > DiffLimits.captureTotalBytes) { + log.warn("snapshot capture exceeded source budget", { + files: staged.length, + totalBytes, + totalLimit: DiffLimits.captureTotalBytes, + oversizedFile: oversized?.item, + fileLimit: DiffLimits.captureFileBytes, + }) + return false + } // Stage only the allowed candidate paths so snapshot updates stay scoped. - yield* stage(allow.filter((item) => !block.has(item))) + return yield* stage(staged) }) + const capture = () => + add().pipe( + Effect.timeoutOrElse({ + duration: DiffLimits.wallTime, + orElse: () => + Effect.logWarning("snapshot capture exceeded wall-time budget").pipe(Effect.as(false)), + }), + ) + const cleanup = Effect.fnUntraced(function* () { return yield* locked( Effect.gen(function* () { @@ -291,8 +430,9 @@ export const layer: Layer.Layer !ignored.has(item)) + .filter((item) => !ignored.files.has(item)) .map((x) => path.join(state.worktree, x).replaceAll("\\", "/")), } }), @@ -477,11 +626,13 @@ export const layer: Layer.Layer() + const status = new Map() + const remaining = () => Duration.millis(Math.max(1, deadline - Date.now())) + + const statuses = yield* git( + [...quote, ...args(["diff", "--no-ext-diff", "--name-status", "--no-renames", from, to, "--", "."])], + { + cwd: state.directory, + maxOutputBytes: DiffLimits.discoveryOutputBytes, + timeout: remaining(), + }, + ) + if (statuses.code !== 0) reasons.add("discovery_failed") + if (statuses.stdoutTruncated) { + reasons.add("discovery_output_limit") + log.warn("snapshot diff candidate discovery exceeded output budget", { + maxOutputBytes: DiffLimits.discoveryOutputBytes, + }) } + if (statuses.timedOut) reasons.add("time_limit") - type Ref = { - file: string - side: "before" | "after" - ref: string + for (const line of statuses.text.trim().split("\n")) { + if (!line) continue + const [code, file] = line.split("\t") + if (!code || !file) continue + status.set(file, code.startsWith("A") ? "added" : code.startsWith("D") ? "deleted" : "modified") } - const show = Effect.fnUntraced(function* (row: Row) { - if (row.binary) return ["", ""] - if (row.status === "added") { + const numstat = yield* git( + [...quote, ...args(["diff", "--no-ext-diff", "--no-renames", "--numstat", from, to, "--", "."])], + { + cwd: state.directory, + maxOutputBytes: DiffLimits.discoveryOutputBytes, + timeout: remaining(), + }, + ) + if (numstat.code !== 0) reasons.add("discovery_failed") + if (numstat.stdoutTruncated) reasons.add("discovery_output_limit") + if (numstat.timedOut) reasons.add("time_limit") + + const discovered = numstat.text + .trim() + .split("\n") + .filter(Boolean) + .flatMap((line) => { + const [adds, dels, file] = line.split("\t") + if (!file) return [] + const binary = adds === "-" && dels === "-" + const additions = binary ? 0 : parseInt(adds) + const deletions = binary ? 0 : parseInt(dels) return [ - "", - yield* git([...cfg, ...args(["show", `${to}:${row.file}`])]).pipe(Effect.map((item) => item.text)), + { + file, + status: status.get(file) ?? "modified", + binary, + additions: Number.isFinite(additions) ? additions : 0, + deletions: Number.isFinite(deletions) ? deletions : 0, + } satisfies DiffRow, ] + }) + // Hide ignored-file removals from the user-facing diff output. + const ignored = yield* ignore( + discovered.map((r) => r.file), + remaining(), + ) + if (!ignored.ok) { + reasons.add(ignored.timedOut ? "time_limit" : "discovery_failed") + if (ignored.outputLimited) reasons.add("discovery_output_limit") + } + const visibleDiscovered = ignored.ok + ? discovered.filter((row) => !ignored.files.has(row.file)) + : discovered + if (visibleDiscovered.length > DiffLimits.candidateFiles || numstat.stdoutTruncated) + reasons.add("candidate_file_limit") + const rows = visibleDiscovered.slice(0, DiffLimits.candidateFiles) + + const files: FileDiff[] = [] + let manifestBytes = 2 + for (const row of rows) { + const item = { + file: row.file, + ...(row.binary ? { patch: "" } : {}), + additions: row.additions, + deletions: row.deletions, + status: row.status, + } satisfies FileDiff + const bytes = Buffer.byteLength(JSON.stringify(item)) + (files.length ? 1 : 0) + if (manifestBytes + bytes > DiffLimits.manifestBytes) { + reasons.add("manifest_bytes_limit") + break } - if (row.status === "deleted") { - return [ - yield* git([...cfg, ...args(["show", `${from}:${row.file}`])]).pipe( - Effect.map((item) => item.text), + files.push(item) + manifestBytes += bytes + } + + const totalFilesExact: boolean = + numstat.code === 0 && !numstat.stdoutTruncated && !numstat.timedOut && ignored.ok + const totalFiles = totalFilesExact + ? visibleDiscovered.length + : Math.max(visibleDiscovered.length, files.length + 1) + const manifest = { + files, + additions: files.reduce((sum, item) => sum + item.additions, 0), + deletions: files.reduce((sum, item) => sum + item.deletions, 0), + totalFiles, + totalFilesExact, + statisticsExact: + totalFilesExact && files.length === totalFiles && reasons.size === 0, + includedFiles: files.length, + truncatedFiles: Math.max(0, totalFiles - files.length), + manifestHash: `sha256:${Hash.sha256( + JSON.stringify({ + version: 1, + from, + to, + files, + totalFiles, + totalFilesExact, + reasons: [...reasons].sort(), + }), + )}`, + completeness: + reasons.size || files.length < totalFiles ? ("truncated" as const) : ("complete" as const), + truncationReasons: [...reasons].sort(), + } satisfies DiffManifest + return manifest + }) + + const diffManifest = Effect.fnUntraced(function* (from: string, to: string) { + const deadline = Date.now() + Duration.toMillis(DiffLimits.wallTime) + return yield* locked(buildDiffManifest(from, to, deadline)) + }) + + const materializeDiff = Effect.fnUntraced(function* ( + from: string, + to: string, + manifest: DiffManifest, + deadline: number, + ) { + const remaining = () => Duration.millis(Math.max(1, deadline - Date.now())) + const result: FileDiff[] = [] + const reasons = new Set(manifest.truncationReasons) + const rows: DiffRow[] = manifest.files.map((item) => ({ + file: item.file ?? "", + status: item.status ?? "modified", + binary: item.patch === "", + additions: item.additions, + deletions: item.deletions, + })) + type Ref = { file: string; side: "before" | "after"; ref: string } + const show = Effect.fnUntraced(function* (row: DiffRow) { + if (row.binary) return ["", ""] + const read = (ref: string) => + git([...cfg, ...args(["show", ref])], { + cwd: state.directory, + maxOutputBytes: DiffLimits.sourceFileBytes, + timeout: Duration.millis( + Math.max( + 1, + Math.min(Duration.toMillis(DiffLimits.fallbackWallTime), deadline - Date.now()), + ), ), - "", - ] + }) + const accept = (result: GitResult) => + result.code === 0 && !result.stdoutTruncated && !result.timedOut ? result.text : undefined + if (row.status === "added") { + const after = accept(yield* read(`${to}:${row.file}`)) + return after === undefined ? undefined : (["", after] as const) } - return yield* Effect.all( - [ - git([...cfg, ...args(["show", `${from}:${row.file}`])]).pipe(Effect.map((item) => item.text)), - git([...cfg, ...args(["show", `${to}:${row.file}`])]).pipe(Effect.map((item) => item.text)), - ], - { concurrency: 2 }, - ) + if (row.status === "deleted") { + const before = accept(yield* read(`${from}:${row.file}`)) + return before === undefined ? undefined : ([before, ""] as const) + } + const pair = yield* Effect.all([read(`${from}:${row.file}`), read(`${to}:${row.file}`)], { + concurrency: 2, + }) + const before = accept(pair[0]) + const after = accept(pair[1]) + return before === undefined || after === undefined ? undefined : ([before, after] as const) }) const load = Effect.fnUntraced( - function* (rows: Row[]) { + function* (rows: DiffRow[], remainingSourceBytes: number) { const refs = rows.flatMap((row) => { if (row.binary) return [] if (row.status === "added") return [{ file: row.file, side: "after", ref: `${to}:${row.file}` } satisfies Ref] - if (row.status === "deleted") { + if (row.status === "deleted") return [{ file: row.file, side: "before", ref: `${from}:${row.file}` } satisfies Ref] - } return [ { file: row.file, side: "before", ref: `${from}:${row.file}` } satisfies Ref, { file: row.file, side: "after", ref: `${to}:${row.file}` } satisfies Ref, ] }) if (!refs.length) return new Map() - const batch = yield* appProcess.run( ChildProcess.make("git", [...cfg, ...args(["cat-file", "--batch"])], { cwd: state.directory, extendEnv: true, }), - { stdin: refs.map((item) => item.ref).join("\n") + "\n" }, + { + stdin: refs.map((item) => item.ref).join("\n") + "\n", + maxOutputBytes: + Math.max(1, Math.min(DiffLimits.sourceTotalBytes, remainingSourceBytes)) + refs.length * 128, + maxErrorBytes: DiffLimits.discoveryOutputBytes, + timeout: remaining(), + }, ) - if (batch.exitCode !== 0) { - log.info("git cat-file --batch failed during snapshot diff, falling back to per-file git show", { - stderr: batch.stderr.toString("utf8"), - refs: refs.length, - }) - return - } - const out = batch.stdout - - const fail = (msg: string, extra?: Record) => { - log.info(msg, { ...extra, refs: refs.length }) - return undefined - } - + if (batch.exitCode !== 0 || batch.stdoutTruncated || batch.stderrTruncated) return const map = new Map() - const dec = new TextDecoder() - let i = 0 + const decoder = new TextDecoder() + let offset = 0 for (const ref of refs) { - let end = i - while (end < out.length && out[end] !== 10) end += 1 - if (end >= out.length) { - return fail( - "git cat-file --batch returned a truncated header during snapshot diff, falling back to per-file git show", - ) - } - - const head = dec.decode(out.slice(i, end)) - i = end + 1 + let end = offset + while (end < batch.stdout.length && batch.stdout[end] !== 10) end += 1 + if (end >= batch.stdout.length) return + const head = decoder.decode(batch.stdout.slice(offset, end)) + offset = end + 1 const hit = map.get(ref.file) ?? { before: "", after: "" } if (head.endsWith(" missing")) { map.set(ref.file, hit) continue } - const match = head.match(/^[0-9a-f]+ blob (\d+)$/) - if (!match) { - return fail( - "git cat-file --batch returned an unexpected header during snapshot diff, falling back to per-file git show", - { head }, - ) - } - - const size = Number(match[1]) - if (!Number.isInteger(size) || size < 0 || i + size >= out.length || out[i + size] !== 10) { - return fail( - "git cat-file --batch returned truncated content during snapshot diff, falling back to per-file git show", - { head }, - ) + const size = Number(match?.[1]) + if (!match || !Number.isInteger(size) || size < 0) return + if (size > DiffLimits.sourceFileBytes) { + reasons.add("source_file_limit") + return } - - const text = dec.decode(out.slice(i, i + size)) + if (offset + size >= batch.stdout.length || batch.stdout[offset + size] !== 10) return + const text = decoder.decode(batch.stdout.slice(offset, offset + size)) if (ref.side === "before") hit.before = text if (ref.side === "after") hit.after = text map.set(ref.file, hit) - i += size + 1 + offset += size + 1 } - - if (i !== out.length) { - return fail( - "git cat-file --batch returned trailing data during snapshot diff, falling back to per-file git show", - ) - } - - return map + return offset === batch.stdout.length ? map : undefined }, Effect.scoped, Effect.catch(() => @@ -630,71 +891,107 @@ export const layer: Layer.Layer() - - const statuses = yield* git( - [...quote, ...args(["diff", "--no-ext-diff", "--name-status", "--no-renames", from, to, "--", "."])], - { cwd: state.directory }, - ) - - for (const line of statuses.text.trim().split("\n")) { - if (!line) continue - const [code, file] = line.split("\t") - if (!code || !file) continue - status.set(file, code.startsWith("A") ? "added" : code.startsWith("D") ? "deleted" : "modified") - } - - const numstat = yield* git( - [...quote, ...args(["diff", "--no-ext-diff", "--no-renames", "--numstat", from, to, "--", "."])], - { - cwd: state.directory, - }, - ) - - const rows = numstat.text - .trim() - .split("\n") - .filter(Boolean) - .flatMap((line) => { - const [adds, dels, file] = line.split("\t") - if (!file) return [] - const binary = adds === "-" && dels === "-" - const additions = binary ? 0 : parseInt(adds) - const deletions = binary ? 0 : parseInt(dels) - return [ - { - file, - status: status.get(file) ?? "modified", - binary, - additions: Number.isFinite(additions) ? additions : 0, - deletions: Number.isFinite(deletions) ? deletions : 0, - } satisfies Row, - ] - }) - - // Hide ignored-file removals from the user-facing diff output. - const ignored = yield* ignore(rows.map((r) => r.file)) - if (ignored.size > 0) { - const filtered = rows.filter((r) => !ignored.has(r.file)) - rows.length = 0 - rows.push(...filtered) - } - const step = 100 + let sourceBytes = 0 + let patchBytes = 0 + let fallbackFiles = 0 const patch = (file: string, before: string, after: string) => formatPatch(structuredPatch(file, file, before, after, "", "", { context: Number.MAX_SAFE_INTEGER })) for (let i = 0; i < rows.length; i += step) { const run = rows.slice(i, i + step) - const text = yield* load(run) + if ( + Date.now() >= deadline || + sourceBytes >= DiffLimits.sourceTotalBytes || + patchBytes >= DiffLimits.patchTotalBytes + ) { + if (Date.now() >= deadline) reasons.add("time_limit") + if (sourceBytes >= DiffLimits.sourceTotalBytes) reasons.add("source_total_limit") + if (patchBytes >= DiffLimits.patchTotalBytes) reasons.add("patch_total_limit") + result.push(...rows.slice(i).map(({ file, additions, deletions, status }) => ({ file, additions, deletions, status }))) + break + } + const text = yield* load(run, DiffLimits.sourceTotalBytes - sourceBytes) for (const row of run) { - const hit = text?.get(row.file) ?? { before: "", after: "" } - const [before, after] = row.binary ? ["", ""] : text ? [hit.before, hit.after] : yield* show(row) + if ( + Date.now() >= deadline || + sourceBytes >= DiffLimits.sourceTotalBytes || + patchBytes >= DiffLimits.patchTotalBytes + ) { + if (Date.now() >= deadline) reasons.add("time_limit") + if (sourceBytes >= DiffLimits.sourceTotalBytes) reasons.add("source_total_limit") + if (patchBytes >= DiffLimits.patchTotalBytes) reasons.add("patch_total_limit") + result.push( + ...run + .slice(run.indexOf(row)) + .map(({ file, additions, deletions, status }) => ({ file, additions, deletions, status })), + ) + break + } + if (text && !text.has(row.file) && !row.binary) { + reasons.add("materialization_failed") + result.push({ + file: row.file, + patch: "", + additions: row.additions, + deletions: row.deletions, + status: row.status, + }) + continue + } + const hit = text?.get(row.file) + const canFallback = !text && !row.binary && fallbackFiles < DiffLimits.fallbackFiles + if (canFallback) fallbackFiles += 1 + const contents = row.binary + ? (["", ""] as const) + : hit + ? ([hit.before, hit.after] as const) + : canFallback + ? yield* show(row) + : undefined + if (!contents) { + reasons.add("materialization_failed") + result.push({ + file: row.file, + patch: "", + additions: row.additions, + deletions: row.deletions, + status: row.status, + }) + continue + } + const [before, after] = contents + const observedSourceBytes = Buffer.byteLength(before) + Buffer.byteLength(after) + if ( + observedSourceBytes > DiffLimits.sourceFileBytes || + sourceBytes + observedSourceBytes > DiffLimits.sourceTotalBytes + ) { + reasons.add( + observedSourceBytes > DiffLimits.sourceFileBytes ? "source_file_limit" : "source_total_limit", + ) + result.push({ + file: row.file, + patch: "", + additions: row.additions, + deletions: row.deletions, + status: row.status, + }) + continue + } + sourceBytes += observedSourceBytes + const body = row.binary ? "" : patch(row.file, before, after) + const patchSize = Buffer.byteLength(body) + const bounded = + patchSize > DiffLimits.patchFileBytes || patchBytes + patchSize > DiffLimits.patchTotalBytes + ? "" + : body + if (patchSize > DiffLimits.patchFileBytes) reasons.add("patch_file_limit") + else if (patchBytes + patchSize > DiffLimits.patchTotalBytes) reasons.add("patch_total_limit") + patchBytes += Buffer.byteLength(bounded) result.push({ file: row.file, - patch: row.binary ? "" : patch(row.file, before, after), + patch: bounded, additions: row.additions, deletions: row.deletions, status: row.status, @@ -702,11 +999,35 @@ export const layer: Layer.Layer 0 ? ("truncated" as const) : manifest.completeness, + truncationReasons, + } satisfies DiffManifest + }) + + const diffFullManifest = Effect.fnUntraced(function* (from: string, to: string) { + const deadline = Date.now() + Duration.toMillis(DiffLimits.wallTime) + return yield* locked( + Effect.gen(function* () { + const manifest = yield* buildDiffManifest(from, to, deadline) + if (Date.now() < deadline) return yield* materializeDiff(from, to, manifest, deadline) + return { + ...manifest, + completeness: "truncated" as const, + truncationReasons: [...new Set([...manifest.truncationReasons, "time_limit" as const])].sort(), + } }), ) }) + const diffFull = Effect.fnUntraced(function* (from: string, to: string) { + return (yield* diffFullManifest(from, to)).files + }) + yield* cleanup().pipe( Effect.catchCause((cause) => { log.error("cleanup loop failed", { cause: Cause.pretty(cause) }) @@ -717,7 +1038,7 @@ export const layer: Layer.Layer s.diff(hash)) }), + diffManifest: Effect.fn("Snapshot.diffManifest")(function* (from: string, to: string) { + return yield* InstanceState.useEffect(state, (s) => s.diffManifest(from, to)) + }), + diffFullManifest: Effect.fn("Snapshot.diffFullManifest")(function* (from: string, to: string) { + return yield* InstanceState.useEffect(state, (s) => s.diffFullManifest(from, to)) + }), diffFull: Effect.fn("Snapshot.diffFull")(function* (from: string, to: string) { return yield* InstanceState.useEffect(state, (s) => s.diffFull(from, to)) }), diff --git a/packages/deepagent-code/src/sync/replay-protocol.ts b/packages/deepagent-code/src/sync/replay-protocol.ts new file mode 100644 index 000000000..bd61194e1 --- /dev/null +++ b/packages/deepagent-code/src/sync/replay-protocol.ts @@ -0,0 +1,61 @@ +import { EventV2 } from "@deepagent-code/core/event" + +export const SyncReplayLimits = { + events: EventV2.AGGREGATE_READ_BATCH_EVENTS, + eventDataBytes: EventV2.AGGREGATE_READ_BATCH_BYTES, + eventsBytes: EventV2.AGGREGATE_READ_BATCH_BYTES + 1024 * 1024, + requestBytes: EventV2.AGGREGATE_READ_BATCH_BYTES + 1024 * 1024 + 64 * 1024, + directoryCharacters: 4_096, + eventIDCharacters: 200, + aggregateIDCharacters: 4_096, + typeCharacters: 256, +} as const + +type ReplayEvent = Pick + +export function encodeReplayRequestPrefix(directory: string, events: readonly ReplayEvent[]) { + const prefix = `{"directory":${JSON.stringify(directory)},"events":[` + const suffix = "]}" + const prefixBytes = Buffer.byteLength(prefix) + const suffixBytes = Buffer.byteLength(suffix) + const selected = events.reduce<{ + events: ReplayEvent[] + encoded: string[] + dataBytes: number + contentBytes: number + closed: boolean + }>( + (result, event) => { + if (result.closed) return result + const encoded = JSON.stringify(event) + const dataBytes = result.dataBytes + Buffer.byteLength(JSON.stringify(event.data)) + const contentBytes = result.contentBytes + Buffer.byteLength(encoded) + (result.encoded.length ? 1 : 0) + const eventsBytes = contentBytes + 2 + const requestBytes = prefixBytes + contentBytes + suffixBytes + if ( + dataBytes > SyncReplayLimits.eventDataBytes || + eventsBytes > SyncReplayLimits.eventsBytes || + requestBytes > SyncReplayLimits.requestBytes + ) + return { ...result, closed: true } + return { + events: [...result.events, event], + encoded: [...result.encoded, encoded], + dataBytes, + contentBytes, + closed: false, + } + }, + { events: [], encoded: [], dataBytes: 0, contentBytes: 0, closed: false }, + ) + const eventsBytes = selected.contentBytes + 2 + const requestBytes = prefixBytes + selected.contentBytes + suffixBytes + return { + events: selected.events, + json: prefix + selected.encoded.join(",") + suffix, + dataBytes: selected.dataBytes, + eventsBytes, + requestBytes, + complete: selected.events.length === events.length, + } +} diff --git a/packages/deepagent-code/test/control-plane/workspace.test.ts b/packages/deepagent-code/test/control-plane/workspace.test.ts index 6386ddca5..ca13c674b 100644 --- a/packages/deepagent-code/test/control-plane/workspace.test.ts +++ b/packages/deepagent-code/test/control-plane/workspace.test.ts @@ -4,7 +4,7 @@ import fs from "node:fs/promises" import Http from "node:http" import path from "node:path" import { NodeHttpServer } from "@effect/platform-node" -import { Effect, Exit, Fiber, Layer, Schema } from "effect" +import { Cause, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect" import { FetchHttpClient, HttpServer, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" import { eq } from "drizzle-orm" import { FSUtil } from "@deepagent-code/core/fs-util" @@ -18,6 +18,7 @@ import { Session as SessionNs } from "@/session/session" import { SessionID } from "@/session/schema" import { SessionTable } from "@deepagent-code/core/session/sql" import { EventSequenceTable } from "@deepagent-code/core/event/sql" +import { SessionToolRequestReceiptTable } from "@/session/tool-request-receipt.sql" import { resetDatabase } from "../fixture/db" import { disposeAllInstances, provideTmpdirInstance, requireInstance, TestInstance } from "../fixture/fixture" import { testEffect } from "../lib/effect" @@ -867,6 +868,72 @@ describe("workspace CRUD", () => { { git: true }, ) + it.instance( + "sessionWarp rejects unresolved provider recovery before ownership or workspace changes", + () => { + return Effect.gen(function* () { + const { directory: dir } = yield* TestInstance + const instance = yield* requireInstance + const workspace = yield* Workspace.Service + const sessionSvc = yield* SessionNs.Service + const previousType = unique("warp-recovery-prev") + const targetType = unique("warp-recovery-target") + const previous = workspaceInfo(instance.project.id, previousType) + const target = workspaceInfo(instance.project.id, targetType) + yield* insertWorkspace(previous) + yield* insertWorkspace(target) + const previousAdapter = localAdapter(path.join(dir, "warp-recovery-prev")) + const targetAdapter = localAdapter(path.join(dir, "warp-recovery-target")) + registerAdapter(instance.project.id, previousType, previousAdapter.adapter) + registerAdapter(instance.project.id, targetType, targetAdapter.adapter) + const session = yield* sessionSvc.create({}) + yield* attachSessionToWorkspace(session.id, previous.id) + const { db } = yield* Database.Service + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: "receipt-warp-recovery-required", + request_ordinal: 1, + session_id: session.id, + user_message_id: "message-user", + assistant_message_id: "message-assistant", + provider_attempt_id: null, + provider_id: "test", + model_id: "test", + registry_tool_ids: [], + permission_filtered_tool_ids: [], + final_offered_tool_ids: [], + call_ids: [], + provider_state: "indeterminate_after_crash", + terminal_at: Date.now(), + request_state: "dispatched", + created_at: Date.now(), + }) + .run() + .pipe(Effect.orDie) + const owner = yield* sessionSequenceOwner(session.id) + + const error = yield* workspace + .sessionWarp({ workspaceID: target.id, sessionID: session.id, copyChanges: true }) + .pipe(Effect.flip) + + expect(error).toBeInstanceOf(Workspace.SessionWarpRecoveryProjectionError) + expect( + (yield* db + .select({ workspaceID: SessionTable.workspace_id }) + .from(SessionTable) + .where(eq(SessionTable.id, session.id)) + .get() + .pipe(Effect.orDie))?.workspaceID, + ).toBe(previous.id) + expect(yield* sessionSequenceOwner(session.id)).toBe(owner) + expect(previousAdapter.calls.target).toEqual([]) + expect(targetAdapter.calls.target).toEqual([]) + }) + }, + { git: true }, + ) + it.instance( "sessionWarp applies source workspace patch to local target workspace", () => { @@ -995,6 +1062,8 @@ describe("workspace CRUD", () => { } calls.push(call) if (call.url.pathname === "/warp-source/sync/history") { + if ((call.json as Record)[historySessionID!] >= historyNextSeq) + return yield* HttpServerResponse.json([]) return yield* HttpServerResponse.json([ { id: `evt_${unique("warp-source-history")}`, @@ -1038,15 +1107,17 @@ describe("workspace CRUD", () => { yield* workspace.sessionWarp({ workspaceID: target.id, sessionID: session.id, copyChanges: true }) expect(calls.map((call) => `${call.method} ${call.url.pathname}`)).toEqual([ - "POST /warp-source/sync/history", "GET /warp-source/vcs/diff/raw", "POST /warp-target/vcs/apply", + "POST /warp-source/sync/history", + "POST /warp-source/sync/history", "POST /warp-target/sync/replay", "POST /warp-target/sync/steal", ]) - expect(calls[0].json).toEqual({ [session.id]: historyNextSeq - 1 }) - expect(calls[2].json).toEqual({ patch: "remote patch" }) - expect(calls[3].json).toMatchObject({ + expect(calls[1].json).toEqual({ patch: "remote patch" }) + expect(calls[2].json).toEqual({ [session.id]: historyNextSeq - 1 }) + expect(calls[3].json).toEqual({ [session.id]: historyNextSeq }) + expect(calls[4].json).toMatchObject({ directory: "remote-target-dir", events: [ { @@ -1061,7 +1132,7 @@ describe("workspace CRUD", () => { }, ], }) - expect(calls[4].json).toEqual({ sessionID: session.id }) + expect(calls[5].json).toEqual({ sessionID: session.id }) expect((yield* sessionSvc.get(session.id)).title).toBe("from source history") expect(yield* sessionSequenceOwner(session.id)).toBe(target.id) }), @@ -1069,6 +1140,73 @@ describe("workspace CRUD", () => { ) }) }) + + it.live("sessionWarp rejects an oversized chunked source patch before side effects or ownership changes", () => { + const calls: FetchCall[] = [] + return Effect.gen(function* () { + yield* HttpServer.serveEffect()( + Effect.gen(function* () { + const req = yield* HttpServerRequest.HttpServerRequest + const bodyText = yield* req.text + const call = { + url: new URL(req.url, "http://localhost"), + method: req.method, + headers: new Headers(req.headers), + bodyText, + json: bodyText ? JSON.parse(bodyText) : undefined, + } + calls.push(call) + if (call.url.pathname === "/raw-limit-source/vcs/diff/raw") + return HttpServerResponse.stream( + Stream.make( + new Uint8Array(Vcs.RawDiffLimits.patchBytes), + new Uint8Array([120]), + ), + { contentType: "text/x-diff" }, + ) + return HttpServerResponse.text("unexpected", { status: 500 }) + }), + ) + const url = yield* serverUrl() + yield* provideTmpdirInstance( + () => + Effect.gen(function* () { + const workspace = yield* Workspace.Service + const sessionSvc = yield* SessionNs.Service + const instance = yield* requireInstance + const previousType = unique("raw-limit-remote-source") + const targetType = unique("raw-limit-remote-target") + const previous = workspaceInfo(instance.project.id, previousType) + const target = workspaceInfo(instance.project.id, targetType, { directory: "remote-target-dir" }) + yield* insertWorkspace(previous) + yield* insertWorkspace(target) + registerAdapter(instance.project.id, previousType, remoteAdapter(`${url}/raw-limit-source`).adapter) + registerAdapter(instance.project.id, targetType, remoteAdapter(`${url}/raw-limit-target`).adapter) + const session = yield* sessionSvc.create({}) + yield* attachSessionToWorkspace(session.id, previous.id) + const owner = yield* sessionSequenceOwner(session.id) + + const exit = yield* workspace + .sessionWarp({ workspaceID: target.id, sessionID: session.id, copyChanges: true }) + .pipe(Effect.exit) + + expect(exit._tag).toBe("Failure") + if (exit._tag === "Failure") + expect(Cause.squash(exit.cause)).toMatchObject({ + _tag: "VcsRawDiffError", + reason: "remote-output", + limit: Vcs.RawDiffLimits.patchBytes, + }) + expect(calls.map((call) => `${call.method} ${call.url.pathname}`)).toEqual([ + "GET /raw-limit-source/vcs/diff/raw", + ]) + expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(previous.id) + expect(yield* sessionSequenceOwner(session.id)).toBe(owner) + }), + { git: true }, + ) + }) + }) }) describe("workspace sync state", () => { @@ -1357,7 +1495,10 @@ describe("workspace sync state", () => { const url = new URL(req.url, "http://localhost") if (url.pathname === "/history/global/event") return HttpServerResponse.fromWeb(eventStreamResponse()) if (url.pathname === "/history/sync/history") { - historyBodies.push(bodyText ? JSON.parse(bodyText) : undefined) + const body = bodyText ? JSON.parse(bodyText) : undefined + historyBodies.push(body) + if (body?.[historySessionID!] === historyNextSeq) + return HttpServerResponse.fromWeb(Response.json([])) return HttpServerResponse.fromWeb( Response.json([ { @@ -1399,7 +1540,10 @@ describe("workspace sync state", () => { expect((yield* sessionSvc.get(session.id).pipe(Effect.orDie)).title).toBe("from history") }), ) - expect(historyBodies).toEqual([{ [session.id]: historyNextSeq - 1 }]) + expect(historyBodies).toEqual([ + { [session.id]: historyNextSeq - 1 }, + { [session.id]: historyNextSeq }, + ]) expect( captured.events.some( (event) => diff --git a/packages/deepagent-code/test/project/vcs.test.ts b/packages/deepagent-code/test/project/vcs.test.ts index b10633c44..29224cfbd 100644 --- a/packages/deepagent-code/test/project/vcs.test.ts +++ b/packages/deepagent-code/test/project/vcs.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect } from "bun:test" import { FSUtil } from "@deepagent-code/core/fs-util" import { parsePatch } from "diff" -import { Deferred, Effect, Layer } from "effect" +import { Cause, Deferred, Effect, Layer } from "effect" import { CrossSpawnSpawner } from "@deepagent-code/core/cross-spawn-spawner" import fs from "fs/promises" import path from "path" @@ -333,4 +333,110 @@ describe("Vcs diff", () => { }), { git: true }, ) + + it.instance( + "diffRaw returns a complete patch for tracked and untracked files", + () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* write(path.join(test.directory, "tracked.txt"), "before\n") + yield* git(test.directory, ["add", "tracked.txt"]) + yield* git(test.directory, ["commit", "--no-gpg-sign", "-m", "add tracked"]) + yield* write(path.join(test.directory, "tracked.txt"), "after\n") + yield* write(path.join(test.directory, "untracked.txt"), "new\n") + + const patch = yield* (yield* init()).diffRaw() + + expect(patch).toContain("diff --git a/tracked.txt b/tracked.txt") + expect(patch).toContain("diff --git a/untracked.txt b/untracked.txt") + expect(Buffer.byteLength(patch)).toBeLessThanOrEqual(Vcs.RawDiffLimits.patchBytes) + }), + { git: true }, + ) + + it.instance( + "diffRaw includes staged files before the repository has its first commit", + () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* write(path.join(test.directory, "staged.txt"), "staged\n") + yield* git(test.directory, ["add", "staged.txt"]) + + const patch = yield* (yield* init()).diffRaw() + + expect(patch).toContain("diff --git a/staged.txt b/staged.txt") + expect(patch).toContain("+staged") + }), + { git: true }, + ) + + it.instance( + "diffRaw fails closed when a tracked patch exceeds the UTF-8 budget", + () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* write(path.join(test.directory, "tracked.txt"), "before\n") + yield* git(test.directory, ["add", "tracked.txt"]) + yield* git(test.directory, ["commit", "--no-gpg-sign", "-m", "add tracked"]) + yield* write(path.join(test.directory, "tracked.txt"), "x".repeat(Vcs.RawDiffLimits.patchBytes + 1)) + + const exit = yield* (yield* init()).diffRaw().pipe(Effect.exit) + + expect(exit._tag).toBe("Failure") + if (exit._tag === "Failure") + expect(Cause.squash(exit.cause)).toMatchObject({ + _tag: "VcsRawDiffError", + reason: "tracked-output", + limit: Vcs.RawDiffLimits.patchBytes, + }) + }), + { git: true }, + 20_000, + ) + + it.instance( + "diffRaw fails closed when an untracked patch exceeds the UTF-8 budget", + () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* write(path.join(test.directory, "untracked.txt"), "x".repeat(Vcs.RawDiffLimits.patchBytes + 1)) + + const exit = yield* (yield* init()).diffRaw().pipe(Effect.exit) + + expect(exit._tag).toBe("Failure") + if (exit._tag === "Failure") + expect(Cause.squash(exit.cause)).toMatchObject({ + _tag: "VcsRawDiffError", + reason: "untracked-output", + }) + }), + { git: true }, + 20_000, + ) + + it.instance( + "diffRaw fails closed before patch generation when candidate files exceed the limit", + () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* Effect.forEach( + Array.from({ length: Vcs.RawDiffLimits.candidateFiles + 1 }, (_, index) => index), + (index) => write(path.join(test.directory, `candidate-${index}.txt`), "x\n"), + { concurrency: 32, discard: true }, + ) + + const exit = yield* (yield* init()).diffRaw().pipe(Effect.exit) + + expect(exit._tag).toBe("Failure") + if (exit._tag === "Failure") + expect(Cause.squash(exit.cause)).toMatchObject({ + _tag: "VcsRawDiffError", + reason: "candidate-files", + limit: Vcs.RawDiffLimits.candidateFiles, + actual: Vcs.RawDiffLimits.candidateFiles + 1, + }) + }), + { git: true }, + 20_000, + ) }) diff --git a/packages/deepagent-code/test/server/httpapi-instance.test.ts b/packages/deepagent-code/test/server/httpapi-instance.test.ts index dc38e537d..6b078cef4 100644 --- a/packages/deepagent-code/test/server/httpapi-instance.test.ts +++ b/packages/deepagent-code/test/server/httpapi-instance.test.ts @@ -9,6 +9,7 @@ import { WorkspaceV2 } from "@deepagent-code/core/workspace" import { ControlPaths } from "../../src/server/routes/instance/httpapi/groups/control" import { InstancePaths } from "../../src/server/routes/instance/httpapi/groups/instance" import { SessionPaths } from "../../src/server/routes/instance/httpapi/groups/session" +import { Vcs } from "../../src/project/vcs" import { DeepAgentGoalPlanBusyError, DeepAgentGoalPlanConflictError, @@ -450,4 +451,26 @@ describe("instance HttpApi", () => { ) }), ) + + it.live("maps an oversized raw VCS patch to the typed 503 response", () => + Effect.gen(function* () { + const dir = yield* tmpdirScoped({ git: true }) + const fs = yield* FileSystem.FileSystem + const path = yield* Path.Path + yield* fs.writeFileString(path.join(dir, "oversized.txt"), "x".repeat(Vcs.RawDiffLimits.patchBytes + 1)) + + const response = yield* HttpClientRequest.get(InstancePaths.vcsDiffRaw).pipe( + directoryHeader(dir), + HttpClient.execute, + ) + + expect(response.status).toBe(503) + expect(yield* response.json).toMatchObject({ + name: "VcsRawDiffError", + data: { + reason: "untracked-output", + }, + }) + }), + ) }) diff --git a/packages/deepagent-code/test/server/httpapi-provider-recovery.test.ts b/packages/deepagent-code/test/server/httpapi-provider-recovery.test.ts new file mode 100644 index 000000000..6310c07ce --- /dev/null +++ b/packages/deepagent-code/test/server/httpapi-provider-recovery.test.ts @@ -0,0 +1,254 @@ +import { afterEach, describe, expect } from "bun:test" +import { Database } from "@deepagent-code/core/database/database" +import { Flag } from "@deepagent-code/core/flag/flag" +import { Hash } from "@deepagent-code/core/util/hash" +import { ModelV2 } from "@deepagent-code/core/model" +import { ProviderV2 } from "@deepagent-code/core/provider" +import { WorkspaceV2 } from "@deepagent-code/core/workspace" +import { SessionHistoryStateTable, SessionTable } from "@deepagent-code/core/session/sql" +import { and, eq } from "drizzle-orm" +import { Effect, Layer } from "effect" +import { MessageV2 } from "@/session/message-v2" +import { SessionPromptEpochTable } from "@/session/prompt-epoch.sql" +import { MessageID, PartID } from "@/session/schema" +import { Session } from "@/session/session" +import { SessionToolRequestReceiptTable } from "@/session/tool-request-receipt.sql" +import { SessionPaths } from "@/server/routes/instance/httpapi/groups/session" +import { resetDatabase } from "../fixture/db" +import { disposeAllInstances, TestInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" +import { httpApiLayer, requestInDirectory } from "./httpapi-layer" + +const originalWorkspaceID = Flag.DEEPAGENT_CODE_WORKSPACE_ID +const it = testEffect(Layer.mergeAll(Session.defaultLayer, Database.defaultLayer, httpApiLayer)) +const model = { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") } + +type Descriptor = { + receiptID: string + providerState: "indeterminate_after_crash" + promptEpoch: number + requestHash: string + historyHash: string + worldStateBaselineHash?: string + sessionMutationEpoch: number + workspaceRecoverySupported: boolean +} + +const pathFor = (sessionID: string) => SessionPaths.providerResolution.replace(":sessionID", sessionID) + +const parseJson = (response: { readonly json: Effect.Effect }) => + response.json.pipe(Effect.map((body) => body as A)) + +const seedRecovery = Effect.fn("HttpProviderRecoveryTest.seedRecovery")(function* (title: string) { + const sessions = yield* Session.Service + const { db } = yield* Database.Service + const parent = yield* sessions.create({ title: `${title} parent` }) + const session = yield* sessions.fork({ sessionID: parent.id, intentID: `fork-${Hash.sha256(title).slice(0, 24)}` }) + const user = yield* sessions.updateMessage({ + id: MessageID.ascending(), + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: "build", + model, + }) + yield* sessions.updatePart({ + id: PartID.ascending(), + sessionID: session.id, + messageID: user.id, + type: "text", + text: "dispatch this provider request", + }) + const assistant = yield* sessions.updateMessage({ + id: MessageID.ascending(), + sessionID: session.id, + role: "assistant", + parentID: user.id, + time: { created: Date.now() }, + mode: "build", + agent: "build", + cost: 0, + path: { cwd: session.directory, root: session.directory }, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + modelID: model.modelID, + providerID: model.providerID, + }) + const authority = yield* MessageV2.promptHistoryProjectionEffect(session.id) + if (!authority.worldStateBaselineHash) return yield* Effect.die("expected frozen source World State baseline") + const now = Date.now() + const receiptID = `receipt-${Hash.sha256(title).slice(0, 24)}` + const requestHash = `request-${Hash.sha256(title).slice(0, 24)}` + + yield* db + .update(SessionPromptEpochTable) + .set({ + authority_state: "recovery_required", + recovery_reason: "provider outcome unknown after restart", + }) + .where(and(eq(SessionPromptEpochTable.session_id, session.id), eq(SessionPromptEpochTable.epoch, authority.epoch))) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: receiptID, + request_ordinal: 1, + session_id: session.id, + user_message_id: user.id, + assistant_message_id: assistant.id, + provider_attempt_id: null, + provider_id: model.providerID, + model_id: model.modelID, + protocol: "chat", + registry_tool_ids: [], + permission_filtered_tool_ids: [], + final_offered_tool_ids: [], + call_ids: [], + prompt_epoch: authority.epoch, + prompt_window_id: authority.window.windowID, + effective_history_hash: authority.effectiveHistoryHash, + request_input_hash: `input-${requestHash}`, + final_request_hash: requestHash, + provider_state: "indeterminate_after_crash", + adapter_prepared_at: now, + dispatching_at: now, + terminal_at: now, + owner_token: "dead-process", + request_state: "dispatched", + request_error_code: "provider_started_outcome_unknown_after_process_restart", + created_at: now, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(SessionHistoryStateTable) + .set({ + state: "recovery_required", + reason: "provider outcome unknown after restart", + time_updated: now, + }) + .where(eq(SessionHistoryStateTable.session_id, session.id)) + .run() + .pipe(Effect.orDie) + + return { session, receiptID } +}) + +const command = (descriptor: Descriptor, commandID: string) => ({ + commandID, + receiptID: descriptor.receiptID, + decision: "abandoned" as const, + expected: { + providerState: descriptor.providerState, + promptEpoch: descriptor.promptEpoch, + sessionMutationEpoch: descriptor.sessionMutationEpoch, + requestHash: descriptor.requestHash, + historyHash: descriptor.historyHash, + worldStateBaselineHash: descriptor.worldStateBaselineHash!, + }, +}) + +afterEach(async () => { + Flag.DEEPAGENT_CODE_WORKSPACE_ID = originalWorkspaceID + await disposeAllInstances() + await resetDatabase() +}) + +describe("provider recovery HttpApi", () => { + it.instance( + "enforces GET authority, exact retry, command conflict, stale CAS, and workspace ownership", + () => + Effect.gen(function* () { + const instance = yield* TestInstance + const { db } = yield* Database.Service + const headers = { "content-type": "application/json" } + + const exact = yield* seedRecovery("http exact retry") + const listed = yield* requestInDirectory(pathFor(exact.session.id), instance.directory, { headers }) + expect(listed.status).toBe(200) + const [descriptor] = yield* parseJson(listed) + expect(descriptor).toMatchObject({ + receiptID: exact.receiptID, + providerState: "indeterminate_after_crash", + workspaceRecoverySupported: true, + }) + expect(descriptor?.worldStateBaselineHash).toStartWith("wsb1_") + if (!descriptor) return + const exactCommand = command(descriptor, "http-provider-recovery-exact") + const first = yield* requestInDirectory(pathFor(exact.session.id), instance.directory, { + method: "POST", + headers, + body: JSON.stringify(exactCommand), + }) + expect(first.status).toBe(200) + const firstResolution = yield* parseJson>(first) + const retry = yield* requestInDirectory(pathFor(exact.session.id), instance.directory, { + method: "POST", + headers, + body: JSON.stringify(exactCommand), + }) + expect(retry.status).toBe(200) + expect(yield* parseJson>(retry)).toEqual(firstResolution) + + const conflict = yield* requestInDirectory(pathFor(exact.session.id), instance.directory, { + method: "POST", + headers, + body: JSON.stringify({ ...exactCommand, reason: "reuse the command ID with different authority" }), + }) + expect(conflict.status).toBe(409) + expect(yield* parseJson>(conflict)).toMatchObject({ + _tag: "ConflictError", + resource: "command_id_conflict", + }) + const cleared = yield* requestInDirectory(pathFor(exact.session.id), instance.directory, { headers }) + expect(cleared.status).toBe(200) + expect(yield* parseJson(cleared)).toEqual([]) + + const stale = yield* seedRecovery("http stale cas") + const staleList = yield* requestInDirectory(pathFor(stale.session.id), instance.directory, { headers }) + const [staleDescriptor] = yield* parseJson(staleList) + if (!staleDescriptor) return + yield* db + .update(SessionTable) + .set({ mutation_epoch: staleDescriptor.sessionMutationEpoch + 1 }) + .where(eq(SessionTable.id, stale.session.id)) + .run() + .pipe(Effect.orDie) + const staleResponse = yield* requestInDirectory(pathFor(stale.session.id), instance.directory, { + method: "POST", + headers, + body: JSON.stringify(command(staleDescriptor, "http-provider-recovery-stale")), + }) + expect(staleResponse.status).toBe(409) + expect(yield* parseJson>(staleResponse)).toMatchObject({ + _tag: "ConflictError", + resource: "stale_session_mutation", + }) + + const workspace = yield* seedRecovery("http workspace unsupported") + const workspaceList = yield* requestInDirectory(pathFor(workspace.session.id), instance.directory, { headers }) + const [workspaceDescriptor] = yield* parseJson(workspaceList) + if (!workspaceDescriptor) return + const workspaceID = WorkspaceV2.ID.make("wrk_00000000000000000000000000") + Flag.DEEPAGENT_CODE_WORKSPACE_ID = workspaceID + yield* db + .update(SessionTable) + .set({ workspace_id: workspaceID }) + .where(eq(SessionTable.id, workspace.session.id)) + .run() + .pipe(Effect.orDie) + const workspaceResponse = yield* requestInDirectory(pathFor(workspace.session.id), instance.directory, { + method: "POST", + headers, + body: JSON.stringify(command(workspaceDescriptor, "http-provider-recovery-workspace")), + }) + expect(workspaceResponse.status).toBe(409) + expect(yield* parseJson>(workspaceResponse)).toMatchObject({ + _tag: "ConflictError", + resource: "workspace_recovery_not_supported", + }) + }), + { git: true, config: { formatter: false, lsp: false } }, + 30_000, + ) +}) diff --git a/packages/deepagent-code/test/server/httpapi-session.test.ts b/packages/deepagent-code/test/server/httpapi-session.test.ts index 6db15813f..39aa80e5b 100644 --- a/packages/deepagent-code/test/server/httpapi-session.test.ts +++ b/packages/deepagent-code/test/server/httpapi-session.test.ts @@ -383,7 +383,7 @@ describe("session HttpApi", () => { (yield* request(`${pathFor(SessionPaths.messages, { sessionID: parent.id })}?before=${nextCursor}`, { headers, })).status, - ).toBe(400) + ).toBe(200) expect( (yield* request(`${pathFor(SessionPaths.messages, { sessionID: parent.id })}?limit=1&before=invalid`, { headers, @@ -848,7 +848,7 @@ describe("session HttpApi", () => { }) expect(response.status).toBe(200) - expect((yield* json(response)).summary?.diffs).toEqual([{ additions: 1, deletions: 0 }]) + expect((yield* json(response)).summary?.diffs).toBeUndefined() }), { git: true, config: { formatter: false, lsp: false } }, ) diff --git a/packages/deepagent-code/test/server/httpapi-sync.test.ts b/packages/deepagent-code/test/server/httpapi-sync.test.ts index de5ac6821..205d28e8f 100644 --- a/packages/deepagent-code/test/server/httpapi-sync.test.ts +++ b/packages/deepagent-code/test/server/httpapi-sync.test.ts @@ -1,20 +1,32 @@ import { afterEach, describe, expect, mock, spyOn } from "bun:test" import { Context, Effect, Layer } from "effect" import { Flag } from "@deepagent-code/core/flag/flag" -import { SyncPaths } from "../../src/server/routes/instance/httpapi/groups/sync" +import { SyncPaths, SyncReplayLimits } from "../../src/server/routes/instance/httpapi/groups/sync" import { HttpApiApp } from "../../src/server/routes/instance/httpapi/server" import { Session } from "@/session/session" +import { MessageV2 } from "@/session/message-v2" +import { MessageID } from "@/session/schema" +import { ProviderV2 } from "@deepagent-code/core/provider" +import { ModelV2 } from "@deepagent-code/core/model" import * as Log from "@deepagent-code/core/util/log" import { resetDatabase } from "../fixture/db" import { disposeAllInstances, TestInstance } from "../fixture/fixture" import { testEffect } from "../lib/effect" import { httpApiLayer, requestInDirectory } from "./httpapi-layer" +import { SyncHistoryLimits } from "@/server/routes/instance/httpapi/handlers/sync" +import { Database } from "@deepagent-code/core/database/database" +import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { EventV2 } from "@deepagent-code/core/event" +import { eq } from "drizzle-orm" +import { HttpServer } from "effect/unstable/http" +import { SessionV1 } from "@deepagent-code/core/v1/session" +import { encodeReplayRequestPrefix } from "@/sync/replay-protocol" void Log.init({ print: false }) const originalWorkspaces = Flag.DEEPAGENT_CODE_EXPERIMENTAL_WORKSPACES const context = Context.empty() as Context.Context -const it = testEffect(Layer.mergeAll(Session.defaultLayer, httpApiLayer)) +const it = testEffect(Layer.mergeAll(Session.defaultLayer, Database.defaultLayer, httpApiLayer)) afterEach(async () => { mock.restore() @@ -75,6 +87,461 @@ describe("sync HttpApi", () => { expect(info.mock.calls.some(([message]) => message === "sync replay complete")).toBe(true) }), { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "bounds legacy message.updated rows in sync history", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "sync-legacy-diff" }) + yield* Session.use.updateMessage({ + id: MessageID.ascending(), + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + summary: { + diffs: Array.from({ length: MessageV2.ClientDiffLimits.files + 5 }, (_, index) => ({ + file: `legacy-${index}.ts`, + patch: "x", + additions: 1, + deletions: 0, + status: "modified" as const, + })), + }, + }) + + const response = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({}), + }) + const rows = (yield* response.json) as Array<{ type: string; data: Record }> + const event = rows.find((row) => row.type === "message.updated.1") + const info = event?.data.info as { summary?: { diffs?: Array<{ patch?: string }> } } | undefined + expect(info?.summary?.diffs).toHaveLength(MessageV2.ClientDiffLimits.files) + expect(info?.summary?.diffs?.every((item) => item.patch === undefined)).toBe(true) + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "pages bounded sync history until the aggregate cursor catches up", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "sync-pages" }) + const extra = SyncHistoryLimits.events + 5 + yield* Effect.forEach( + Array.from({ length: extra }, (_, index) => index), + (index) => + Session.use.updateMessage({ + id: MessageID.ascending(), + sessionID: session.id, + role: "user", + time: { created: Date.now() + index }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + }), + { discard: true }, + ) + + const first = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({}), + }) + const firstRows = (yield* first.json) as Array<{ id: string; aggregate_id: string; seq: number }> + expect(firstRows).toHaveLength(SyncHistoryLimits.events) + const state = Object.fromEntries(firstRows.map((row) => [row.aggregate_id, row.seq])) + const second = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify(state), + }) + const secondRows = (yield* second.json) as Array<{ id: string; aggregate_id: string; seq: number }> + expect(secondRows.length).toBeGreaterThan(0) + expect(secondRows.length).toBeLessThanOrEqual(SyncHistoryLimits.events) + expect(new Set([...firstRows, ...secondRows].map((row) => row.id)).size).toBe( + firstRows.length + secondRows.length, + ) + secondRows.forEach((row) => (state[row.aggregate_id] = row.seq)) + const done = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify(state), + }) + expect(yield* done.json).toEqual([]) + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "fails closed before decoding an oversized legacy sync event", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "sync-legacy-oversized" }) + const { db } = yield* Database.Service + const sequence = yield* db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + yield* db + .insert(EventTable) + .values({ + id: EventV2.ID.make("evt_sync_legacy_oversized"), + aggregate_id: session.id, + seq: sequence!.seq + 1, + type: "sync.legacy.1", + data: { value: "x".repeat(SyncHistoryLimits.bytes + 1) }, + }) + .run() + .pipe(Effect.orDie) + + const response = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ [session.id]: sequence!.seq }), + }) + const body = yield* response.text + expect(response.status, body).toBe(503) + expect(body).toContain("requires artifact migration") + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "returns a contiguous prefix before an oversized legacy sync event", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "sync-prefix" }) + const { db } = yield* Database.Service + const sequence = yield* db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + yield* db + .insert(EventTable) + .values({ + id: EventV2.ID.make("evt_sync_legacy_oversized_after_prefix"), + aggregate_id: session.id, + seq: sequence!.seq + 1, + type: "sync.legacy.1", + data: { value: "x".repeat(SyncHistoryLimits.bytes + 1) }, + }) + .run() + .pipe(Effect.orDie) + + const first = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({}), + }) + const rows = (yield* first.json) as Array<{ aggregate_id: string; seq: number }> + expect(first.status).toBe(200) + expect(rows).toHaveLength(1) + expect(rows[0]?.aggregate_id).toBe(session.id) + + const blocked = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ [session.id]: rows[0]!.seq }), + }) + const body = yield* blocked.text + expect(blocked.status, body).toBe(503) + expect(body).toContain("requires artifact migration") + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "never crosses a UTF-8 event that exceeds the serialized page budget", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "sync-wire-prefix" }) + const { db } = yield* Database.Service + const sequence = yield* db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + expect(sequence).toBeDefined() + const target = SyncHistoryLimits.bytes - 100 + const overhead = Buffer.byteLength(JSON.stringify({ value: "" })) + const available = target - overhead + const unicode = "界".repeat(Math.floor(available / 3)) + "x".repeat(available % 3) + const middle = { value: unicode } + expect(Buffer.byteLength(JSON.stringify(middle))).toBe(target) + yield* db + .insert(EventTable) + .values([ + { + id: EventV2.ID.make("evt_sync_wire_small_before"), + aggregate_id: session.id, + seq: sequence!.seq + 1, + type: "sync.test.1", + data: { value: "before" }, + }, + { + id: EventV2.ID.make("evt_sync_wire_middle"), + aggregate_id: session.id, + seq: sequence!.seq + 2, + type: "sync.test.1", + data: middle, + }, + { + id: EventV2.ID.make("evt_sync_wire_small_after"), + aggregate_id: session.id, + seq: sequence!.seq + 3, + type: "sync.test.1", + data: { value: "after" }, + }, + ]) + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ seq: sequence!.seq + 3 }) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .run() + .pipe(Effect.orDie) + + const first = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ [session.id]: sequence!.seq }), + }) + const rows = (yield* first.json) as Array<{ id: string; seq: number }> + expect(first.status).toBe(200) + expect(rows.map((row) => row.id)).toEqual(["evt_sync_wire_small_before"]) + + const blocked = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ [session.id]: sequence!.seq + 1 }), + }) + const body = yield* blocked.text + expect(blocked.status, body).toBe(503) + expect(body).toContain("Projected sync event evt_sync_wire_middle exceeds") + expect(body).not.toContain("evt_sync_wire_small_after") + }), + { git: true, config: { formatter: false, lsp: false } }, + 20_000, + ) + + it.instance( + "rejects replay batches above the event and byte budgets before replay", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const event = (index: number, data: Record = {}) => ({ + id: EventV2.ID.make(`evt_sync_replay_limit_${index}`), + aggregateID: "aggregate-sync-replay-limit", + seq: index, + type: "sync.replay.limit.1", + data, + }) + + const tooMany = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ + directory: tmp.directory, + events: Array.from({ length: SyncReplayLimits.events + 1 }, (_, index) => event(index)), + }), + }) + expect(tooMany.status).toBe(400) + + const oversized = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ + directory: tmp.directory, + events: [event(0, { value: "x".repeat(SyncReplayLimits.requestBytes) })], + }), + }) + expect(oversized.status).toBe(413) + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "replays a maximum admitted event through Web and the real Node listener", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const session = yield* Session.use.create({ title: "sync-max-replay" }) + const event = (aggregateID: string, id: EventV2.ID, seq: number) => { + const base = { + sessionID: aggregateID, + info: { + id: MessageID.ascending(), + sessionID: aggregateID, + role: "user" as const, + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + system: "", + }, + } + const remaining = EventV2.MAX_ENCODED_PAYLOAD_BYTES - Buffer.byteLength(JSON.stringify(base)) + const data = { ...base, info: { ...base.info, system: "x".repeat(remaining) } } + expect(Buffer.byteLength(JSON.stringify(data))).toBe(EventV2.MAX_ENCODED_PAYLOAD_BYTES) + return { + id, + aggregateID, + seq, + type: EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1), + data, + } + } + const { db } = yield* Database.Service + const webSequence = yield* db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + const webEvent = event(session.id, EventV2.ID.make("evt_sync_max_web"), webSequence!.seq + 1) + const webBody = encodeReplayRequestPrefix(tmp.directory, [webEvent]) + expect(webBody.complete).toBe(true) + expect(webBody.dataBytes).toBe(SyncReplayLimits.eventDataBytes) + expect(webBody.requestBytes).toBeGreaterThan(SyncReplayLimits.eventDataBytes) + expect(webBody.requestBytes).toBeLessThanOrEqual(SyncReplayLimits.requestBytes) + const web = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { + method: "POST", + headers: { "content-type": "application/json" }, + body: webBody.json, + }) + expect(web.status, yield* web.text).toBe(200) + + const nodeSession = yield* Session.use.create({ title: "sync-max-node" }) + const nodeSequence = yield* db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, nodeSession.id)) + .get() + .pipe(Effect.orDie) + const nodeEvent = event(nodeSession.id, EventV2.ID.make("evt_sync_max_node"), nodeSequence!.seq + 1) + const nodeBody = encodeReplayRequestPrefix(tmp.directory, [nodeEvent]) + const server = yield* HttpServer.HttpServer + const node = yield* Effect.promise(() => + fetch(`${HttpServer.formatAddress(server.address)}${SyncPaths.replay}`, { + method: "POST", + headers: { + "x-deepagent-code-directory": tmp.directory, + "content-type": "application/json", + }, + body: nodeBody.json, + }), + ) + expect(node.status, yield* Effect.promise(() => node.text())).toBe(200) + }), + { git: true, config: { formatter: false, lsp: false } }, + 30_000, + ) + + it.instance( + "rejects an unbounded replay stream before reading or decoding the full body", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const { db } = yield* Database.Service + const before = yield* db.select({ id: EventTable.id }).from(EventTable).all().pipe(Effect.orDie) + const chunk = new TextEncoder().encode("x".repeat(256 * 1024)) + let pulls = 0 + let cancelled = false + const body = new ReadableStream({ + pull(controller) { + pulls++ + if (pulls > 1024) return controller.close() + controller.enqueue(chunk) + }, + cancel() { + cancelled = true + }, + }) + const response = yield* Effect.promise(() => + HttpApiApp.webHandler().handler( + new Request(`http://localhost${SyncPaths.replay}`, { + method: "POST", + headers: { + "x-deepagent-code-directory": tmp.directory, + "content-type": "application/json", + }, + body, + duplex: "half", + } as RequestInit), + context, + ), + ) + + expect(response.status).toBe(413) + expect(pulls).toBeLessThan(64) + expect(cancelled).toBe(true) + expect(yield* db.select({ id: EventTable.id }).from(EventTable).all().pipe(Effect.orDie)).toEqual(before) + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "enforces the replay stream budget on the real Node listener without content-length", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const { db } = yield* Database.Service + const before = yield* db.select({ id: EventTable.id }).from(EventTable).all().pipe(Effect.orDie) + const chunk = new TextEncoder().encode("x".repeat(256 * 1024)) + let pulls = 0 + const body = new ReadableStream({ + pull(controller) { + pulls++ + if (pulls > 1024) return controller.close() + controller.enqueue(chunk) + }, + }) + const server = yield* HttpServer.HttpServer + const response = yield* Effect.promise(() => + fetch(`${HttpServer.formatAddress(server.address)}${SyncPaths.replay}`, { + method: "POST", + headers: { + "x-deepagent-code-directory": tmp.directory, + "content-type": "application/json", + }, + body, + duplex: "half", + } as RequestInit), + ) + + expect(response.status).toBe(400) + expect(pulls).toBeLessThan(64) + expect(yield* db.select({ id: EventTable.id }).from(EventTable).all().pipe(Effect.orDie)).toEqual(before) + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, ) it.instance( diff --git a/packages/deepagent-code/test/server/session-diff-missing-patch.test.ts b/packages/deepagent-code/test/server/session-diff-missing-patch.test.ts index 72503d424..9ae0146cd 100644 --- a/packages/deepagent-code/test/server/session-diff-missing-patch.test.ts +++ b/packages/deepagent-code/test/server/session-diff-missing-patch.test.ts @@ -14,11 +14,15 @@ import { afterEach, describe, expect } from "bun:test" import { Effect, Layer } from "effect" import { SessionPaths } from "@/server/routes/instance/httpapi/groups/session" import { Session } from "@/session/session" +import { MessageV2 } from "@/session/message-v2" import { Storage } from "@/storage/storage" import { SessionV1 } from "@deepagent-code/core/v1/session" import { MessageID } from "@/session/schema" import { ProviderV2 } from "@deepagent-code/core/provider" import { ModelV2 } from "@deepagent-code/core/model" +import { Database } from "@deepagent-code/core/database/database" +import { MessageTable } from "@deepagent-code/core/session/sql" +import { and, eq } from "drizzle-orm" import { resetDatabase } from "../fixture/db" import { disposeAllInstances, TestInstance } from "../fixture/fixture" import { testEffect } from "../lib/effect" @@ -27,7 +31,7 @@ import { httpApiLayer, requestInDirectory } from "./httpapi-layer" void Log.init({ print: false }) -const it = testEffect(Layer.mergeAll(Session.defaultLayer, Storage.defaultLayer, httpApiLayer)) +const it = testEffect(Layer.mergeAll(Session.defaultLayer, Storage.defaultLayer, Database.defaultLayer, httpApiLayer)) afterEach(async () => { await disposeAllInstances() @@ -65,6 +69,7 @@ describe("session diff with missing patch (#26574)", () => { expect(yield* response.json).toEqual([]) }), { git: true, config: { formatter: false, lsp: false } }, + 15_000, ) it.instance( @@ -95,5 +100,106 @@ describe("session diff with missing patch (#26574)", () => { expect(yield* response.json).toEqual([{ file: "turn.ts", additions: 1, deletions: 0, status: "modified" }]) }), { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "message endpoints bound legacy inline patches without rewriting storage", + () => + Effect.gen(function* () { + const test = yield* TestInstance + const session = yield* withSession({ title: "legacy-inline-patch" }) + const messageID = MessageID.ascending() + const patch = "x".repeat(32 * 1024) + yield* Session.use.updateMessage({ + id: messageID, + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + summary: { + diffs: Array.from({ length: MessageV2.ClientDiffLimits.files + 10 }, (_, index) => ({ + file: `legacy-${index}.ts`, + patch, + additions: 1, + deletions: 0, + status: "modified" as const, + })), + }, + } satisfies SessionV1.User) + + const list = yield* requestInDirectory( + `${pathFor(SessionPaths.messages, { sessionID: session.id })}?limit=20`, + test.directory, + ) + const listed = (yield* list.json) as SessionV1.WithParts[] + const projected = listed.find((item) => item.info.id === messageID)?.info + expect(projected?.role === "user" ? projected.summary?.diffs : undefined).toHaveLength( + MessageV2.ClientDiffLimits.files, + ) + expect( + projected?.role === "user" ? projected.summary?.diffs.every((item) => item.patch === undefined) : false, + ).toBe(true) + + const { db } = yield* Database.Service + const stored = yield* db + .select({ data: MessageTable.data }) + .from(MessageTable) + .where(and(eq(MessageTable.session_id, session.id), eq(MessageTable.id, messageID))) + .get() + .pipe(Effect.orDie) + const original = + stored?.data.role === "user" && typeof stored.data.summary === "object" + ? stored.data.summary + : undefined + expect(original?.diffs).toHaveLength( + MessageV2.ClientDiffLimits.files + 10, + ) + expect(original?.diffs[0]?.patch).toBe(patch) + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "GET /session//diff strips legacy inline patch bodies", + () => + Effect.gen(function* () { + const test = yield* TestInstance + const session = yield* withSession({ title: "giant-inline-turn-diff" }) + const messageID = MessageID.ascending() + yield* Session.use.updateMessage({ + id: messageID, + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + summary: { + diffs: [ + { + file: "legacy-giant.ts", + patch: "x".repeat(2 * 1024 * 1024), + additions: 1, + deletions: 0, + status: "modified", + }, + ], + }, + } satisfies SessionV1.User) + + const response = yield* requestInDirectory( + `${pathFor(SessionPaths.diff, { sessionID: session.id })}?messageID=${messageID}`, + test.directory, + ) + + expect(response.status).toBe(200) + expect(yield* response.json).toEqual([ + { file: "legacy-giant.ts", additions: 1, deletions: 0, status: "modified" }, + ]) + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, ) }) diff --git a/packages/deepagent-code/test/server/session-messages.test.ts b/packages/deepagent-code/test/server/session-messages.test.ts index c2abca6da..644cdd6db 100644 --- a/packages/deepagent-code/test/server/session-messages.test.ts +++ b/packages/deepagent-code/test/server/session-messages.test.ts @@ -5,7 +5,7 @@ import { HttpClientResponse } from "effect/unstable/http" import { Session as SessionNs } from "@/session/session" import { MessageV2 } from "../../src/session/message-v2" -import { MessageID, PartID, type SessionID } from "../../src/session/schema" +import { MessageID, type SessionID } from "../../src/session/schema" import * as Log from "@deepagent-code/core/util/log" import { disposeAllInstances, TestInstance } from "../fixture/fixture" import { testEffect } from "../lib/effect" @@ -55,27 +55,18 @@ const fill = Effect.fn("SessionMessagesTest.fill")(function* ( const session = yield* SessionNs.Service return yield* Effect.forEach( Array.from({ length: count }, (_, i) => i), - (i) => - Effect.gen(function* () { - const id = MessageID.ascending() - yield* session.updateMessage({ - id, - sessionID, - role: "user", - time: { created: time(i) }, - agent: "test", - model, - tools: {}, - } satisfies SessionV1.User) - yield* session.updatePart({ - id: PartID.ascending(), - sessionID, - messageID: id, - type: "text", - text: `m${i}`, - } satisfies SessionV1.TextPart) - return id - }), + (i) => { + const id = MessageID.ascending() + return session.updateMessage({ + id, + sessionID, + role: "user" as const, + time: { created: time(i) }, + agent: "test", + model, + tools: {}, + }).pipe(Effect.as(id)) + }, ) }) @@ -113,19 +104,21 @@ describe("session messages endpoint", () => { ) it.instance( - "keeps full-history responses when limit is omitted", + "uses a bounded default when limit is omitted", withoutWatcher( Effect.gen(function* () { const session = yield* sessionScoped - const ids = yield* fill(session.id, 3) + const ids = yield* fill(session.id, 110) const res = yield* request(`/session/${session.id}/message`) expect(res.status).toBe(200) const body = yield* json(res) - expect(body.map((item) => item.info.id)).toEqual(ids) + expect(body.map((item) => item.info.id)).toEqual(ids.slice(-MessageV2.ClientMessageLimits.page)) + expect(res.headers["x-next-cursor"]).toBeTruthy() }), ), { git: true }, + 15_000, ) it.instance( @@ -145,16 +138,14 @@ describe("session messages endpoint", () => { ) it.instance( - "does not truncate large legacy limit requests", + "rejects message pages above the bounded server limit", withoutWatcher( Effect.gen(function* () { const session = yield* sessionScoped yield* fill(session.id, 520) const res = yield* request(`/session/${session.id}/message?limit=510`) - expect(res.status).toBe(200) - const body = yield* json(res) - expect(body).toHaveLength(510) + expect(res.status).toBe(400) }), ), { git: true }, diff --git a/packages/deepagent-code/test/session/compaction.test.ts b/packages/deepagent-code/test/session/compaction.test.ts index 1a23c2e70..852eaf91c 100644 --- a/packages/deepagent-code/test/session/compaction.test.ts +++ b/packages/deepagent-code/test/session/compaction.test.ts @@ -51,6 +51,7 @@ const summary = Layer.succeed( summarize: () => Effect.void, diff: () => Effect.succeed([]), computeDiff: () => Effect.succeed([]), + computeManifest: () => Effect.succeed(SessionSummary.emptyManifest()), }), ) diff --git a/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts b/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts new file mode 100644 index 000000000..68b49f6cb --- /dev/null +++ b/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts @@ -0,0 +1,519 @@ +import { expect } from "bun:test" +import { Database } from "@deepagent-code/core/database/database" +import { ModelV2 } from "@deepagent-code/core/model" +import { ProviderV2 } from "@deepagent-code/core/provider" +import { + SessionHistoryStateTable, + SessionIntentTable, + SessionPromptEpochMessageTable, + SessionPromptEpochRecoveryTable, + SessionTable, + SessionToolRequestResolutionCommandTable, + SessionToolRequestResolutionTable, + SessionWorldStateBaselineTable, +} from "@deepagent-code/core/session/sql" +import { and, eq } from "drizzle-orm" +import { Effect, Layer } from "effect" +import { EventV2Bridge } from "@/event-v2-bridge" +import { EventTable } from "@deepagent-code/core/event/sql" +import { SessionLegacyProviderResolution } from "@/session/legacy-provider-resolution" +import { MessageV2 } from "@/session/message-v2" +import { SessionPromptEpochTable } from "@/session/prompt-epoch.sql" +import { recoverProviderReceiptsOnStartup } from "@/session/prompt" +import { MessageID, PartID } from "@/session/schema" +import { Session } from "@/session/session" +import { SessionToolRequestReceiptTable } from "@/session/tool-request-receipt.sql" +import { testEffect } from "../lib/effect" +import { TestInstance } from "../fixture/fixture" + +const layer = Layer.mergeAll( + Session.defaultLayer, + Database.defaultLayer, + EventV2Bridge.defaultLayer, + SessionLegacyProviderResolution.defaultLayer, +) +const it = testEffect(layer) +const model = { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") } +let nextMessageTime = 1 + +const addUser = Effect.fn("LegacyProviderResolutionTest.addUser")(function* ( + sessionID: Parameters[0], + text: string, +) { + const session = yield* Session.Service + const message = yield* session.updateMessage({ + id: MessageID.ascending(`msg_${String(nextMessageTime++).padStart(26, "0")}`), + sessionID, + role: "user", + time: { created: Date.now() }, + agent: "build", + model, + }) + yield* session.updatePart({ + id: PartID.ascending(), + sessionID, + messageID: message.id, + type: "text", + text, + }) + return message +}) + +const addAssistant = Effect.fn("LegacyProviderResolutionTest.addAssistant")(function* ( + sessionID: Parameters[0], + parentID: MessageID, + completed = false, +) { + const session = yield* Session.Service + return yield* session.updateMessage({ + id: MessageID.ascending(`msg_${String(nextMessageTime++).padStart(26, "0")}`), + sessionID, + role: "assistant", + parentID, + time: { created: Date.now(), ...(completed ? { completed: Date.now() } : {}) }, + mode: "build", + agent: "build", + cost: 0, + path: { cwd: "/tmp", root: "/tmp" }, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + modelID: model.modelID, + providerID: model.providerID, + ...(completed ? { finish: "stop" as const } : {}), + }) +}) + +it.instance( + "forks the safe prefix and resolves an ambiguous legacy provider turn exactly once", + () => + Effect.gen(function* () { + const sessions = yield* Session.Service + const recovery = yield* SessionLegacyProviderResolution.Service + yield* TestInstance + const { db } = yield* Database.Service + const parent = yield* sessions.create({ title: "recovery baseline parent" }) + const source = yield* sessions.fork({ + sessionID: parent.id, + intentID: "recovery-baseline-source", + }) + const safeUser = yield* addUser(source.id, "completed before provider crash") + const safeAssistant = yield* addAssistant(source.id, safeUser.id, true) + const user = yield* addUser(source.id, "dispatch then crash") + const assistant = yield* addAssistant(source.id, user.id) + const unsafeTail = yield* addUser(source.id, "physical tail after ambiguous provider turn") + const authority = yield* MessageV2.promptHistoryProjectionEffect(source.id) + if (!authority.worldStateBaselineHash) return yield* Effect.die("expected frozen source World State baseline") + const baselineHash = authority.worldStateBaselineHash + const requestHash = "final-request-hash" + const receiptID = "receipt-ambiguous-provider-turn" + const now = Date.now() + + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: receiptID, + request_ordinal: 1, + session_id: source.id, + user_message_id: user.id, + assistant_message_id: assistant.id, + provider_attempt_id: null, + provider_id: model.providerID, + model_id: model.modelID, + protocol: "chat", + registry_tool_ids: [], + permission_filtered_tool_ids: [], + final_offered_tool_ids: [], + call_ids: [], + prompt_epoch: authority.epoch, + prompt_window_id: authority.window.windowID, + effective_history_hash: authority.effectiveHistoryHash, + request_input_hash: "input-request-hash", + final_request_hash: requestHash, + provider_state: "indeterminate_after_crash", + adapter_prepared_at: now, + dispatching_at: now, + terminal_at: now, + owner_token: "stale-process", + request_state: "dispatched", + request_error_code: "provider_started_outcome_unknown_after_process_restart", + created_at: now, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(SessionPromptEpochTable) + .set({ authority_state: "recovery_required", recovery_reason: "provider outcome is unknown after restart" }) + .where( + and(eq(SessionPromptEpochTable.session_id, source.id), eq(SessionPromptEpochTable.epoch, authority.epoch)), + ) + .run() + .pipe(Effect.orDie) + yield* db + .update(SessionHistoryStateTable) + .set({ + state: "recovery_required", + reason: "provider outcome is unknown after restart", + time_updated: now, + }) + .where(eq(SessionHistoryStateTable.session_id, source.id)) + .run() + .pipe(Effect.orDie) + + const safeCutoff = yield* MessageV2.promptHistoryCutoffProjectionEffect({ + sessionID: source.id, + cutoffMessageID: user.id, + }) + expect( + safeCutoff.messages.map((message) => [ + message.info.id, + message.info.role, + message.info.role === "assistant" ? message.info.parentID : undefined, + ]), + ).toEqual([ + [safeUser.id, "user", undefined], + [safeAssistant.id, "assistant", safeUser.id], + ]) + + const fork = yield* sessions.fork({ + sessionID: source.id, + messageID: user.id, + intentID: "fork-before-ambiguous-provider-turn", + }) + expect((yield* sessions.messages({ sessionID: fork.id })).map((message) => message.info.role)).toEqual([ + "user", + "assistant", + ]) + const ambiguousFork = yield* sessions + .fork({ + sessionID: source.id, + messageID: assistant.id, + intentID: "fork-at-ambiguous-provider-turn", + }) + .pipe(Effect.flip) + expect(ambiguousFork).toMatchObject({ + _tag: "Session.ForkConflict", + reason: expect.stringContaining("fork cutoff is outside the recoverable history prefix"), + }) + const tailFork = yield* sessions + .fork({ + sessionID: source.id, + messageID: unsafeTail.id, + intentID: "fork-after-ambiguous-provider-turn", + }) + .pipe(Effect.flip) + expect(tailFork).toMatchObject({ + _tag: "Session.ForkConflict", + reason: expect.stringContaining("fork cutoff is outside the recoverable history prefix"), + }) + + const [descriptor] = yield* recovery.describe(source.id) + expect(descriptor).toMatchObject({ + receiptID, + sessionID: source.id, + providerState: "indeterminate_after_crash", + promptEpoch: authority.epoch, + requestHash, + sessionMutationEpoch: 0, + continuationRecoverySupported: true, + workspaceRecoverySupported: true, + sourceWorldStateBaselineStatus: "available", + worldStateBaselineHash: baselineHash, + }) + if (!descriptor) return + const command = { + sessionID: source.id, + commandID: "resolve-ambiguous-provider-turn", + receiptID, + decision: "abandoned" as const, + expected: { + providerState: "indeterminate_after_crash" as const, + promptEpoch: descriptor.promptEpoch, + sessionMutationEpoch: descriptor.sessionMutationEpoch, + requestHash: descriptor.requestHash, + historyHash: descriptor.historyHash, + worldStateBaselineHash: descriptor.worldStateBaselineHash!, + }, + actorID: "test-user", + } + const sourceBaseline = yield* db + .select() + .from(SessionWorldStateBaselineTable) + .where( + and( + eq(SessionWorldStateBaselineTable.session_id, source.id), + eq(SessionWorldStateBaselineTable.prompt_epoch, authority.epoch), + ), + ) + .all() + .pipe(Effect.orDie) + expect(sourceBaseline.length).toBeGreaterThan(0) + const assertRecoveryUnchanged = Effect.fnUntraced(function* () { + expect( + yield* db + .select() + .from(SessionToolRequestResolutionCommandTable) + .where(eq(SessionToolRequestResolutionCommandTable.session_id, source.id)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + expect( + yield* db + .select() + .from(SessionToolRequestResolutionTable) + .where(eq(SessionToolRequestResolutionTable.session_id, source.id)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + expect( + yield* db + .select({ epoch: SessionPromptEpochTable.epoch, state: SessionPromptEpochTable.state }) + .from(SessionPromptEpochTable) + .where(eq(SessionPromptEpochTable.session_id, source.id)) + .orderBy(SessionPromptEpochTable.epoch) + .all() + .pipe(Effect.orDie), + ).toEqual([{ epoch: authority.epoch, state: "active" }]) + expect( + yield* db + .select({ mutationEpoch: SessionTable.mutation_epoch }) + .from(SessionTable) + .where(eq(SessionTable.id, source.id)) + .get() + .pipe(Effect.orDie), + ).toEqual({ mutationEpoch: 0 }) + expect( + yield* db + .select({ state: SessionHistoryStateTable.state }) + .from(SessionHistoryStateTable) + .where(eq(SessionHistoryStateTable.session_id, source.id)) + .get() + .pipe(Effect.orDie), + ).toEqual({ state: "recovery_required" }) + }) + + yield* db + .delete(SessionWorldStateBaselineTable) + .where( + and( + eq(SessionWorldStateBaselineTable.session_id, source.id), + eq(SessionWorldStateBaselineTable.prompt_epoch, authority.epoch), + ), + ) + .run() + .pipe(Effect.orDie) + const missingBaseline = yield* recovery + .resolve({ ...command, commandID: "resolve-missing-world-state-baseline" }) + .pipe(Effect.flip) + expect(missingBaseline).toMatchObject({ code: "source_world_state_baseline_missing" }) + yield* assertRecoveryUnchanged() + yield* db.insert(SessionWorldStateBaselineTable).values(sourceBaseline).run().pipe(Effect.orDie) + + yield* db + .delete(SessionWorldStateBaselineTable) + .where( + and( + eq(SessionWorldStateBaselineTable.session_id, source.id), + eq(SessionWorldStateBaselineTable.prompt_epoch, authority.epoch), + ), + ) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionWorldStateBaselineTable) + .values( + sourceBaseline.map((row, index) => + index === 0 ? { ...row, fragment: `${row.fragment}\ninvalid recovery baseline` } : row, + ), + ) + .run() + .pipe(Effect.orDie) + const invalidBaseline = yield* recovery + .resolve({ ...command, commandID: "resolve-invalid-world-state-baseline" }) + .pipe(Effect.flip) + expect(invalidBaseline).toMatchObject({ code: "source_world_state_baseline_invalid" }) + yield* assertRecoveryUnchanged() + yield* db + .delete(SessionWorldStateBaselineTable) + .where( + and( + eq(SessionWorldStateBaselineTable.session_id, source.id), + eq(SessionWorldStateBaselineTable.prompt_epoch, authority.epoch), + ), + ) + .run() + .pipe(Effect.orDie) + yield* db.insert(SessionWorldStateBaselineTable).values(sourceBaseline).run().pipe(Effect.orDie) + + const first = yield* recovery.resolve(command) + const retry = yield* recovery.resolve(command) + expect(retry).toEqual(first) + expect( + yield* db + .select({ id: EventTable.id }) + .from(EventTable) + .where( + and( + eq(EventTable.aggregate_id, source.id), + eq(EventTable.type, `${SessionLegacyProviderResolution.Event.Completed.type}.1`), + ), + ) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + const commandConflict = yield* recovery + .resolve({ ...command, reason: "same command ID with different input" }) + .pipe(Effect.flip) + expect(commandConflict).toBeInstanceOf(SessionLegacyProviderResolution.Conflict) + if (!(commandConflict instanceof SessionLegacyProviderResolution.Conflict)) return + expect(commandConflict.code).toBe("command_id_conflict") + expect(first).toMatchObject({ + decision: "abandoned", + sourcePromptEpoch: authority.epoch, + successorPromptEpoch: authority.epoch + 1, + sourceMutationEpoch: 0, + successorMutationEpoch: 1, + }) + expect(yield* recovery.describe(source.id)).toEqual([]) + + const projection = yield* MessageV2.promptHistoryProjectionEffect(source.id) + expect(projection.epoch).toBe(authority.epoch + 1) + expect(projection.messages.map((message) => message.info.id)).toEqual([safeUser.id, safeAssistant.id]) + expect(projection.recoveryResolutionID).toBe(first.resolutionID) + expect( + yield* db + .select({ + state: SessionPromptEpochTable.state, + authorityState: SessionPromptEpochTable.authority_state, + recoveryReason: SessionPromptEpochTable.recovery_reason, + }) + .from(SessionPromptEpochTable) + .where( + and(eq(SessionPromptEpochTable.session_id, source.id), eq(SessionPromptEpochTable.epoch, authority.epoch)), + ) + .get() + .pipe(Effect.orDie), + ).toEqual({ + state: "retired", + authorityState: "recovery_required", + recoveryReason: "provider outcome is unknown after restart", + }) + expect( + yield* db + .select({ messageID: SessionPromptEpochMessageTable.message_id }) + .from(SessionPromptEpochMessageTable) + .where( + and( + eq(SessionPromptEpochMessageTable.session_id, source.id), + eq(SessionPromptEpochMessageTable.prompt_epoch, authority.epoch + 1), + ), + ) + .orderBy(SessionPromptEpochMessageTable.ordinal) + .all() + .pipe(Effect.orDie), + ).toEqual([{ messageID: safeUser.id }, { messageID: safeAssistant.id }]) + const successor = yield* db + .select({ baselineHash: SessionPromptEpochTable.world_state_baseline_hash }) + .from(SessionPromptEpochTable) + .where( + and( + eq(SessionPromptEpochTable.session_id, source.id), + eq(SessionPromptEpochTable.epoch, authority.epoch + 1), + ), + ) + .get() + .pipe(Effect.orDie) + const baseline = yield* db + .select({ provenance: SessionWorldStateBaselineTable.provenance }) + .from(SessionWorldStateBaselineTable) + .where( + and( + eq(SessionWorldStateBaselineTable.session_id, source.id), + eq(SessionWorldStateBaselineTable.prompt_epoch, authority.epoch + 1), + ), + ) + .all() + .pipe(Effect.orDie) + expect(successor?.baselineHash).toStartWith("wsb1_") + expect(baseline.length).toBeGreaterThan(0) + expect(new Set(baseline.map((row) => row.provenance))).toEqual(new Set(["recovery_copied"])) + expect( + yield* db + .select() + .from(SessionPromptEpochRecoveryTable) + .where(eq(SessionPromptEpochRecoveryTable.resolution_id, first.resolutionID)) + .get() + .pipe(Effect.orDie), + ).toBeDefined() + expect( + yield* db + .select() + .from(SessionToolRequestResolutionTable) + .where(eq(SessionToolRequestResolutionTable.resolution_id, first.resolutionID)) + .get() + .pipe(Effect.orDie), + ).toBeDefined() + expect( + yield* db + .select({ mutationEpoch: SessionTable.mutation_epoch }) + .from(SessionTable) + .where(eq(SessionTable.id, source.id)) + .get() + .pipe(Effect.orDie), + ).toEqual({ mutationEpoch: 1 }) + + yield* recoverProviderReceiptsOnStartup() + expect( + yield* db + .select({ state: SessionHistoryStateTable.state }) + .from(SessionHistoryStateTable) + .where(eq(SessionHistoryStateTable.session_id, source.id)) + .get() + .pipe(Effect.orDie), + ).toEqual({ state: "ready" }) + + const continued = yield* addUser(source.id, "continue after explicit abandon") + expect( + (yield* MessageV2.promptHistoryProjectionEffect(source.id)).messages.map((message) => message.info.id), + ).toEqual([safeUser.id, safeAssistant.id]) + const admittedAt = Date.now() + yield* db + .insert(SessionIntentTable) + .values({ + intent_id: "continue-after-explicit-abandon", + session_id: source.id, + source: "composer", + state: "admitted", + selected_variant: "original", + selected_payload_hash: "test", + delivery: "turn", + admitted_message_id: continued.id, + execution_mode: "legacy", + execution_state: "legacy", + mutation_epoch: first.successorMutationEpoch, + version: 1, + time_created: admittedAt, + time_selected: admittedAt, + time_admitted: admittedAt, + time_updated: admittedAt, + }) + .run() + .pipe(Effect.orDie) + const continuedProjection = yield* MessageV2.promptHistoryProjectionEffect(source.id) + expect(continuedProjection.messages.map((message) => message.info.id)).toEqual([ + safeUser.id, + safeAssistant.id, + continued.id, + ]) + yield* sessions.remove(source.id) + yield* sessions.remove(parent.id) + expect( + yield* db + .select({ id: SessionTable.id }) + .from(SessionTable) + .where(eq(SessionTable.id, source.id)) + .get() + .pipe(Effect.orDie), + ).toBeUndefined() + }), + { git: true }, + 30_000, +) diff --git a/packages/deepagent-code/test/session/processor-effect.test.ts b/packages/deepagent-code/test/session/processor-effect.test.ts index 8b23e2eca..05a2383e2 100644 --- a/packages/deepagent-code/test/session/processor-effect.test.ts +++ b/packages/deepagent-code/test/session/processor-effect.test.ts @@ -49,6 +49,7 @@ const summary = Layer.succeed( summarize: () => Effect.void, diff: () => Effect.succeed([]), computeDiff: () => Effect.succeed([]), + computeManifest: () => Effect.succeed(SessionSummary.emptyManifest()), }), ) diff --git a/packages/deepagent-code/test/session/prompt.test.ts b/packages/deepagent-code/test/session/prompt.test.ts index 6820a7376..96bcd4dd4 100644 --- a/packages/deepagent-code/test/session/prompt.test.ts +++ b/packages/deepagent-code/test/session/prompt.test.ts @@ -106,6 +106,7 @@ const summary = Layer.succeed( summarize: () => Effect.void, diff: () => Effect.succeed([]), computeDiff: () => Effect.succeed([]), + computeManifest: () => Effect.succeed(SessionSummary.emptyManifest()), }), ) @@ -882,6 +883,7 @@ it.instance("loop exits without an LLM request for interrupted orphan tool calls Effect.gen(function* () { const { llm } = yield* useServerConfig(providerCfg) const prompt = yield* SessionPrompt.Service + const steer = yield* SessionSteer.Service const sessions = yield* Session.Service const chat = yield* sessions.create({ title: "Pinned" }) const seeded = yield* seed(chat.id, { finish: "stop" }) diff --git a/packages/deepagent-code/test/session/revert-compact.test.ts b/packages/deepagent-code/test/session/revert-compact.test.ts index e7be7ce7c..99101222f 100644 --- a/packages/deepagent-code/test/session/revert-compact.test.ts +++ b/packages/deepagent-code/test/session/revert-compact.test.ts @@ -15,6 +15,7 @@ import { provideTmpdirInstance } from "../fixture/fixture" import { testEffect } from "../lib/effect" import { ProviderV2 } from "@deepagent-code/core/provider" import { ModelV2 } from "@deepagent-code/core/model" +import { Storage } from "@/storage/storage" void Log.init({ print: false }) @@ -22,6 +23,7 @@ const env = Layer.mergeAll( Session.defaultLayer, SessionRevert.defaultLayer, Snapshot.defaultLayer, + Storage.defaultLayer, CrossSpawnSpawner.defaultLayer, ) @@ -549,7 +551,14 @@ describe("revert + compact workflow", () => { sessionID: sid, messageID: first, }) - expect((yield* session.get(sid)).revert?.messageID).toBe(first) + const reverted = yield* session.get(sid) + expect(reverted.revert?.messageID).toBe(first) + expect(reverted.summary?.files).toBe(3) + const persisted = yield* Storage.Service.use((storage) => + storage.read(["session_diff", sid]), + ) + expect(persisted).toHaveLength(3) + expect(persisted.every((item) => item.patch === undefined)).toBe(true) expect(yield* read(path.join(dir, "a.txt"))).toBe("a0") expect(yield* read(path.join(dir, "b.txt"))).toBe("b0") expect(yield* read(path.join(dir, "c.txt"))).toBe("c0") diff --git a/packages/deepagent-code/test/session/snapshot-tool-race.test.ts b/packages/deepagent-code/test/session/snapshot-tool-race.test.ts index e48d4ab30..99e41fbc1 100644 --- a/packages/deepagent-code/test/session/snapshot-tool-race.test.ts +++ b/packages/deepagent-code/test/session/snapshot-tool-race.test.ts @@ -152,8 +152,7 @@ const stubRuntimeBaseLayer = Layer.succeed( // Fully-inert DebugService (D1) stub — avoids InstanceState.make + finalizer side effects // at registry-build time. This test never invokes the debug tool. -const debugStubDie = (): Effect.Effect => - Effect.die("DebugService stub (not used in this test)") +const debugStubDie = (): Effect.Effect => Effect.die("DebugService stub (not used in this test)") const stubDebugServiceLayer = Layer.succeed( DebugService.Service, DebugService.Service.of({ @@ -336,13 +335,48 @@ it.live("tool execution produces non-empty session diff (snapshot race)", () => if (!user) throw new Error("Expected user message") // Poll for the turn diff — summarize() is fire-and-forget. - let diff: Array<{ file?: string }> = [] + let diff: Snapshot.FileDiff[] = [] for (let i = 0; i < 50; i++) { diff = yield* summary.diff({ sessionID: session.id, messageID: user.info.id }) if (diff.length > 0) break yield* Effect.sleep("100 millis") } expect(diff.length).toBeGreaterThan(0) + expect(diff.every((item) => item.patch === undefined)).toBe(true) + const persisted = (yield* sessions.messages({ sessionID: session.id })).find( + (message) => message.info.id === user.info.id && message.info.role === "user", + ) + expect( + persisted?.info.role === "user" + ? persisted.info.summary?.diffs.every((item) => item.patch === undefined) + : false, + ).toBe(true) + expect(persisted?.info.role === "user" ? persisted.info.summary?.diffManifest : undefined).toMatchObject({ + completeness: "complete", + totalFiles: 1, + totalFilesExact: true, + statisticsExact: true, + includedFiles: 1, + truncatedFiles: 0, + }) + expect( + persisted?.info.role === "user" ? persisted.info.summary?.diffManifest?.manifestHash : undefined, + ).toStartWith("sha256:") + let sessionSummary = (yield* sessions.get(session.id)).summary + for (let i = 0; i < 20 && sessionSummary?.files === 0; i++) { + yield* Effect.sleep("50 millis") + sessionSummary = (yield* sessions.get(session.id)).summary + } + expect(sessionSummary).toMatchObject({ additions: 1, deletions: 0, files: 1 }) + expect(sessionSummary?.diffManifest).toMatchObject({ + completeness: "complete", + totalFiles: 1, + totalFilesExact: true, + statisticsExact: true, + includedFiles: 1, + truncatedFiles: 0, + }) + expect(sessionSummary?.diffManifest?.manifestHash).toStartWith("sha256:") }), { git: true, config: providerCfg }, ), diff --git a/packages/deepagent-code/test/session/steer.test.ts b/packages/deepagent-code/test/session/steer.test.ts index baad448b6..1781cdd75 100644 --- a/packages/deepagent-code/test/session/steer.test.ts +++ b/packages/deepagent-code/test/session/steer.test.ts @@ -94,6 +94,7 @@ const summary = Layer.succeed( summarize: () => Effect.void, diff: () => Effect.succeed([]), computeDiff: () => Effect.succeed([]), + computeManifest: () => Effect.succeed(SessionSummary.emptyManifest()), }), ) diff --git a/packages/deepagent-code/test/snapshot/snapshot.test.ts b/packages/deepagent-code/test/snapshot/snapshot.test.ts index d76d43529..9e2f9e056 100644 --- a/packages/deepagent-code/test/snapshot/snapshot.test.ts +++ b/packages/deepagent-code/test/snapshot/snapshot.test.ts @@ -479,6 +479,13 @@ it.instance( expect(patch.files).not.toContain(fwd(tmp.path, "later-ignored.txt")) expect(patch.files).toContain(fwd(tmp.path, ".gitignore")) expect(patch.files).toContain(fwd(tmp.path, "still-tracked.txt")) + const after = yield* snapshot.track() + const manifest = yield* snapshot.diffManifest(before!, after!) + expect(manifest.files.map((item) => item.file)).toEqual([".gitignore", "still-tracked.txt"]) + expect(manifest.totalFiles).toBe(2) + expect(manifest.totalFilesExact).toBe(true) + expect(manifest.completeness).toBe("complete") + expect(manifest.truncationReasons).toEqual([]) }), { git: true }, ) @@ -1041,6 +1048,131 @@ it.instance( { git: true }, ) +it.instance( + "diffFull bounds candidate files before materialization", + withTrackedSnapshot(({ tmp, snapshot, before }) => + Effect.gen(function* () { + yield* Effect.forEach( + Array.from({ length: Snapshot.DiffLimits.candidateFiles + 5 }, (_, index) => index), + (index) => write(`${tmp.path}/generated/file-${String(index).padStart(4, "0")}.txt`, "content"), + { concurrency: 50, discard: true }, + ) + const after = yield* snapshot.track() + expect(after).toBeTruthy() + const diffs = yield* snapshot.diffFull(before, after!) + expect(diffs.length).toBe(Snapshot.DiffLimits.candidateFiles) + expect(diffs[0].file).toBe("generated/file-0000.txt") + expect(diffs.at(-1)?.file).toBe("generated/file-0999.txt") + }), + ), + { git: true }, +) + +it.instance( + "diffManifest is metadata-only, deterministic, and reports candidate truncation", + withTrackedSnapshot(({ tmp, snapshot, before }) => + Effect.gen(function* () { + yield* Effect.forEach( + Array.from({ length: Snapshot.DiffLimits.candidateFiles + 5 }, (_, index) => index), + (index) => write(`${tmp.path}/manifest/file-${String(index).padStart(4, "0")}.txt`, "content"), + { concurrency: 50, discard: true }, + ) + const after = yield* snapshot.track() + expect(after).toBeTruthy() + const first = yield* snapshot.diffManifest(before, after!) + const second = yield* snapshot.diffManifest(before, after!) + + expect(first).toEqual(second) + expect(first.files).toHaveLength(Snapshot.DiffLimits.candidateFiles) + expect(first.files.every((item) => item.patch === undefined)).toBe(true) + expect(first.totalFiles).toBe(Snapshot.DiffLimits.candidateFiles + 5) + expect(first.totalFilesExact).toBe(true) + expect(first.statisticsExact).toBe(false) + expect(first.completeness).toBe("truncated") + expect(first.truncationReasons).toContain("candidate_file_limit") + expect(first.manifestHash).toMatch(/^sha256:[a-f0-9]{64}$/) + expect(Buffer.byteLength(JSON.stringify(first.files))).toBeLessThanOrEqual(Snapshot.DiffLimits.manifestBytes) + }), + ), + { git: true }, +) + +it.instance( + "diffManifest preserves visible statuses after a large ignored prefix", + withTrackedSnapshot(({ tmp, snapshot, before }) => + Effect.gen(function* () { + const ignored = Array.from( + { length: Snapshot.DiffLimits.candidateFiles + 5 }, + (_, index) => `ignored/file-${String(index).padStart(4, "0")}.txt`, + ) + yield* Effect.forEach(ignored, (file) => write(`${tmp.path}/${file}`, "ignored"), { + concurrency: 50, + discard: true, + }) + yield* write(`${tmp.path}/visible-added.txt`, "visible") + const after = yield* snapshot.track() + expect(after).toBeTruthy() + yield* write(`${tmp.path}/.gitignore`, "ignored/\n") + const filtered = yield* snapshot.track() + expect(filtered).toBeTruthy() + const manifest = yield* snapshot.diffManifest(before, filtered!) + const visible = manifest.files.find((item) => item.file === "visible-added.txt") + expect(visible?.status).toBe("added") + expect(manifest.files.some((item) => item.file?.startsWith("ignored/"))).toBe(false) + }), + ), + { git: true }, + 30_000, +) + +it.instance( + "diffFull does not materialize a source file above the hard limit", + withTrackedSnapshot(({ tmp, snapshot, before }) => + Effect.gen(function* () { + yield* write(`${tmp.path}/too-large.txt`, "x".repeat(Snapshot.DiffLimits.sourceFileBytes + 1)) + const after = yield* snapshot.track() + expect(after).toBeTruthy() + const diffs = yield* snapshot.diffFull(before, after!) + expect(diffs).toHaveLength(1) + expect(diffs[0].file).toBe("too-large.txt") + expect(diffs[0].patch).toBe("") + expect(diffs[0].additions).toBe(1) + const manifest = yield* snapshot.diffFullManifest(before, after!) + expect(manifest.statisticsExact).toBe(true) + expect(manifest.completeness).toBe("truncated") + expect(manifest.truncationReasons).toContain("source_file_limit") + }), + ), + { git: true }, +) + +it.instance( + "diffFull bounds cumulative patch bytes", + withTrackedSnapshot(({ tmp, snapshot, before }) => + Effect.gen(function* () { + const content = Array.from({ length: 60_000 }, (_, index) => `line-${index}`).join("\n") + yield* Effect.forEach( + Array.from({ length: 12 }, (_, index) => index), + (index) => write(`${tmp.path}/patch-budget/file-${index}.txt`, content), + { concurrency: 6, discard: true }, + ) + const after = yield* snapshot.track() + expect(after).toBeTruthy() + const diffs = yield* snapshot.diffFull(before, after!) + expect(diffs).toHaveLength(12) + expect(diffs.reduce((bytes, item) => bytes + Buffer.byteLength(item.patch ?? ""), 0)).toBeLessThanOrEqual( + Snapshot.DiffLimits.patchTotalBytes, + ) + expect(diffs.some((item) => item.patch === "")).toBe(true) + const manifest = yield* snapshot.diffFullManifest(before, after!) + expect(manifest.statisticsExact).toBe(true) + expect(manifest.completeness).toBe("truncated") + expect(manifest.truncationReasons).toContain("patch_file_limit") + }), + ), + { git: true }, +) + it.instance( "revert with overlapping files across patches uses first patch hash", Effect.gen(function* () { diff --git a/packages/sdk/js/src/gen/sdk.gen.ts b/packages/sdk/js/src/gen/sdk.gen.ts index c11202899..35c290b0c 100644 --- a/packages/sdk/js/src/gen/sdk.gen.ts +++ b/packages/sdk/js/src/gen/sdk.gen.ts @@ -413,6 +413,10 @@ import type { SessionPromptResponses, SessionPromptSuggestionErrors, SessionPromptSuggestionResponses, + SessionProviderResolutionListErrors, + SessionProviderResolutionListResponses, + SessionProviderResolutionResolveErrors, + SessionProviderResolutionResolveResponses, SessionRevertErrors, SessionRevertResponses, SessionShareErrors, @@ -8283,6 +8287,102 @@ export class Session2 extends HeyApiClient { }, }) } + + /** + * List provider recovery decisions + * + * List unresolved legacy provider outcomes that require explicit recovery. + */ + public providerResolutionList( + parameters: { + sessionID: string + directory?: string + workspace?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get< + SessionProviderResolutionListResponses, + SessionProviderResolutionListErrors, + ThrowOnError + >({ + url: "/session/{sessionID}/provider-resolution", + ...options, + ...params, + }) + } + + /** + * Resolve a provider outcome + * + * Append an audited abandoned resolution and activate a safe successor history epoch. + */ + public providerResolutionResolve( + parameters: { + sessionID: string + directory?: string + workspace?: string + commandID: string + receiptID: string + decision: "abandoned" + expected: { + providerState: "indeterminate_after_crash" + promptEpoch: number + sessionMutationEpoch: number + requestHash: string + historyHash: string + worldStateBaselineHash: string + } + reason?: string + riskAcknowledged?: boolean + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "commandID" }, + { in: "body", key: "receiptID" }, + { in: "body", key: "decision" }, + { in: "body", key: "expected" }, + { in: "body", key: "reason" }, + { in: "body", key: "riskAcknowledged" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + SessionProviderResolutionResolveResponses, + SessionProviderResolutionResolveErrors, + ThrowOnError + >({ + url: "/session/{sessionID}/provider-resolution", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } } export class Part extends HeyApiClient { @@ -8366,7 +8466,7 @@ export class History extends HeyApiClient { /** * List sync events * - * List sync events for all aggregates. Keys are aggregate IDs the client already knows about, values are the last known sequence ID. Events with seq > value are returned for those aggregates. Aggregates not listed in the input get their full history. + * List a bounded page of sync events. Keys are aggregate IDs the client already knows about, values are the last known sequence ID. Oversized legacy events fail closed until maintenance migration externalizes their inline payload. */ public list( parameters?: { diff --git a/packages/sdk/js/src/gen/types.gen.ts b/packages/sdk/js/src/gen/types.gen.ts index d8d300136..ad320bc40 100644 --- a/packages/sdk/js/src/gen/types.gen.ts +++ b/packages/sdk/js/src/gen/types.gen.ts @@ -101,6 +101,7 @@ export type Event = | EventWorkspaceReady | EventWorkspaceFailed | EventWorkspaceStatus + | EventSessionProviderResolutionCompleted | EventServerConnected | EventGlobalDisposed | EventServerInstanceDisposed @@ -167,6 +168,28 @@ export type SnapshotFileDiff = { status?: "added" | "deleted" | "modified" } +export type SnapshotDiffManifestDescriptor = { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number +} + export type PermissionAction = "allow" | "deny" | "ask" export type PermissionRule = { @@ -190,6 +213,7 @@ export type Session = { deletions: number files: number diffs?: Array + diffManifest?: SnapshotDiffManifestDescriptor } cost?: number tokens?: { @@ -259,6 +283,7 @@ export type UserMessage = { title?: string body?: string diffs: Array + diffManifest?: SnapshotDiffManifestDescriptor } agent: string model: { @@ -1489,6 +1514,27 @@ export type GlobalEvent = { properties: { sessionID: string diff: Array + manifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } } } | { @@ -1799,6 +1845,22 @@ export type GlobalEvent = { status: "connected" | "connecting" | "disconnected" | "error" } } + | { + id: string + type: "session.provider-resolution.completed" + properties: { + sessionID: string + resolutionID: string + commandID: string + receiptID: string + decision: "abandoned" + sourcePromptEpoch: number + successorPromptEpoch: number + sourceMutationEpoch: number + successorMutationEpoch: number + createdAt: number + } + } | { id: string type: "server.connected" @@ -2711,6 +2773,27 @@ export type GlobalSession = { deletions: number files: number diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } } cost?: number tokens?: { @@ -3045,6 +3128,29 @@ export type VcsFileDiff = { status?: "added" | "deleted" | "modified" } +export type VcsRawDiffError = { + name: "VcsRawDiffError" + data: { + message: string + reason: + | "candidate-files" + | "status-output" + | "status-failed" + | "tracked-output" + | "tracked-failed" + | "untracked-output" + | "untracked-failed" + | "total-output" + | "time-limit" + | "remote-status" + | "remote-output" + | "remote-decode" + limit?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + actual?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + file?: string + } +} + export type VcsApplyError = { name: "VcsApplyError" data: { @@ -3334,18 +3440,543 @@ export type ReferenceDescriptor = repository?: string message: string } - -export type NotFoundError = { - name: "NotFoundError" - data: { - message: string + +export type Session1 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type Session2 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type NotFoundError = { + name: "NotFoundError" + data: { + message: string + } +} + +export type Session3 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type Session4 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type Session5 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type ConflictError = { + _tag: "ConflictError" + message: string + resource?: string +} + +export type Session6 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type Session7 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } } -} - -export type ConflictError = { - _tag: "ConflictError" - message: string - resource?: string + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string } export type TextPartInput = { @@ -3402,6 +4033,162 @@ export type SessionBusyError = { message: string } +export type Session8 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type ServiceUnavailableError = { + _tag: "ServiceUnavailableError" + message: string + service?: string +} + +export type Session9 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + export type EventTuiPromptAppend = { type: "tui.prompt.append" properties: { @@ -3504,12 +4291,6 @@ export type InvalidCursorError = { message: string } -export type ServiceUnavailableError = { - _tag: "ServiceUnavailableError" - message: string - service?: string -} - export type SessionNotFoundError = { _tag: "SessionNotFoundError" sessionID: string @@ -5830,6 +6611,27 @@ export type EventSessionDiff = { properties: { sessionID: string diff: Array + manifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } } } @@ -6175,6 +6977,23 @@ export type EventWorkspaceStatus = { } } +export type EventSessionProviderResolutionCompleted = { + id: string + type: "session.provider-resolution.completed" + properties: { + sessionID: string + resolutionID: string + commandID: string + receiptID: string + decision: "abandoned" + sourcePromptEpoch: number + successorPromptEpoch: number + sourceMutationEpoch: number + successorMutationEpoch: number + createdAt: number + } +} + export type EventServerConnected = { id: string type: "server.connected" @@ -10818,6 +11637,10 @@ export type VcsDiffRawErrors = { * Bad request */ 400: BadRequestError + /** + * VcsRawDiffError + */ + 503: VcsRawDiffError } export type VcsDiffRawError = VcsDiffRawErrors[keyof VcsDiffRawErrors] @@ -12346,7 +13169,7 @@ export type SessionListResponses = { /** * List of sessions */ - 200: Array + 200: Array } export type SessionListResponse = SessionListResponses[keyof SessionListResponses] @@ -12388,7 +13211,7 @@ export type SessionCreateResponses = { /** * Successfully created session */ - 200: Session + 200: Session3 } export type SessionCreateResponse = SessionCreateResponses[keyof SessionCreateResponses] @@ -12486,7 +13309,7 @@ export type SessionGetResponses = { /** * Get session */ - 200: Session + 200: Session2 } export type SessionGetResponse = SessionGetResponses[keyof SessionGetResponses] @@ -12529,7 +13352,7 @@ export type SessionUpdateResponses = { /** * Successfully updated session */ - 200: Session + 200: Session4 } export type SessionUpdateResponse = SessionUpdateResponses[keyof SessionUpdateResponses] @@ -12563,7 +13386,7 @@ export type SessionChildrenResponses = { /** * List of children */ - 200: Array + 200: Array } export type SessionChildrenResponse = SessionChildrenResponses[keyof SessionChildrenResponses] @@ -12904,7 +13727,7 @@ export type SessionForkResponses = { /** * 200 */ - 200: Session + 200: Session5 } export type SessionForkResponse = SessionForkResponses[keyof SessionForkResponses] @@ -13010,7 +13833,7 @@ export type SessionUnshareResponses = { /** * Successfully unshared session */ - 200: Session + 200: Session7 } export type SessionUnshareResponse = SessionUnshareResponses[keyof SessionUnshareResponses] @@ -13048,7 +13871,7 @@ export type SessionShareResponses = { /** * Successfully shared session */ - 200: Session + 200: Session6 } export type SessionShareResponse = SessionShareResponses[keyof SessionShareResponses] @@ -13417,6 +14240,10 @@ export type SessionRevertErrors = { * SessionBusyError */ 409: SessionBusyError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError } export type SessionRevertError = SessionRevertErrors[keyof SessionRevertErrors] @@ -13425,7 +14252,7 @@ export type SessionRevertResponses = { /** * Updated session */ - 200: Session + 200: Session8 } export type SessionRevertResponse = SessionRevertResponses[keyof SessionRevertResponses] @@ -13463,7 +14290,7 @@ export type SessionUnrevertResponses = { /** * Updated session */ - 200: Session + 200: Session9 } export type SessionUnrevertResponse = SessionUnrevertResponses[keyof SessionUnrevertResponses] @@ -14169,6 +14996,127 @@ export type SessionContextAttemptResolveResponses = { export type SessionContextAttemptResolveResponse = SessionContextAttemptResolveResponses[keyof SessionContextAttemptResolveResponses] +export type SessionProviderResolutionListData = { + body?: never + path: { + sessionID: string + } + query?: { + directory?: string + workspace?: string + } + url: "/session/{sessionID}/provider-resolution" +} + +export type SessionProviderResolutionListErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * NotFoundError + */ + 404: NotFoundError +} + +export type SessionProviderResolutionListError = + SessionProviderResolutionListErrors[keyof SessionProviderResolutionListErrors] + +export type SessionProviderResolutionListResponses = { + /** + * Pending provider recovery decisions + */ + 200: Array<{ + receiptID: string + sessionID: string + assistantMessageID: string + providerID: string + modelID: string + providerState: "indeterminate_after_crash" + promptEpoch: number + promptWindowID: string + historyHash: string + requestHash: string + sessionMutationEpoch: number + continuationRecoverySupported: boolean + workspaceRecoverySupported: boolean + sourceWorldStateBaselineStatus: "available" | "missing" | "invalid" + worldStateBaselineHash?: string + }> +} + +export type SessionProviderResolutionListResponse = + SessionProviderResolutionListResponses[keyof SessionProviderResolutionListResponses] + +export type SessionProviderResolutionResolveData = { + body?: { + commandID: string + receiptID: string + decision: "abandoned" + expected: { + providerState: "indeterminate_after_crash" + promptEpoch: number + sessionMutationEpoch: number + requestHash: string + historyHash: string + worldStateBaselineHash: string + } + reason?: string + riskAcknowledged?: boolean + } + path: { + sessionID: string + } + query?: { + directory?: string + workspace?: string + } + url: "/session/{sessionID}/provider-resolution" +} + +export type SessionProviderResolutionResolveErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * NotFoundError + */ + 404: NotFoundError + /** + * ConflictError + */ + 409: ConflictError +} + +export type SessionProviderResolutionResolveError = + SessionProviderResolutionResolveErrors[keyof SessionProviderResolutionResolveErrors] + +export type SessionProviderResolutionResolveResponses = { + /** + * Resolved provider outcome + */ + 200: { + resolutionID: string + commandID: string + receiptID: string + sessionID: string + decision: "abandoned" + sourcePromptEpoch: number + successorPromptEpoch: number + sourceMutationEpoch: number + successorMutationEpoch: number + safeEndMessageID?: string + safeHistoryHash: string + successorWindowID: string + successorHistoryHash: string + createdAt: number + } +} + +export type SessionProviderResolutionResolveResponse = + SessionProviderResolutionResolveResponses[keyof SessionProviderResolutionResolveResponses] + export type SyncStartData = { body?: never path?: never @@ -14255,6 +15203,10 @@ export type SyncStealErrors = { * BadRequest | InvalidRequestError */ 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * ConflictError + */ + 409: ConflictError } export type SyncStealError = SyncStealErrors[keyof SyncStealErrors] @@ -14287,6 +15239,10 @@ export type SyncHistoryListErrors = { * BadRequest | InvalidRequestError */ 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError } export type SyncHistoryListError = SyncHistoryListErrors[keyof SyncHistoryListErrors] @@ -14909,6 +15865,10 @@ export type ExperimentalWorkspaceWarpErrors = { * NotFoundError */ 404: NotFoundError + /** + * VcsRawDiffError + */ + 503: VcsRawDiffError } export type ExperimentalWorkspaceWarpError = ExperimentalWorkspaceWarpErrors[keyof ExperimentalWorkspaceWarpErrors] diff --git a/packages/sdk/js/src/v2/gen/sdk.gen.ts b/packages/sdk/js/src/v2/gen/sdk.gen.ts index c11202899..35c290b0c 100644 --- a/packages/sdk/js/src/v2/gen/sdk.gen.ts +++ b/packages/sdk/js/src/v2/gen/sdk.gen.ts @@ -413,6 +413,10 @@ import type { SessionPromptResponses, SessionPromptSuggestionErrors, SessionPromptSuggestionResponses, + SessionProviderResolutionListErrors, + SessionProviderResolutionListResponses, + SessionProviderResolutionResolveErrors, + SessionProviderResolutionResolveResponses, SessionRevertErrors, SessionRevertResponses, SessionShareErrors, @@ -8283,6 +8287,102 @@ export class Session2 extends HeyApiClient { }, }) } + + /** + * List provider recovery decisions + * + * List unresolved legacy provider outcomes that require explicit recovery. + */ + public providerResolutionList( + parameters: { + sessionID: string + directory?: string + workspace?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get< + SessionProviderResolutionListResponses, + SessionProviderResolutionListErrors, + ThrowOnError + >({ + url: "/session/{sessionID}/provider-resolution", + ...options, + ...params, + }) + } + + /** + * Resolve a provider outcome + * + * Append an audited abandoned resolution and activate a safe successor history epoch. + */ + public providerResolutionResolve( + parameters: { + sessionID: string + directory?: string + workspace?: string + commandID: string + receiptID: string + decision: "abandoned" + expected: { + providerState: "indeterminate_after_crash" + promptEpoch: number + sessionMutationEpoch: number + requestHash: string + historyHash: string + worldStateBaselineHash: string + } + reason?: string + riskAcknowledged?: boolean + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "commandID" }, + { in: "body", key: "receiptID" }, + { in: "body", key: "decision" }, + { in: "body", key: "expected" }, + { in: "body", key: "reason" }, + { in: "body", key: "riskAcknowledged" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + SessionProviderResolutionResolveResponses, + SessionProviderResolutionResolveErrors, + ThrowOnError + >({ + url: "/session/{sessionID}/provider-resolution", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } } export class Part extends HeyApiClient { @@ -8366,7 +8466,7 @@ export class History extends HeyApiClient { /** * List sync events * - * List sync events for all aggregates. Keys are aggregate IDs the client already knows about, values are the last known sequence ID. Events with seq > value are returned for those aggregates. Aggregates not listed in the input get their full history. + * List a bounded page of sync events. Keys are aggregate IDs the client already knows about, values are the last known sequence ID. Oversized legacy events fail closed until maintenance migration externalizes their inline payload. */ public list( parameters?: { diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index d8d300136..ad320bc40 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -101,6 +101,7 @@ export type Event = | EventWorkspaceReady | EventWorkspaceFailed | EventWorkspaceStatus + | EventSessionProviderResolutionCompleted | EventServerConnected | EventGlobalDisposed | EventServerInstanceDisposed @@ -167,6 +168,28 @@ export type SnapshotFileDiff = { status?: "added" | "deleted" | "modified" } +export type SnapshotDiffManifestDescriptor = { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number +} + export type PermissionAction = "allow" | "deny" | "ask" export type PermissionRule = { @@ -190,6 +213,7 @@ export type Session = { deletions: number files: number diffs?: Array + diffManifest?: SnapshotDiffManifestDescriptor } cost?: number tokens?: { @@ -259,6 +283,7 @@ export type UserMessage = { title?: string body?: string diffs: Array + diffManifest?: SnapshotDiffManifestDescriptor } agent: string model: { @@ -1489,6 +1514,27 @@ export type GlobalEvent = { properties: { sessionID: string diff: Array + manifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } } } | { @@ -1799,6 +1845,22 @@ export type GlobalEvent = { status: "connected" | "connecting" | "disconnected" | "error" } } + | { + id: string + type: "session.provider-resolution.completed" + properties: { + sessionID: string + resolutionID: string + commandID: string + receiptID: string + decision: "abandoned" + sourcePromptEpoch: number + successorPromptEpoch: number + sourceMutationEpoch: number + successorMutationEpoch: number + createdAt: number + } + } | { id: string type: "server.connected" @@ -2711,6 +2773,27 @@ export type GlobalSession = { deletions: number files: number diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } } cost?: number tokens?: { @@ -3045,6 +3128,29 @@ export type VcsFileDiff = { status?: "added" | "deleted" | "modified" } +export type VcsRawDiffError = { + name: "VcsRawDiffError" + data: { + message: string + reason: + | "candidate-files" + | "status-output" + | "status-failed" + | "tracked-output" + | "tracked-failed" + | "untracked-output" + | "untracked-failed" + | "total-output" + | "time-limit" + | "remote-status" + | "remote-output" + | "remote-decode" + limit?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + actual?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + file?: string + } +} + export type VcsApplyError = { name: "VcsApplyError" data: { @@ -3334,18 +3440,543 @@ export type ReferenceDescriptor = repository?: string message: string } - -export type NotFoundError = { - name: "NotFoundError" - data: { - message: string + +export type Session1 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type Session2 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type NotFoundError = { + name: "NotFoundError" + data: { + message: string + } +} + +export type Session3 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type Session4 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type Session5 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type ConflictError = { + _tag: "ConflictError" + message: string + resource?: string +} + +export type Session6 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type Session7 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } } -} - -export type ConflictError = { - _tag: "ConflictError" - message: string - resource?: string + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string } export type TextPartInput = { @@ -3402,6 +4033,162 @@ export type SessionBusyError = { message: string } +export type Session8 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + +export type ServiceUnavailableError = { + _tag: "ServiceUnavailableError" + message: string + service?: string +} + +export type Session9 = { + id: string + slug: string + projectID: string + workspaceID?: string + directory: string + path?: string + parentID?: string + summary?: { + additions: number + deletions: number + files: number + diffs?: Array + diffManifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } + } + cost?: number + tokens?: { + input: number + output: number + reasoning: number + cache: { + read: number + write: number + } + } + share?: { + url: string + } + title: string + agent?: string + model?: { + id: string + providerID: string + variant?: string + } + version: string + metadata?: { + [key: string]: unknown + } + time: { + created: number + updated: number + compacting?: number + archived?: number + } + permission?: PermissionRuleset + revert?: { + messageID: string + partID?: string + snapshot?: string + diff?: string + } + preview?: string +} + export type EventTuiPromptAppend = { type: "tui.prompt.append" properties: { @@ -3504,12 +4291,6 @@ export type InvalidCursorError = { message: string } -export type ServiceUnavailableError = { - _tag: "ServiceUnavailableError" - message: string - service?: string -} - export type SessionNotFoundError = { _tag: "SessionNotFoundError" sessionID: string @@ -5830,6 +6611,27 @@ export type EventSessionDiff = { properties: { sessionID: string diff: Array + manifest?: { + completeness: "complete" | "truncated" + truncationReasons: Array< + | "candidate_file_limit" + | "discovery_output_limit" + | "discovery_failed" + | "manifest_bytes_limit" + | "source_file_limit" + | "source_total_limit" + | "patch_file_limit" + | "patch_total_limit" + | "materialization_failed" + | "time_limit" + > + manifestHash: string + totalFiles: number + totalFilesExact: boolean + statisticsExact?: boolean + includedFiles: number + truncatedFiles: number + } } } @@ -6175,6 +6977,23 @@ export type EventWorkspaceStatus = { } } +export type EventSessionProviderResolutionCompleted = { + id: string + type: "session.provider-resolution.completed" + properties: { + sessionID: string + resolutionID: string + commandID: string + receiptID: string + decision: "abandoned" + sourcePromptEpoch: number + successorPromptEpoch: number + sourceMutationEpoch: number + successorMutationEpoch: number + createdAt: number + } +} + export type EventServerConnected = { id: string type: "server.connected" @@ -10818,6 +11637,10 @@ export type VcsDiffRawErrors = { * Bad request */ 400: BadRequestError + /** + * VcsRawDiffError + */ + 503: VcsRawDiffError } export type VcsDiffRawError = VcsDiffRawErrors[keyof VcsDiffRawErrors] @@ -12346,7 +13169,7 @@ export type SessionListResponses = { /** * List of sessions */ - 200: Array + 200: Array } export type SessionListResponse = SessionListResponses[keyof SessionListResponses] @@ -12388,7 +13211,7 @@ export type SessionCreateResponses = { /** * Successfully created session */ - 200: Session + 200: Session3 } export type SessionCreateResponse = SessionCreateResponses[keyof SessionCreateResponses] @@ -12486,7 +13309,7 @@ export type SessionGetResponses = { /** * Get session */ - 200: Session + 200: Session2 } export type SessionGetResponse = SessionGetResponses[keyof SessionGetResponses] @@ -12529,7 +13352,7 @@ export type SessionUpdateResponses = { /** * Successfully updated session */ - 200: Session + 200: Session4 } export type SessionUpdateResponse = SessionUpdateResponses[keyof SessionUpdateResponses] @@ -12563,7 +13386,7 @@ export type SessionChildrenResponses = { /** * List of children */ - 200: Array + 200: Array } export type SessionChildrenResponse = SessionChildrenResponses[keyof SessionChildrenResponses] @@ -12904,7 +13727,7 @@ export type SessionForkResponses = { /** * 200 */ - 200: Session + 200: Session5 } export type SessionForkResponse = SessionForkResponses[keyof SessionForkResponses] @@ -13010,7 +13833,7 @@ export type SessionUnshareResponses = { /** * Successfully unshared session */ - 200: Session + 200: Session7 } export type SessionUnshareResponse = SessionUnshareResponses[keyof SessionUnshareResponses] @@ -13048,7 +13871,7 @@ export type SessionShareResponses = { /** * Successfully shared session */ - 200: Session + 200: Session6 } export type SessionShareResponse = SessionShareResponses[keyof SessionShareResponses] @@ -13417,6 +14240,10 @@ export type SessionRevertErrors = { * SessionBusyError */ 409: SessionBusyError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError } export type SessionRevertError = SessionRevertErrors[keyof SessionRevertErrors] @@ -13425,7 +14252,7 @@ export type SessionRevertResponses = { /** * Updated session */ - 200: Session + 200: Session8 } export type SessionRevertResponse = SessionRevertResponses[keyof SessionRevertResponses] @@ -13463,7 +14290,7 @@ export type SessionUnrevertResponses = { /** * Updated session */ - 200: Session + 200: Session9 } export type SessionUnrevertResponse = SessionUnrevertResponses[keyof SessionUnrevertResponses] @@ -14169,6 +14996,127 @@ export type SessionContextAttemptResolveResponses = { export type SessionContextAttemptResolveResponse = SessionContextAttemptResolveResponses[keyof SessionContextAttemptResolveResponses] +export type SessionProviderResolutionListData = { + body?: never + path: { + sessionID: string + } + query?: { + directory?: string + workspace?: string + } + url: "/session/{sessionID}/provider-resolution" +} + +export type SessionProviderResolutionListErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * NotFoundError + */ + 404: NotFoundError +} + +export type SessionProviderResolutionListError = + SessionProviderResolutionListErrors[keyof SessionProviderResolutionListErrors] + +export type SessionProviderResolutionListResponses = { + /** + * Pending provider recovery decisions + */ + 200: Array<{ + receiptID: string + sessionID: string + assistantMessageID: string + providerID: string + modelID: string + providerState: "indeterminate_after_crash" + promptEpoch: number + promptWindowID: string + historyHash: string + requestHash: string + sessionMutationEpoch: number + continuationRecoverySupported: boolean + workspaceRecoverySupported: boolean + sourceWorldStateBaselineStatus: "available" | "missing" | "invalid" + worldStateBaselineHash?: string + }> +} + +export type SessionProviderResolutionListResponse = + SessionProviderResolutionListResponses[keyof SessionProviderResolutionListResponses] + +export type SessionProviderResolutionResolveData = { + body?: { + commandID: string + receiptID: string + decision: "abandoned" + expected: { + providerState: "indeterminate_after_crash" + promptEpoch: number + sessionMutationEpoch: number + requestHash: string + historyHash: string + worldStateBaselineHash: string + } + reason?: string + riskAcknowledged?: boolean + } + path: { + sessionID: string + } + query?: { + directory?: string + workspace?: string + } + url: "/session/{sessionID}/provider-resolution" +} + +export type SessionProviderResolutionResolveErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * NotFoundError + */ + 404: NotFoundError + /** + * ConflictError + */ + 409: ConflictError +} + +export type SessionProviderResolutionResolveError = + SessionProviderResolutionResolveErrors[keyof SessionProviderResolutionResolveErrors] + +export type SessionProviderResolutionResolveResponses = { + /** + * Resolved provider outcome + */ + 200: { + resolutionID: string + commandID: string + receiptID: string + sessionID: string + decision: "abandoned" + sourcePromptEpoch: number + successorPromptEpoch: number + sourceMutationEpoch: number + successorMutationEpoch: number + safeEndMessageID?: string + safeHistoryHash: string + successorWindowID: string + successorHistoryHash: string + createdAt: number + } +} + +export type SessionProviderResolutionResolveResponse = + SessionProviderResolutionResolveResponses[keyof SessionProviderResolutionResolveResponses] + export type SyncStartData = { body?: never path?: never @@ -14255,6 +15203,10 @@ export type SyncStealErrors = { * BadRequest | InvalidRequestError */ 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * ConflictError + */ + 409: ConflictError } export type SyncStealError = SyncStealErrors[keyof SyncStealErrors] @@ -14287,6 +15239,10 @@ export type SyncHistoryListErrors = { * BadRequest | InvalidRequestError */ 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError } export type SyncHistoryListError = SyncHistoryListErrors[keyof SyncHistoryListErrors] @@ -14909,6 +15865,10 @@ export type ExperimentalWorkspaceWarpErrors = { * NotFoundError */ 404: NotFoundError + /** + * VcsRawDiffError + */ + 503: VcsRawDiffError } export type ExperimentalWorkspaceWarpError = ExperimentalWorkspaceWarpErrors[keyof ExperimentalWorkspaceWarpErrors] From 084bf20e80e9376807a82c5606758ed2010f70c1 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 22:56:41 +0800 Subject: [PATCH 08/44] fix(sync): bound replay requests before decoding --- .../middleware/sync-replay-body-limit.ts | 22 ++++++++++--------- .../src/sync/replay-protocol.ts | 14 ++++++++++-- .../test/server/httpapi-sync.test.ts | 2 +- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts index efea76dca..92aceaaf4 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts @@ -13,22 +13,24 @@ export const syncReplayBodyLimitLayer = HttpRouter.middleware<{ handles: unknown if (Number.isFinite(contentLength) && contentLength > SyncReplayLimits.requestBytes) return HttpServerResponse.empty({ status: 413 }) - if (!(request.source instanceof Request)) - return yield* effect.pipe( - Effect.provideService(HttpServerRequest.MaxBodySize, FileSystem.Size(SyncReplayLimits.requestBytes)), - ) - - const buffered = yield* bufferWebBody(request.source) + const buffered = yield* ( + request.source instanceof Request + ? bufferWebBody(request.source) + : request.text.pipe( + Effect.provideService(HttpServerRequest.MaxBodySize, FileSystem.Size(SyncReplayLimits.requestBytes)), + Effect.map((body) => new TextEncoder().encode(body)), + Effect.catch(() => Effect.succeed(undefined)), + ) + ) if (!buffered) return HttpServerResponse.empty({ status: 413 }) return yield* effect.pipe( Effect.provideService( HttpServerRequest.HttpServerRequest, HttpServerRequest.fromWeb( - new Request(request.source.url, { - method: request.source.method, - headers: request.source.headers, + new Request(new URL(request.url, "http://localhost"), { + method: request.method, + headers: request.headers, body: buffered, - signal: request.source.signal, }), ), ), diff --git a/packages/deepagent-code/src/sync/replay-protocol.ts b/packages/deepagent-code/src/sync/replay-protocol.ts index bd61194e1..a2b84563b 100644 --- a/packages/deepagent-code/src/sync/replay-protocol.ts +++ b/packages/deepagent-code/src/sync/replay-protocol.ts @@ -18,7 +18,7 @@ export function encodeReplayRequestPrefix(directory: string, events: readonly Re const suffix = "]}" const prefixBytes = Buffer.byteLength(prefix) const suffixBytes = Buffer.byteLength(suffix) - const selected = events.reduce<{ + const selected = events.slice(0, SyncReplayLimits.events).reduce<{ events: ReplayEvent[] encoded: string[] dataBytes: number @@ -56,6 +56,16 @@ export function encodeReplayRequestPrefix(directory: string, events: readonly Re dataBytes: selected.dataBytes, eventsBytes, requestBytes, - complete: selected.events.length === events.length, + complete: + selected.events.length === events.length && + directory.length <= SyncReplayLimits.directoryCharacters && + selected.events.every( + (event) => + event.id.length <= SyncReplayLimits.eventIDCharacters && + event.aggregateID.length > 0 && + event.aggregateID.length <= SyncReplayLimits.aggregateIDCharacters && + event.type.length > 0 && + event.type.length <= SyncReplayLimits.typeCharacters, + ), } } diff --git a/packages/deepagent-code/test/server/httpapi-sync.test.ts b/packages/deepagent-code/test/server/httpapi-sync.test.ts index 205d28e8f..e4a07792c 100644 --- a/packages/deepagent-code/test/server/httpapi-sync.test.ts +++ b/packages/deepagent-code/test/server/httpapi-sync.test.ts @@ -536,7 +536,7 @@ describe("sync HttpApi", () => { } as RequestInit), ) - expect(response.status).toBe(400) + expect(response.status).toBe(413) expect(pulls).toBeLessThan(64) expect(yield* db.select({ id: EventTable.id }).from(EventTable).all().pipe(Effect.orDie)).toEqual(before) }), From e6596d1521ec6c75914c1ce6f5284f60fb8844a2 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 23:05:31 +0800 Subject: [PATCH 09/44] test(session): cover rejected question lifecycle --- .../live-llm/activity-progress-oracle.ts | 43 ++++- .../script/live-llm/dispatcher.ts | 4 + .../deepagent-code/script/live-llm/routes.ts | 13 +- .../deepagent-code/script/live-llm/runtime.ts | 12 ++ .../live-llm-activity-progress-oracle.test.ts | 39 ++++- .../test/script/live-llm-routes.test.ts | 28 +++- .../desktop/scripts/live-llm/long-session.ts | 148 +++++++++++++++++- packages/desktop/scripts/live-llm/runtime.ts | 2 +- 8 files changed, 277 insertions(+), 12 deletions(-) diff --git a/packages/deepagent-code/script/live-llm/activity-progress-oracle.ts b/packages/deepagent-code/script/live-llm/activity-progress-oracle.ts index 0ec9f4a75..52d0aacd6 100644 --- a/packages/deepagent-code/script/live-llm/activity-progress-oracle.ts +++ b/packages/deepagent-code/script/live-llm/activity-progress-oracle.ts @@ -16,6 +16,23 @@ type ActivityDurability = { state: string terminal_reason: string | null }> + legacyActivityRuns: ReadonlyArray<{ + run_id: string + activity_id: string + owner_token: string + state: string + terminal_reason: string | null + }> + legacyActivityTerminals: ReadonlyArray<{ + activity_id: string + state: string + reason_code: string + source: string + run_id: string | null + progress_revision: number | null + membership_ordinal: number + owner_token: string + }> legacyActivityAdmissions: ReadonlyArray<{ activity_id: string admission_id: string @@ -27,6 +44,7 @@ type ActivityDurability = { revision: number assistant_message_id: string provider_receipt_id: string + input_membership_ordinal: number state: string }> activityTextParts: ReadonlyArray<{ @@ -108,7 +126,7 @@ export function assertActivityProgressObservation(input: { durability.legacyActivities.length !== 1 || !activity || activity.state !== "settled" || - activity.terminal_reason !== "stop" || + activity.terminal_reason !== "assistant_completed" || activity.owner_token.length === 0 || activity.owner_token === "pre-owner-migration" ) { @@ -146,6 +164,29 @@ export function assertActivityProgressObservation(input: { if (progress.some((item) => !receiptIDs.has(item.provider_receipt_id))) { throw new Error(`${input.caseName} progress row lacked a dispatched provider receipt`) } + const run = durability.legacyActivityRuns[0] + const terminal = durability.legacyActivityTerminals[0] + const final = progress.at(-1) + if ( + durability.legacyActivityRuns.length !== 1 || + !run || + run.activity_id !== activity.activity_id || + run.owner_token !== activity.owner_token || + run.state !== "completed" || + run.terminal_reason !== "assistant_completed" || + durability.legacyActivityTerminals.length !== 1 || + !terminal || + terminal.activity_id !== activity.activity_id || + terminal.state !== "settled" || + terminal.reason_code !== "assistant_completed" || + terminal.source !== "provider_final" || + terminal.run_id !== run.run_id || + terminal.progress_revision !== final?.revision || + terminal.membership_ordinal !== final?.input_membership_ordinal || + terminal.owner_token !== activity.owner_token + ) { + throw new Error(`${input.caseName} lacked one matching run and terminal receipt`) + } progress.forEach((item) => { const parts = durability.activityTextParts.filter((part) => part.message_id === item.assistant_message_id) parts.forEach((part) => { diff --git a/packages/deepagent-code/script/live-llm/dispatcher.ts b/packages/deepagent-code/script/live-llm/dispatcher.ts index 5f7e6cbe5..771046b10 100644 --- a/packages/deepagent-code/script/live-llm/dispatcher.ts +++ b/packages/deepagent-code/script/live-llm/dispatcher.ts @@ -223,6 +223,10 @@ const modelCommands = new Map([ "ext:renderer-ui:activity-progress-package", command("packages/desktop", "bun", "run", "test:llm-release:activity-progress-package"), ], + [ + "ext:packaged-sidecar:long-session", + command("packages/desktop", "bun", "run", "test:llm-release:long-session"), + ], [ "ext:legacy-session:subagent-worktree-routing", command("packages/deepagent-code", "bun", "run", "test:llm-ext:subagent-worktree"), diff --git a/packages/deepagent-code/script/live-llm/routes.ts b/packages/deepagent-code/script/live-llm/routes.ts index 3003b5210..e25db03f8 100644 --- a/packages/deepagent-code/script/live-llm/routes.ts +++ b/packages/deepagent-code/script/live-llm/routes.ts @@ -27,6 +27,7 @@ export const modelSuites = [ "packaged-sidecar", "desktop-ui", "desktop-subagents", + "long-session", "shell-exit-contract", "stale-validation", "continuation-repetition", @@ -112,6 +113,7 @@ const degeneration = modelRun("live", "legacy-session", "degeneration") const activityProgressLifecycle = modelRun("live", "legacy-session", "activity-progress-lifecycle") const activityProgressRestart = modelRun("live", "packaged-sidecar", "activity-progress-restart") const activityProgressPackage = modelRun("ext", "renderer-ui", "activity-progress-package") +const longSession = modelRun("ext", "packaged-sidecar", "long-session") const finalizerIsolation = modelRun("ext", "legacy-session", "subagent-finalizer-isolation") const steerBoundary = modelRun("live", "legacy-session", "steer-boundary") const worktreeRouting = modelRun("ext", "legacy-session", "subagent-worktree-routing") @@ -151,6 +153,7 @@ const allHarnessRuns = [ activityProgressLifecycle, activityProgressRestart, activityProgressPackage, + longSession, finalizerIsolation, steerBoundary, worktreeRouting, @@ -458,7 +461,8 @@ export const routeManifest = [ { id: "live-llm-long-session-harness", paths: ["packages/desktop/scripts/live-llm/long-session.ts"], - checks: ["desktop-runtime"], + checks: ["desktop-runtime", "prompt-intent", "session-continuation"], + runs: [longSession], }, { id: "live-llm-headless-long-session-harness", @@ -658,16 +662,21 @@ export const routeManifest = [ "packages/app/src/pages/session/message-timeline.data.ts", "packages/core/src/database/migration/20260811090000_legacy_activity_progress.ts", "packages/core/src/database/migration/20260811100000_legacy_activity_owner.ts", + "packages/core/src/database/migration/20260812130000_legacy_activity_lifecycle_expand.ts", "packages/deepagent-code/src/session/activity-crash-test.ts", "packages/deepagent-code/src/session/activity-owner.ts", "packages/deepagent-code/src/session/activity-sql.ts", + "packages/deepagent-code/src/session/legacy-provider-resolution.ts", + "packages/deepagent-code/src/session/processor.ts", + "packages/deepagent-code/src/session/recovery-transfer-guard.ts", + "packages/deepagent-code/src/session/run-state.ts", "packages/deepagent-code/src/session/message-v2.ts", "packages/deepagent-code/src/session/prompt-intent.ts", "packages/deepagent-code/src/session/prompt.ts", "packages/deepagent-code/src/session/steer.ts", ], checks: ["live-llm-routes", "prompt-intent", "session-continuation", "ui-runtime"], - runs: [activityProgressLifecycle, activityProgressRestart, activityProgressPackage], + runs: [activityProgressLifecycle, activityProgressRestart, activityProgressPackage, longSession], }, { id: "legacy-session-prompt", diff --git a/packages/deepagent-code/script/live-llm/runtime.ts b/packages/deepagent-code/script/live-llm/runtime.ts index dd0cbaa79..6324f8a71 100644 --- a/packages/deepagent-code/script/live-llm/runtime.ts +++ b/packages/deepagent-code/script/live-llm/runtime.ts @@ -259,7 +259,9 @@ export async function runLegacyLiveCases(input: { SessionActivityAdmissionTable, SessionActivityProgressTable, SessionLegacyActivityAdmissionTable, + SessionLegacyActivityRunTable, SessionLegacyActivityTable, + SessionLegacyActivityTerminalTable, } = await import("../../src/session/activity-sql") const { SessionPromptEpochTable } = await import("../../src/session/prompt-epoch.sql") const { PartTable, SessionIntentTable, SessionWorldStateBaselineTable } = await import( @@ -1174,6 +1176,16 @@ export async function runLegacyLiveCases(input: { .all() .pipe(Effect.orDie), legacyActivities, + legacyActivityRuns: (yield* database.db + .select() + .from(SessionLegacyActivityRunTable) + .all() + .pipe(Effect.orDie)).filter((run) => activityIDs.has(run.activity_id)), + legacyActivityTerminals: (yield* database.db + .select() + .from(SessionLegacyActivityTerminalTable) + .all() + .pipe(Effect.orDie)).filter((terminal) => activityIDs.has(terminal.activity_id)), legacyActivityAdmissions: (yield* database.db .select() .from(SessionLegacyActivityAdmissionTable) diff --git a/packages/deepagent-code/test/script/live-llm-activity-progress-oracle.test.ts b/packages/deepagent-code/test/script/live-llm-activity-progress-oracle.test.ts index 44b9b80d4..86547972f 100644 --- a/packages/deepagent-code/test/script/live-llm-activity-progress-oracle.test.ts +++ b/packages/deepagent-code/test/script/live-llm-activity-progress-oracle.test.ts @@ -32,6 +32,21 @@ describe("activity progress live oracle", () => { }), ).toThrow("lacked the durable progress marker") }) + + test("rejects settled progress without one matching terminal receipt", () => { + const value = observation() + value.durability.legacyActivityTerminals = [] + expect(() => + assertActivityProgressObservation({ + caseName: "missing-terminal", + triggerText, + steerText, + marker: "MARKER", + expectedTools: ["read", "read"], + observation: value, + }), + ).toThrow("lacked one matching run and terminal receipt") + }) }) function observation() { @@ -62,7 +77,28 @@ function observation() { activity_id: "activity_1", owner_token: "123:owner", state: "settled", - terminal_reason: "stop", + terminal_reason: "assistant_completed", + }, + ], + legacyActivityRuns: [ + { + run_id: "run_1", + activity_id: "activity_1", + owner_token: "123:owner", + state: "completed", + terminal_reason: "assistant_completed", + }, + ], + legacyActivityTerminals: [ + { + activity_id: "activity_1", + state: "settled", + reason_code: "assistant_completed", + source: "provider_final", + run_id: "run_1", + progress_revision: 2, + membership_ordinal: 1, + owner_token: "123:owner", }, ], legacyActivityAdmissions: [ @@ -84,6 +120,7 @@ function observation() { revision, assistant_message_id: `assistant_${revision}`, provider_receipt_id: `receipt_${revision}`, + input_membership_ordinal: revision === 0 ? 0 : 1, state: revision === 2 ? "final" : "progress", })), activityTextParts: [0, 1, 2].flatMap((revision) => diff --git a/packages/deepagent-code/test/script/live-llm-routes.test.ts b/packages/deepagent-code/test/script/live-llm-routes.test.ts index a3f134ab5..4f4f0cd11 100644 --- a/packages/deepagent-code/test/script/live-llm-routes.test.ts +++ b/packages/deepagent-code/test/script/live-llm-routes.test.ts @@ -151,6 +151,7 @@ describe("live LLM route manifest", () => { "ext:legacy-session:subagent-resume", "ext:legacy-session:subagent-takeover", "ext:legacy-session:subagent-worktree-routing", + "ext:packaged-sidecar:long-session", "ext:renderer-ui:activity-progress-package", "ext:v4-event-runtime:v4-multi-agent-runtime", "live:adapter:provider-smoke", @@ -203,7 +204,7 @@ describe("live LLM route manifest", () => { }, { path: "packages/desktop/scripts/live-llm/long-session.ts", - runs: [], + runs: ["ext:packaged-sidecar:long-session"], }, { path: "packages/deepagent-code/script/live-llm/autonomous-eval.ts", @@ -356,6 +357,7 @@ describe("live LLM route manifest", () => { expect(selected.runs.map(modelRunKey)).toContain("live:packaged-sidecar:activity-progress-restart") expect(selected.runs.map(modelRunKey)).toContain("ext:renderer-ui:activity-progress-package") + expect(selected.runs.map(modelRunKey)).toContain("ext:packaged-sidecar:long-session") expect( commandForModelRun( selected.runs.find((run) => modelRunKey(run) === "live:packaged-sidecar:activity-progress-restart")!, @@ -372,6 +374,30 @@ describe("live LLM route manifest", () => { cwd: "packages/desktop", args: ["bun", "run", "test:llm-release:activity-progress-package"], }) + expect( + commandForModelRun(selected.runs.find((run) => modelRunKey(run) === "ext:packaged-sidecar:long-session")!), + ).toEqual({ + cwd: "packages/desktop", + args: ["bun", "run", "test:llm-release:long-session"], + }) + }) + + test("routes question-rejection and provider-recovery seams through the real Desktop continuation gate", () => { + for (const path of [ + "packages/deepagent-code/src/session/legacy-provider-resolution.ts", + "packages/deepagent-code/src/session/processor.ts", + "packages/deepagent-code/src/session/recovery-transfer-guard.ts", + "packages/desktop/scripts/live-llm/long-session.ts", + ]) { + const run = selectRoutes([path]).runs.find( + (item) => modelRunKey(item) === "ext:packaged-sidecar:long-session", + ) + expect(run).toBeDefined() + expect(commandForModelRun(run!)).toEqual({ + cwd: "packages/desktop", + args: ["bun", "run", "test:llm-release:long-session"], + }) + } }) test("keeps bounded takeover reachable from its harness and supervision seams", () => { diff --git a/packages/desktop/scripts/live-llm/long-session.ts b/packages/desktop/scripts/live-llm/long-session.ts index c73250f61..96cadf04a 100644 --- a/packages/desktop/scripts/live-llm/long-session.ts +++ b/packages/desktop/scripts/live-llm/long-session.ts @@ -2,6 +2,7 @@ import { strict as assert } from "node:assert" import { createHash, randomUUID } from "node:crypto" import { writeFile } from "node:fs/promises" import path from "node:path" +import { DatabaseSync } from "node:sqlite" import { assertModel, assertNoPermissionRequests, @@ -71,10 +72,12 @@ try { assert.ok(summary) assert.match(visibleText([summary]), new RegExp(marker)) + const questionPrompt = + "Call question exactly once to ask whether the retained release proof should be reported, with one Continue option. Wait for the answer and do not call other tools." const interruptedBefore = await startPrompt( runtime, session.id, - "Call question exactly once to ask whether the retained release proof should be reported, with one Continue option. Wait for the answer and do not call other tools.", + questionPrompt, "live-long", ) const question = await waitFor( @@ -82,7 +85,7 @@ try { "long-session Question latch", config.timeoutMs, ) - await request(runtime, `/session/${session.id}/abort`, { method: "POST" }) + await request(runtime, `/question/${question.id}/reject`, { method: "POST" }) await waitFor( async () => { const [all, statuses, pending] = await Promise.all([ @@ -93,19 +96,36 @@ try { const interrupted = all .filter((message) => !interruptedBefore.has(message.info.id)) .flatMap((message) => message.parts) - .some((part) => part.type === "tool" && part.tool === "question" && part.state.status === "error") + .some( + (part) => + part.type === "tool" && + part.tool === "question" && + part.state.status === "error" && + part.state.metadata?.failureCode === "user_rejected_question", + ) if (statuses[session.id]?.type !== "busy" && !pending.some((item) => item.id === question.id) && interrupted) { return true } }, - "interrupted Session terminal state", + "rejected Question terminal state", config.timeoutMs, ) + const rejected = inspectLatestActivity(path.join(runtime.root, "deepagent.sqlite"), session.id) + assertLifecycle(rejected, { + activity: "interrupted", + reason: "user_rejected_question", + run: "interrupted", + terminal: "interrupted", + source: "host_stop", + progress: "progress", + }) + const continuationPrompt = + "Continue after the rejected question. Without calling any tool, report the exact release proof retained from before compaction." const finalBefore = await startPrompt( runtime, session.id, - "Continue after the interruption. Without calling any tool, report the exact release proof retained from before compaction.", + continuationPrompt, "live-long", ) const finalMessages = await waitForPrompt(runtime, session.id, finalBefore, "post-interruption continuation") @@ -115,6 +135,25 @@ try { assert.equal(tools(finalTurn).length, 0) await assertNoPermissionRequests(runtime) assert.deepEqual(await request(runtime, "/question"), []) + const completed = inspectLatestActivity(path.join(runtime.root, "deepagent.sqlite"), session.id) + assert.notEqual(completed.activity_id, rejected.activity_id) + assertLifecycle(completed, { + activity: "settled", + reason: "assistant_completed", + run: "completed", + terminal: "settled", + source: "provider_final", + progress: "final", + }) + const allMessages = await messages(runtime, session.id) + assert.equal( + allMessages.filter((message) => message.info.role === "user" && visibleText([message]) === questionPrompt).length, + 1, + ) + assert.equal( + allMessages.filter((message) => message.info.role === "user" && visibleText([message]) === continuationPrompt).length, + 1, + ) await writeArtifact(suite, { suite, @@ -134,7 +173,9 @@ try { manualCompactionPersisted: true, summaryRetainedMarker: true, questionCount: question.questions.length, - interruptedTerminalObserved: true, + questionRejectedThroughProductionRoute: true, + rejectedActivity: lifecycleEvidence(rejected), + continuationActivity: lifecycleEvidence(completed), continuationUsedNoTools: true, pendingPermissions: 0, pendingQuestions: 0, @@ -150,3 +191,98 @@ try { } finally { await closeAll() } + +type Lifecycle = ReturnType + +function inspectLatestActivity(databasePath: string, sessionID: string) { + const database = new DatabaseSync(databasePath, { readOnly: true }) + try { + database.exec("PRAGMA busy_timeout = 5000") + const activity = database + .prepare( + `SELECT + activity_id, + state, + terminal_reason + FROM session_legacy_activity + WHERE session_id = ? + ORDER BY ordinal DESC + LIMIT 1`, + ) + .get(sessionID) as { activity_id: string; state: string; terminal_reason: string | null } | undefined + assert.ok(activity) + const runs = database + .prepare( + `SELECT run_id, state, terminal_reason + FROM session_legacy_activity_run + WHERE activity_id = ? + ORDER BY generation`, + ) + .all(activity.activity_id) as Array<{ run_id: string; state: string; terminal_reason: string | null }> + const terminals = database + .prepare( + `SELECT state, reason_code, source, run_id, progress_revision + FROM session_legacy_activity_terminal + WHERE activity_id = ?`, + ) + .all(activity.activity_id) as Array<{ + state: string + reason_code: string + source: string + run_id: string | null + progress_revision: number | null + }> + const progress = database + .prepare( + `SELECT revision, state, provider_receipt_id + FROM session_activity_progress + WHERE activity_id = ? + ORDER BY revision`, + ) + .all(activity.activity_id) as Array<{ revision: number; state: string; provider_receipt_id: string }> + return { ...activity, runs, terminals, progress } + } finally { + database.close() + } +} + +function assertLifecycle( + lifecycle: Lifecycle, + expected: { + activity: string + reason: string + run: string + terminal: string + source: string + progress: string + }, +) { + assert.equal(lifecycle.state, expected.activity) + assert.equal(lifecycle.terminal_reason, expected.reason) + assert.equal(lifecycle.runs.length, 1) + assert.equal(lifecycle.runs[0]?.state, expected.run) + assert.equal(lifecycle.runs[0]?.terminal_reason, expected.reason) + assert.equal(lifecycle.terminals.length, 1) + assert.equal(lifecycle.terminals[0]?.state, expected.terminal) + assert.equal(lifecycle.terminals[0]?.reason_code, expected.reason) + assert.equal(lifecycle.terminals[0]?.source, expected.source) + assert.equal(lifecycle.terminals[0]?.run_id, lifecycle.runs[0]?.run_id) + assert.equal(lifecycle.progress.length, 1) + assert.equal(lifecycle.progress[0]?.revision, 0) + assert.equal(lifecycle.progress[0]?.state, expected.progress) + assert.equal(lifecycle.terminals[0]?.progress_revision, lifecycle.progress[0]?.revision) + assert.equal(new Set(lifecycle.progress.map((item) => item.provider_receipt_id)).size, 1) +} + +function lifecycleEvidence(lifecycle: Lifecycle) { + return { + activityIDHash: createHash("sha256").update(lifecycle.activity_id).digest("hex"), + activityState: lifecycle.state, + terminalReason: lifecycle.terminal_reason, + runState: lifecycle.runs[0]?.state, + terminalState: lifecycle.terminals[0]?.state, + terminalSource: lifecycle.terminals[0]?.source, + progressStates: lifecycle.progress.map((item) => `${item.revision}:${item.state}`), + providerDispatches: new Set(lifecycle.progress.map((item) => item.provider_receipt_id)).size, + } +} diff --git a/packages/desktop/scripts/live-llm/runtime.ts b/packages/desktop/scripts/live-llm/runtime.ts index 873b06bda..c29b41dd5 100644 --- a/packages/desktop/scripts/live-llm/runtime.ts +++ b/packages/desktop/scripts/live-llm/runtime.ts @@ -54,7 +54,7 @@ export type Message = { | { status: "pending"; input?: unknown } | { status: "running"; input: unknown } | { status: "completed"; input: unknown; output: string; metadata: Record } - | { status: "error"; input: unknown; error: string } + | { status: "error"; input: unknown; error: string; metadata?: Record } } | { type: string } > From 0170c0fa7ab7ec35c7dea33b9335d7a26c26bd7c Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 23:16:39 +0800 Subject: [PATCH 10/44] test(desktop): gate rejected question through packaged UI --- .../composer/session-question-dock.tsx | 9 ++- .../script/live-llm/dispatcher.ts | 2 +- .../deepagent-code/script/live-llm/routes.ts | 3 +- .../test/script/live-llm-routes.test.ts | 11 +-- packages/desktop/package.json | 2 +- .../live-llm/activity-progress-package.ts | 25 +------ .../desktop/scripts/live-llm/long-session.ts | 71 ++++++++++++++----- packages/desktop/scripts/live-llm/runtime.ts | 24 ++++++- 8 files changed, 96 insertions(+), 51 deletions(-) diff --git a/packages/app/src/pages/session/composer/session-question-dock.tsx b/packages/app/src/pages/session/composer/session-question-dock.tsx index 0d54e1878..e02d4cfe9 100644 --- a/packages/app/src/pages/session/composer/session-question-dock.tsx +++ b/packages/app/src/pages/session/composer/session-question-dock.tsx @@ -451,7 +451,14 @@ export const SessionQuestionDock: Component<{ request: QuestionRequest; onSubmit } footer={ <> -
diff --git a/packages/deepagent-code/script/live-llm/dispatcher.ts b/packages/deepagent-code/script/live-llm/dispatcher.ts index 771046b10..9c139d684 100644 --- a/packages/deepagent-code/script/live-llm/dispatcher.ts +++ b/packages/deepagent-code/script/live-llm/dispatcher.ts @@ -224,7 +224,7 @@ const modelCommands = new Map([ command("packages/desktop", "bun", "run", "test:llm-release:activity-progress-package"), ], [ - "ext:packaged-sidecar:long-session", + "ext:renderer-ui:long-session", command("packages/desktop", "bun", "run", "test:llm-release:long-session"), ], [ diff --git a/packages/deepagent-code/script/live-llm/routes.ts b/packages/deepagent-code/script/live-llm/routes.ts index e25db03f8..c3bb03bef 100644 --- a/packages/deepagent-code/script/live-llm/routes.ts +++ b/packages/deepagent-code/script/live-llm/routes.ts @@ -113,7 +113,7 @@ const degeneration = modelRun("live", "legacy-session", "degeneration") const activityProgressLifecycle = modelRun("live", "legacy-session", "activity-progress-lifecycle") const activityProgressRestart = modelRun("live", "packaged-sidecar", "activity-progress-restart") const activityProgressPackage = modelRun("ext", "renderer-ui", "activity-progress-package") -const longSession = modelRun("ext", "packaged-sidecar", "long-session") +const longSession = modelRun("ext", "renderer-ui", "long-session") const finalizerIsolation = modelRun("ext", "legacy-session", "subagent-finalizer-isolation") const steerBoundary = modelRun("live", "legacy-session", "steer-boundary") const worktreeRouting = modelRun("ext", "legacy-session", "subagent-worktree-routing") @@ -660,6 +660,7 @@ export const routeManifest = [ id: "activity-progress-lifecycle-production", paths: [ "packages/app/src/pages/session/message-timeline.data.ts", + "packages/app/src/pages/session/composer/session-question-dock.tsx", "packages/core/src/database/migration/20260811090000_legacy_activity_progress.ts", "packages/core/src/database/migration/20260811100000_legacy_activity_owner.ts", "packages/core/src/database/migration/20260812130000_legacy_activity_lifecycle_expand.ts", diff --git a/packages/deepagent-code/test/script/live-llm-routes.test.ts b/packages/deepagent-code/test/script/live-llm-routes.test.ts index 4f4f0cd11..c2b027a68 100644 --- a/packages/deepagent-code/test/script/live-llm-routes.test.ts +++ b/packages/deepagent-code/test/script/live-llm-routes.test.ts @@ -151,8 +151,8 @@ describe("live LLM route manifest", () => { "ext:legacy-session:subagent-resume", "ext:legacy-session:subagent-takeover", "ext:legacy-session:subagent-worktree-routing", - "ext:packaged-sidecar:long-session", "ext:renderer-ui:activity-progress-package", + "ext:renderer-ui:long-session", "ext:v4-event-runtime:v4-multi-agent-runtime", "live:adapter:provider-smoke", "live:adapter:structured-output", @@ -204,7 +204,7 @@ describe("live LLM route manifest", () => { }, { path: "packages/desktop/scripts/live-llm/long-session.ts", - runs: ["ext:packaged-sidecar:long-session"], + runs: ["ext:renderer-ui:long-session"], }, { path: "packages/deepagent-code/script/live-llm/autonomous-eval.ts", @@ -357,7 +357,7 @@ describe("live LLM route manifest", () => { expect(selected.runs.map(modelRunKey)).toContain("live:packaged-sidecar:activity-progress-restart") expect(selected.runs.map(modelRunKey)).toContain("ext:renderer-ui:activity-progress-package") - expect(selected.runs.map(modelRunKey)).toContain("ext:packaged-sidecar:long-session") + expect(selected.runs.map(modelRunKey)).toContain("ext:renderer-ui:long-session") expect( commandForModelRun( selected.runs.find((run) => modelRunKey(run) === "live:packaged-sidecar:activity-progress-restart")!, @@ -375,7 +375,7 @@ describe("live LLM route manifest", () => { args: ["bun", "run", "test:llm-release:activity-progress-package"], }) expect( - commandForModelRun(selected.runs.find((run) => modelRunKey(run) === "ext:packaged-sidecar:long-session")!), + commandForModelRun(selected.runs.find((run) => modelRunKey(run) === "ext:renderer-ui:long-session")!), ).toEqual({ cwd: "packages/desktop", args: ["bun", "run", "test:llm-release:long-session"], @@ -387,10 +387,11 @@ describe("live LLM route manifest", () => { "packages/deepagent-code/src/session/legacy-provider-resolution.ts", "packages/deepagent-code/src/session/processor.ts", "packages/deepagent-code/src/session/recovery-transfer-guard.ts", + "packages/app/src/pages/session/composer/session-question-dock.tsx", "packages/desktop/scripts/live-llm/long-session.ts", ]) { const run = selectRoutes([path]).runs.find( - (item) => modelRunKey(item) === "ext:packaged-sidecar:long-session", + (item) => modelRunKey(item) === "ext:renderer-ui:long-session", ) expect(run).toBeDefined() expect(commandForModelRun(run!)).toEqual({ diff --git a/packages/desktop/package.json b/packages/desktop/package.json index ee70487a3..beb748aca 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -28,7 +28,7 @@ "test:llm-release:ui": "bun run build && node --experimental-strip-types ./scripts/live-llm/desktop-ui.ts", "test:llm-live:activity-progress-restart": "bun run build && node --experimental-strip-types ./scripts/live-llm/activity-progress-restart.ts", "test:llm-release:activity-progress-package": "bun run build && DEEPAGENT_CODE_ALLOW_UNSIGNED=1 bun run package:mac && node --experimental-strip-types ./scripts/live-llm/activity-progress-package.ts", - "test:llm-release:long-session": "bun run build && node --experimental-strip-types ./scripts/live-llm/long-session.ts", + "test:llm-release:long-session": "bun run build && DEEPAGENT_CODE_ALLOW_UNSIGNED=1 bun run package:mac && node --experimental-strip-types ./scripts/live-llm/long-session.ts", "test:llm-release:observed": "bun run build && node --experimental-strip-types ./scripts/live-llm/desktop-observed.ts", "package": "bun ./scripts/package.ts", "package:mac": "bun ./scripts/package.ts --mac", diff --git a/packages/desktop/scripts/live-llm/activity-progress-package.ts b/packages/desktop/scripts/live-llm/activity-progress-package.ts index 5bed179c9..2db356319 100644 --- a/packages/desktop/scripts/live-llm/activity-progress-package.ts +++ b/packages/desktop/scripts/live-llm/activity-progress-package.ts @@ -2,7 +2,7 @@ import { strict as assert } from "node:assert" import { createHash, randomUUID } from "node:crypto" import { execFile } from "node:child_process" import { createReadStream } from "node:fs" -import { readdir, rm, stat } from "node:fs/promises" +import { rm } from "node:fs/promises" import path from "node:path" import { fileURLToPath } from "node:url" import { promisify } from "node:util" @@ -11,6 +11,7 @@ import { close, closeAll, createSession, + findPackagedExecutable, focusSession, launch, loadLiveConfig, @@ -353,28 +354,6 @@ async function textCounts( } } -async function findPackagedExecutable(directory: string) { - const candidates: Array<{ path: string; modified: number }> = [] - async function visit(current: string) { - for (const entry of await readdir(current, { withFileTypes: true })) { - const target = path.join(current, entry.name) - if (entry.isDirectory()) { - await visit(target) - continue - } - const segments = path.relative(directory, target).split(path.sep) - const app = segments.findIndex((segment) => segment.endsWith(".app")) - if (app === -1 || segments[app + 1] !== "Contents" || segments[app + 2] !== "MacOS") continue - const info = await stat(target) - if ((info.mode & 0o111) !== 0) candidates.push({ path: target, modified: info.mtimeMs }) - } - } - await visit(directory) - const executable = candidates.sort((left, right) => right.modified - left.modified)[0]?.path - if (!executable) throw new Error(`No packaged macOS executable found under ${directory}`) - return executable -} - function hashFile(file: string) { return new Promise((resolve, reject) => { const hash = createHash("sha256") diff --git a/packages/desktop/scripts/live-llm/long-session.ts b/packages/desktop/scripts/live-llm/long-session.ts index 96cadf04a..3570e5187 100644 --- a/packages/desktop/scripts/live-llm/long-session.ts +++ b/packages/desktop/scripts/live-llm/long-session.ts @@ -1,14 +1,17 @@ import { strict as assert } from "node:assert" import { createHash, randomUUID } from "node:crypto" -import { writeFile } from "node:fs/promises" +import { mkdir, writeFile } from "node:fs/promises" import path from "node:path" import { DatabaseSync } from "node:sqlite" +import { fileURLToPath } from "node:url" import { assertModel, assertNoPermissionRequests, close, closeAll, createSession, + findPackagedExecutable, + focusSession, launch, loadLiveConfig, messages, @@ -28,10 +31,14 @@ const suite = "long-session" const config = await loadLiveConfig() const preflightResult = await preflight(config) const startedAt = Date.now() +const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..") +const executablePath = await findPackagedExecutable(path.join(packageRoot, "dist")) try { - const runtime = await launch(suite, config) + const runtime = await launch(suite, config, { executablePath }) try { + const packaged = await runtime.app.evaluate(({ app }) => ({ packaged: app.isPackaged, version: app.getVersion() })) + assert.equal(packaged.packaged, true) const marker = `long-session-${randomUUID()}` await writeFile(path.join(runtime.workspace, "memory.txt"), `${marker}\n`) const session = await createSession(runtime, "DeepSeek V4 Flash long Session", "live-long") @@ -71,21 +78,26 @@ try { .find((message) => message.info.role === "assistant" && message.info.time.completed !== undefined) assert.ok(summary) assert.match(visibleText([summary]), new RegExp(marker)) + await focusSession(runtime, session) + const pageErrors: string[] = [] + runtime.page.on("pageerror", (error) => pageErrors.push(error.stack ?? error.message)) + await runtime.page.reload({ waitUntil: "domcontentloaded" }) + await runtime.page + .getByRole("heading", { name: "DeepSeek V4 Flash long Session" }) + .waitFor({ state: "visible", timeout: 60_000 }) const questionPrompt = "Call question exactly once to ask whether the retained release proof should be reported, with one Continue option. Wait for the answer and do not call other tools." - const interruptedBefore = await startPrompt( - runtime, - session.id, - questionPrompt, - "live-long", - ) + const interruptedBefore = await submitThroughRenderer(runtime, session.id, questionPrompt) const question = await waitFor( async () => (await request(runtime, "/question")).find((item) => item.sessionID === session.id), "long-session Question latch", config.timeoutMs, ) - await request(runtime, `/question/${question.id}/reject`, { method: "POST" }) + await runtime.page.locator('[data-slot="question-text"]').waitFor({ state: "visible", timeout: 30_000 }) + const reject = runtime.page.locator('[data-action="question-reject"]') + assert.equal(await reject.count(), 1) + await reject.click() await waitFor( async () => { const [all, statuses, pending] = await Promise.all([ @@ -122,12 +134,7 @@ try { const continuationPrompt = "Continue after the rejected question. Without calling any tool, report the exact release proof retained from before compaction." - const finalBefore = await startPrompt( - runtime, - session.id, - continuationPrompt, - "live-long", - ) + const finalBefore = await submitThroughRenderer(runtime, session.id, continuationPrompt) const finalMessages = await waitForPrompt(runtime, session.id, finalBefore, "post-interruption continuation") const finalTurn = finalMessages.filter((message) => !finalBefore.has(message.info.id)) assertModel(finalMessages, config.modelID) @@ -154,11 +161,16 @@ try { allMessages.filter((message) => message.info.role === "user" && visibleText([message]) === continuationPrompt).length, 1, ) + assert.deepEqual(pageErrors, []) + const artifactDirectory = path.join(packageRoot, ".artifacts/live-llm") + await mkdir(artifactDirectory, { recursive: true }) + const screenshot = path.join(artifactDirectory, `${suite}.png`) + await runtime.page.screenshot({ path: screenshot, fullPage: false }) await writeArtifact(suite, { suite, - mode: "ext", - stack: "packaged-sidecar", + mode: "release", + stack: "packaged-renderer-ui", status: "passed", fingerprint: { providerID: "deepseek", @@ -168,18 +180,27 @@ try { baseURL: config.baseURL, }, preflight: preflightResult, + package: { + version: packaged.version, + executable: path.relative(packageRoot, executablePath), + isPackaged: packaged.packaged, + }, evidence: { markerHash: createHash("sha256").update(marker).digest("hex"), manualCompactionPersisted: true, summaryRetainedMarker: true, questionCount: question.questions.length, questionRejectedThroughProductionRoute: true, + questionRejectedThroughRenderer: true, rejectedActivity: lifecycleEvidence(rejected), continuationActivity: lifecycleEvidence(completed), continuationUsedNoTools: true, + rendererPromptSubmissions: 2, + rendererQuestionRejectClicks: 1, pendingPermissions: 0, pendingQuestions: 0, - humanReplies: 0, + pageErrors, + screenshot: path.basename(screenshot), }, durationMs: Date.now() - startedAt, completedAt: new Date().toISOString(), @@ -286,3 +307,17 @@ function lifecycleEvidence(lifecycle: Lifecycle) { providerDispatches: new Set(lifecycle.progress.map((item) => item.provider_receipt_id)).size, } } + +async function submitThroughRenderer(runtime: Parameters[0], sessionID: string, text: string) { + const before = new Set((await messages(runtime, sessionID)).map((message) => message.info.id)) + const editor = runtime.page.locator('[data-component="prompt-input"]') + await editor.waitFor({ state: "visible", timeout: 60_000 }) + const direct = runtime.page.locator('[data-action="prompt-scenario-direct"]') + if ((await direct.getAttribute("data-active")) !== "true") await direct.click() + await editor.fill(text) + assert.equal(await editor.textContent(), text) + const submit = runtime.page.locator('[data-action="prompt-submit"]') + assert.equal(await submit.isEnabled(), true) + await submit.click() + return before +} diff --git a/packages/desktop/scripts/live-llm/runtime.ts b/packages/desktop/scripts/live-llm/runtime.ts index c29b41dd5..25353ae68 100644 --- a/packages/desktop/scripts/live-llm/runtime.ts +++ b/packages/desktop/scripts/live-llm/runtime.ts @@ -1,5 +1,5 @@ import { strict as assert } from "node:assert" -import { mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from "node:fs/promises" +import { mkdir, mkdtemp, readFile, readdir, realpath, rm, stat, writeFile } from "node:fs/promises" import os from "node:os" import path from "node:path" import { fileURLToPath } from "node:url" @@ -405,6 +405,28 @@ export async function closeAll() { activeApps.clear() } +export async function findPackagedExecutable(directory = path.join(packageRoot, "dist")) { + const candidates: Array<{ path: string; modified: number }> = [] + async function visit(current: string) { + for (const entry of await readdir(current, { withFileTypes: true })) { + const target = path.join(current, entry.name) + if (entry.isDirectory()) { + await visit(target) + continue + } + const segments = path.relative(directory, target).split(path.sep) + const app = segments.findIndex((segment) => segment.endsWith(".app")) + if (app === -1 || segments[app + 1] !== "Contents" || segments[app + 2] !== "MacOS") continue + const info = await stat(target) + if ((info.mode & 0o111) !== 0) candidates.push({ path: target, modified: info.mtimeMs }) + } + } + await visit(directory) + const executable = candidates.sort((left, right) => right.modified - left.modified)[0]?.path + if (!executable) throw new Error(`No packaged macOS executable found under ${directory}`) + return executable +} + export async function request(runtime: Runtime, pathname: string, init?: RequestInit) { const url = new URL(pathname, runtime.server.url) url.searchParams.set("directory", runtime.workspace) From db480e9f410d312b3e5126588594d6b93f08db25 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 23:26:22 +0800 Subject: [PATCH 11/44] fix(session): recover production crash histories --- .../src/session/legacy-provider-resolution.ts | 6 +- .../deepagent-code/src/session/message-v2.ts | 57 ++++++++++++++--- .../deepagent-code/src/session/session.ts | 38 ++++++------ .../legacy-provider-resolution.test.ts | 54 +++++++++++++++- .../test/session/message-v2.test.ts | 11 ++++ .../test/session/session.test.ts | 62 +++++++++++++++++++ 6 files changed, 198 insertions(+), 30 deletions(-) diff --git a/packages/deepagent-code/src/session/legacy-provider-resolution.ts b/packages/deepagent-code/src/session/legacy-provider-resolution.ts index 3b2236815..10b4f1871 100644 --- a/packages/deepagent-code/src/session/legacy-provider-resolution.ts +++ b/packages/deepagent-code/src/session/legacy-provider-resolution.ts @@ -476,7 +476,11 @@ const service = Effect.gen(function* () { reason: "ambiguous provider turn is not contiguous", }) const safeMessages = cutoffAuthority.projection.messages - if (safeMessages.some((message) => message.info.role === "assistant" && !message.info.finish)) + if ( + safeMessages.some( + (message) => message.info.role === "assistant" && !message.info.finish && !message.info.error, + ) + ) return yield* new Conflict({ code: "unsafe_history_boundary", reason: "safe prefix contains unfinished assistant output", diff --git a/packages/deepagent-code/src/session/message-v2.ts b/packages/deepagent-code/src/session/message-v2.ts index 6989c851d..dde9f3433 100644 --- a/packages/deepagent-code/src/session/message-v2.ts +++ b/packages/deepagent-code/src/session/message-v2.ts @@ -319,7 +319,7 @@ END` export function clientProjection(message: T): T { if (message.role !== "user" || !message.summary) return message - const diffs = message.summary.diffs.slice(0, ClientDiffLimits.files).map((item) => ({ + const diffs = (message.summary.diffs ?? []).slice(0, ClientDiffLimits.files).map((item) => ({ ...(item.file === undefined ? {} : { file: item.file }), additions: item.additions, deletions: item.deletions, @@ -1338,10 +1338,25 @@ export function promptHistoryCutoffAuthorityInTransaction( if (!projection) return if (snapshotExit.value.epoch?.authority_state !== "recovery_required") return { projection, physical: snapshotExit.value.physical } + const epoch = snapshotExit.value.epoch + const selected = selectEpochHistory( + snapshotExit.value.chronological, + epoch, + snapshotExit.value.replacementMessageIDs, + snapshotExit.value.recoveryAdmittedMessageIDs, + ) + if (!selected.ok) return const unresolved = yield* tx .select({ userMessageID: SessionToolRequestReceiptTable.user_message_id, assistantMessageID: SessionToolRequestReceiptTable.assistant_message_id, + providerAttemptID: SessionToolRequestReceiptTable.provider_attempt_id, + promptEpoch: SessionToolRequestReceiptTable.prompt_epoch, + promptWindowID: SessionToolRequestReceiptTable.prompt_window_id, + effectiveHistoryHash: SessionToolRequestReceiptTable.effective_history_hash, + requestInputHash: SessionToolRequestReceiptTable.request_input_hash, + providerRequestHash: SessionToolRequestReceiptTable.provider_request_hash, + finalRequestHash: SessionToolRequestReceiptTable.final_request_hash, }) .from(SessionToolRequestReceiptTable) .leftJoin( @@ -1356,14 +1371,20 @@ export function promptHistoryCutoffAuthorityInTransaction( ), ) .all() - const cutoffIndex = snapshotExit.value.chronological.findIndex( - (message) => message.info.id === input.cutoffMessageID, - ) + const cutoffIndex = selected.messages.findIndex((message) => message.info.id === input.cutoffMessageID) const unsafe = unresolved.some((receipt) => { - const userIndex = snapshotExit.value.chronological.findIndex( - (message) => message.info.id === receipt.userMessageID, + const userIndex = selected.messages.findIndex((message) => message.info.id === receipt.userMessageID) + const assistantIndex = selected.messages.findIndex((message) => message.info.id === receipt.assistantMessageID) + return ( + receipt.providerAttemptID === null && + receipt.promptEpoch === epoch.epoch && + receipt.promptWindowID === epoch.window_id && + !!receipt.effectiveHistoryHash && + !!(receipt.finalRequestHash ?? receipt.providerRequestHash ?? receipt.requestInputHash) && + userIndex >= 0 && + assistantIndex > userIndex && + cutoffIndex > userIndex ) - return userIndex >= 0 && cutoffIndex > userIndex }) if (unsafe) return return { projection, physical: snapshotExit.value.physical } @@ -1382,6 +1403,28 @@ export function promptHistoryCutoffProjectionInTransaction( export const promptHistoryCutoffProjectionEffect = Effect.fn("MessageV2.promptHistoryCutoffProjection")( function* (input: { sessionID: SessionID; cutoffMessageID: MessageID }) { const { db } = yield* Database.Service + const authority = yield* db + .select({ state: SessionPromptEpochTable.authority_state }) + .from(SessionPromptEpochTable) + .where( + and( + eq(SessionPromptEpochTable.session_id, input.sessionID), + eq(SessionPromptEpochTable.state, "active"), + ), + ) + .get() + .pipe(Effect.orDie) + const history = yield* db + .select({ state: SessionHistoryStateTable.state }) + .from(SessionHistoryStateTable) + .where(eq(SessionHistoryStateTable.session_id, input.sessionID)) + .get() + .pipe(Effect.orDie) + if ( + (!authority || authority.state === "legacy_pending") && + history?.state !== "recovery_required" + ) + yield* promptHistoryProjectionEffect(input.sessionID) const projection = yield* db .transaction((tx) => promptHistoryCutoffProjectionInTransaction(tx as unknown as Database.Interface["db"], input)) .pipe(Effect.orDie) diff --git a/packages/deepagent-code/src/session/session.ts b/packages/deepagent-code/src/session/session.ts index 50eeb89e1..feaed6d38 100644 --- a/packages/deepagent-code/src/session/session.ts +++ b/packages/deepagent-code/src/session/session.ts @@ -2359,7 +2359,7 @@ export const layer: Layer.Layer< ) const cloned = sourceMessages.map((message) => { const id = messageIDMap.get(message.info.id)! - const info: SessionV1.Info = + const info = MessageV2.stripActivityProgress( message.info.role === "assistant" ? { ...message.info, @@ -2367,7 +2367,8 @@ export const layer: Layer.Layer< sessionID: targetSessionID, parentID: messageIDMap.get(message.info.parentID)!, } - : { ...message.info, id, sessionID: targetSessionID } + : { ...message.info, id, sessionID: targetSessionID }, + ) if (info.role === "assistant" && !info.parentID) { throw new ForkConflict({ intentID, @@ -2376,17 +2377,19 @@ export const layer: Layer.Layer< } return { info, - parts: message.parts.map((part) => ({ - ...part, - id: PartID.make( - `prt_${part.id.replace(/^prt_?/, "")}_${Hash.sha256(`fork-map:v1:${intentID}:${part.id}`).slice(0, 12)}`, - ), - messageID: id, - sessionID: targetSessionID, - ...(part.type === "compaction" - ? { tail_start_id: part.tail_start_id ? messageIDMap.get(part.tail_start_id) : undefined } - : {}), - })) as SessionV1.Part[], + parts: message.parts.map((part) => + MessageV2.stripActivityProgressPart({ + ...part, + id: PartID.make( + `prt_${part.id.replace(/^prt_?/, "")}_${Hash.sha256(`fork-map:v1:${intentID}:${part.id}`).slice(0, 12)}`, + ), + messageID: id, + sessionID: targetSessionID, + ...(part.type === "compaction" + ? { tail_start_id: part.tail_start_id ? messageIDMap.get(part.tail_start_id) : undefined } + : {}), + }), + ) as SessionV1.Part[], } }) const targetEffectiveHistoryHash = HistoryAuthority.hash(cloned) @@ -2513,24 +2516,21 @@ export const layer: Layer.Layer< time_created: message.info.time.created, time_updated: message.info.time.created, data: Object.fromEntries( - Object.entries(MessageV2.stripActivityProgress(message.info)).filter( - ([key]) => key !== "id" && key !== "sessionID", - ), + Object.entries(message.info).filter(([key]) => key !== "id" && key !== "sessionID"), ) as typeof MessageTable.$inferInsert.data, }) .run() for (const part of message.parts) { - const clonedPart = MessageV2.stripActivityProgressPart(part) yield* db .insert(PartTable) .values({ - id: clonedPart.id, + id: part.id, message_id: message.info.id, session_id: session.id, time_created: message.info.time.created, time_updated: message.info.time.created, data: Object.fromEntries( - Object.entries(clonedPart).filter( + Object.entries(part).filter( ([key]) => key !== "id" && key !== "messageID" && key !== "sessionID", ), ) as typeof PartTable.$inferInsert.data, diff --git a/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts b/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts index 68b49f6cb..a95b4ca7d 100644 --- a/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts +++ b/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts @@ -1,6 +1,7 @@ import { expect } from "bun:test" import { Database } from "@deepagent-code/core/database/database" import { ModelV2 } from "@deepagent-code/core/model" +import { NamedError } from "@deepagent-code/core/util/error" import { ProviderV2 } from "@deepagent-code/core/provider" import { SessionHistoryStateTable, @@ -97,6 +98,13 @@ it.instance( }) const safeUser = yield* addUser(source.id, "completed before provider crash") const safeAssistant = yield* addAssistant(source.id, safeUser.id, true) + const failedUser = yield* addUser(source.id, "failed safely before provider crash") + const failedAssistant = yield* addAssistant(source.id, failedUser.id) + yield* sessions.updateMessage({ + ...failedAssistant, + time: { ...failedAssistant.time, completed: Date.now() }, + error: new NamedError.Unknown({ message: "known terminal provider failure" }).toObject(), + }) const user = yield* addUser(source.id, "dispatch then crash") const assistant = yield* addAssistant(source.id, user.id) const unsafeTail = yield* addUser(source.id, "physical tail after ambiguous provider turn") @@ -139,6 +147,30 @@ it.instance( }) .run() .pipe(Effect.orDie) + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: "legacy-stale-indeterminate-without-prompt-authority", + request_ordinal: 2, + session_id: source.id, + user_message_id: safeUser.id, + assistant_message_id: safeAssistant.id, + provider_attempt_id: null, + provider_id: model.providerID, + model_id: model.modelID, + protocol: "chat", + registry_tool_ids: [], + permission_filtered_tool_ids: [], + final_offered_tool_ids: [], + call_ids: [], + provider_state: "indeterminate_after_crash", + terminal_at: now, + request_state: "dispatched", + request_error_code: "legacy_migration_without_prompt_authority", + created_at: now - 1, + }) + .run() + .pipe(Effect.orDie) yield* db .update(SessionPromptEpochTable) .set({ authority_state: "recovery_required", recovery_reason: "provider outcome is unknown after restart" }) @@ -171,6 +203,8 @@ it.instance( ).toEqual([ [safeUser.id, "user", undefined], [safeAssistant.id, "assistant", safeUser.id], + [failedUser.id, "user", undefined], + [failedAssistant.id, "assistant", failedUser.id], ]) const fork = yield* sessions.fork({ @@ -181,6 +215,8 @@ it.instance( expect((yield* sessions.messages({ sessionID: fork.id })).map((message) => message.info.role)).toEqual([ "user", "assistant", + "user", + "assistant", ]) const ambiguousFork = yield* sessions .fork({ @@ -376,7 +412,12 @@ it.instance( const projection = yield* MessageV2.promptHistoryProjectionEffect(source.id) expect(projection.epoch).toBe(authority.epoch + 1) - expect(projection.messages.map((message) => message.info.id)).toEqual([safeUser.id, safeAssistant.id]) + expect(projection.messages.map((message) => message.info.id)).toEqual([ + safeUser.id, + safeAssistant.id, + failedUser.id, + failedAssistant.id, + ]) expect(projection.recoveryResolutionID).toBe(first.resolutionID) expect( yield* db @@ -409,7 +450,12 @@ it.instance( .orderBy(SessionPromptEpochMessageTable.ordinal) .all() .pipe(Effect.orDie), - ).toEqual([{ messageID: safeUser.id }, { messageID: safeAssistant.id }]) + ).toEqual([ + { messageID: safeUser.id }, + { messageID: safeAssistant.id }, + { messageID: failedUser.id }, + { messageID: failedAssistant.id }, + ]) const successor = yield* db .select({ baselineHash: SessionPromptEpochTable.world_state_baseline_hash }) .from(SessionPromptEpochTable) @@ -473,7 +519,7 @@ it.instance( const continued = yield* addUser(source.id, "continue after explicit abandon") expect( (yield* MessageV2.promptHistoryProjectionEffect(source.id)).messages.map((message) => message.info.id), - ).toEqual([safeUser.id, safeAssistant.id]) + ).toEqual([safeUser.id, safeAssistant.id, failedUser.id, failedAssistant.id]) const admittedAt = Date.now() yield* db .insert(SessionIntentTable) @@ -501,6 +547,8 @@ it.instance( expect(continuedProjection.messages.map((message) => message.info.id)).toEqual([ safeUser.id, safeAssistant.id, + failedUser.id, + failedAssistant.id, continued.id, ]) yield* sessions.remove(source.id) diff --git a/packages/deepagent-code/test/session/message-v2.test.ts b/packages/deepagent-code/test/session/message-v2.test.ts index 492aea951..96c1c778d 100644 --- a/packages/deepagent-code/test/session/message-v2.test.ts +++ b/packages/deepagent-code/test/session/message-v2.test.ts @@ -112,6 +112,17 @@ function basePart(messageID: string, id: string) { } } +describe("session.message-v2.clientProjection", () => { + test("canonicalizes a bounded legacy summary without inline diffs", () => { + const message = { + ...userInfo("msg_manifest_only"), + summary: {}, + } as SessionV1.User + + expect(MessageV2.clientProjection(message).summary?.diffs).toEqual([]) + }) +}) + describe("session.message-v2.toModelMessage", () => { test("does not expose computed activity progress to the provider projection", async () => { const assistantID = "msg_activity_projection" diff --git a/packages/deepagent-code/test/session/session.test.ts b/packages/deepagent-code/test/session/session.test.ts index 164d17b7f..24ae2f7ac 100644 --- a/packages/deepagent-code/test/session/session.test.ts +++ b/packages/deepagent-code/test/session/session.test.ts @@ -3,6 +3,8 @@ import path from "path" import { SessionV1 } from "@deepagent-code/core/v1/session" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" +import { ModelV2 } from "@deepagent-code/core/model" +import { ProviderV2 } from "@deepagent-code/core/provider" import { SessionProjector } from "@deepagent-code/core/session/projector" import { FSUtil } from "@deepagent-code/core/fs-util" import { Deferred, Effect, Exit, Layer } from "effect" @@ -335,6 +337,66 @@ describe("Session", () => { }), ) + it.instance("hashes the same sanitized history that fork persistence and delivery project", () => + Effect.gen(function* () { + const session = yield* SessionNs.Service + const source = yield* Effect.acquireRelease(session.create({ title: "fork-progress-sanitization" }), (info) => + session.remove(info.id).pipe(Effect.ignore), + ) + const userID = MessageID.ascending() + yield* session.updateMessage({ + id: userID, + sessionID: source.id, + role: "user", + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("test") }, + }) + const assistantID = MessageID.ascending() + yield* session.updateMessage({ + id: assistantID, + sessionID: source.id, + role: "assistant", + parentID: userID, + time: { created: Date.now() }, + mode: "build", + agent: "build", + cost: 0, + path: { cwd: source.directory, root: source.directory }, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + modelID: ModelV2.ID.make("test"), + providerID: ProviderV2.ID.make("test"), + activityProgress: { activityID: "activity-fork-progress", revision: 1, state: "progress" }, + }) + yield* session.updatePart({ + id: PartID.ascending(), + sessionID: source.id, + messageID: assistantID, + type: "text", + text: "durable assistant content", + metadata: { + deepagent_activity_progress: { + activity_id: "activity-fork-progress", + revision: 1, + state: "progress", + }, + }, + }) + + const fork = yield* Effect.acquireRelease( + session.fork({ sessionID: source.id, intentID: "session-progress-sanitized-fork" }), + (info) => session.remove(info.id).pipe(Effect.ignore), + ) + yield* session.assertRunnable(fork.id) + const messages = yield* session.messages({ sessionID: fork.id }) + expect(messages).toHaveLength(2) + expect(messages[1]?.info).not.toHaveProperty("activityProgress") + const part = messages[1]?.parts[0] + expect(part?.type).toBe("text") + expect(part?.type === "text" ? part.metadata : undefined).not.toHaveProperty("deepagent_activity_progress") + }), + ) + it.instance("durably relocates a Session through the moved-event projector", () => Effect.gen(function* () { const sessions = yield* SessionNs.Service From 4c31f5f4a14cc19e6ca04541a0a5bbd6f8f22fba Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 23:28:00 +0800 Subject: [PATCH 12/44] fix(core): preserve cross-table triggers during recovery migration --- ...20260812120000_legacy_provider_recovery.ts | 71 ++++++++++++++++++- ...y-provider-recovery-delete-cascade.test.ts | 45 ++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts b/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts index 425f51b21..b00c708aa 100644 --- a/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts +++ b/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts @@ -7,7 +7,10 @@ export default { return Effect.gen(function* () { // The original receipt table predated Session FK ownership. Rebuild it with its two FK // children detached so SQLite cannot cascade evidence away or retain references to a renamed - // old table. Historical orphan receipts are intentionally omitted from the rebuilt authority. + // old table. The compaction trigger also references the receipt table from a different table; + // detach it so Node SQLite can validate the schema while the receipt table is being renamed. + // Historical orphan receipts are intentionally omitted from the rebuilt authority. + yield* tx.run("DROP TRIGGER IF EXISTS compaction_run_continuation_response_validate") yield* tx.run(` CREATE TEMP TABLE legacy_provider_recovery_argument_receipt AS SELECT * FROM session_tool_argument_receipt @@ -153,6 +156,19 @@ export default { SELECT RAISE(ABORT, 'provider response fingerprint requires terminal receipt'); END `) + yield* tx.run(` + CREATE TRIGGER compaction_run_continuation_response_validate + BEFORE UPDATE OF continuation_state ON compaction_run + WHEN NEW.continuation_state IN ('settled', 'failed') AND NOT EXISTS ( + SELECT 1 + FROM session_tool_request_receipt receipt + WHERE receipt.receipt_id = NEW.continuation_receipt_id + AND receipt.response_fingerprint IS NOT NULL + ) + BEGIN + SELECT RAISE(ABORT, 'compaction continuation response is not durable'); + END + `) yield* tx.run(` CREATE TABLE session_tool_argument_receipt ( receipt_id TEXT NOT NULL, @@ -262,6 +278,10 @@ export default { yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_message_validate_insert") yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_message_validate_update") yield* tx.run("DROP TRIGGER IF EXISTS session_prompt_epoch_message_owner_immutable") + yield* tx.run("DROP TRIGGER IF EXISTS session_history_state_ready_validate_insert") + yield* tx.run("DROP TRIGGER IF EXISTS session_history_state_ready_validate_update") + yield* tx.run("DROP TRIGGER IF EXISTS session_history_state_recovery_validate_insert") + yield* tx.run("DROP TRIGGER IF EXISTS session_history_state_recovery_validate_update") yield* tx.run(` CREATE TEMP TABLE legacy_provider_recovery_epoch_message AS SELECT * FROM session_prompt_epoch_message @@ -415,6 +435,55 @@ export default { END `) + yield* tx.run(` + CREATE TRIGGER session_history_state_ready_validate_insert + BEFORE INSERT ON session_history_state + WHEN NEW.state = 'ready' + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_prompt_epoch + WHERE session_id = NEW.session_id AND state = 'active' AND authority_state = 'ready' + ) THEN RAISE(ABORT, 'session_history_ready_without_authority') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_history_state_ready_validate_update + BEFORE UPDATE ON session_history_state + WHEN NEW.state = 'ready' + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_prompt_epoch + WHERE session_id = NEW.session_id AND state = 'active' AND authority_state = 'ready' + ) THEN RAISE(ABORT, 'session_history_ready_without_authority') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_history_state_recovery_validate_insert + BEFORE INSERT ON session_history_state + WHEN NEW.state = 'recovery_required' + BEGIN + SELECT CASE WHEN NEW.reason IS NULL + THEN RAISE(ABORT, 'session_history_recovery_without_reason') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM session_prompt_epoch + WHERE session_id = NEW.session_id AND state = 'active' AND authority_state != 'recovery_required' + ) THEN RAISE(ABORT, 'session_history_recovery_without_quarantined_authority') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_history_state_recovery_validate_update + BEFORE UPDATE ON session_history_state + WHEN NEW.state = 'recovery_required' + BEGIN + SELECT CASE WHEN NEW.reason IS NULL + THEN RAISE(ABORT, 'session_history_recovery_without_reason') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM session_prompt_epoch + WHERE session_id = NEW.session_id AND state = 'active' AND authority_state != 'recovery_required' + ) THEN RAISE(ABORT, 'session_history_recovery_without_quarantined_authority') END; + END + `) + yield* tx.run(` CREATE TABLE session_tool_request_resolution_command ( command_id TEXT NOT NULL PRIMARY KEY, diff --git a/packages/core/test/legacy-provider-recovery-delete-cascade.test.ts b/packages/core/test/legacy-provider-recovery-delete-cascade.test.ts index a98bb7efd..2a1594fb8 100644 --- a/packages/core/test/legacy-provider-recovery-delete-cascade.test.ts +++ b/packages/core/test/legacy-provider-recovery-delete-cascade.test.ts @@ -239,6 +239,51 @@ describe("legacy provider recovery delete cascade", () => { { name: "session_tool_request_receipt_provider_transition" }, { name: "session_tool_request_receipt_response_guard" }, ]) + expect( + yield* db.get(` + SELECT name FROM sqlite_master + WHERE type = 'trigger' AND name = 'compaction_run_continuation_response_validate' + `), + ).toEqual({ name: "compaction_run_continuation_response_validate" }) + expect( + yield* db.all(` + SELECT name FROM sqlite_master + WHERE type = 'trigger' AND name IN ( + 'session_history_state_ready_validate_insert', + 'session_history_state_ready_validate_update', + 'session_history_state_recovery_validate_insert', + 'session_history_state_recovery_validate_update' + ) ORDER BY name + `), + ).toEqual([ + { name: "session_history_state_ready_validate_insert" }, + { name: "session_history_state_ready_validate_update" }, + { name: "session_history_state_recovery_validate_insert" }, + { name: "session_history_state_recovery_validate_update" }, + ]) + yield* db.run(` + INSERT INTO compaction_run ( + run_id, session_id, from_prompt_epoch, trigger, state, created_at, + source_window_id, source_effective_history_hash, source_message_count, + source_projection_version, continuation_state, continuation_receipt_id, + continuation_admitted_at, continuation_dispatching_at + ) VALUES ( + 'compaction-receipt-upgrade', 'ses_receipt_upgrade', 0, 'manual', 'committed', 1, + 'window-upgrade', 'history-upgrade', 1, 1, 'dispatching', + 'receipt-upgrade-valid', 1, 1 + ) + `) + expect( + Exit.isFailure( + yield* db + .run(` + UPDATE compaction_run + SET continuation_state = 'settled', continuation_terminal_at = 2 + WHERE run_id = 'compaction-receipt-upgrade' + `) + .pipe(Effect.exit), + ), + ).toBe(true) expect( yield* db.all(` SELECT "table", "from", "to", on_delete From 0f9846abef94effa1348d0444eb38e15f56e6e5b Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 23:30:52 +0800 Subject: [PATCH 13/44] fix(sync): enforce final replay request budget --- .../src/control-plane/workspace.ts | 4 +- .../routes/instance/httpapi/handlers/sync.ts | 2 +- .../middleware/sync-replay-body-limit.ts | 10 +- .../src/sync/replay-protocol.ts | 25 ++--- .../test/control-plane/workspace.test.ts | 94 ++++++++++++++++++- .../test/server/httpapi-sync.test.ts | 22 ++++- 6 files changed, 138 insertions(+), 19 deletions(-) diff --git a/packages/deepagent-code/src/control-plane/workspace.ts b/packages/deepagent-code/src/control-plane/workspace.ts index 1f1da2038..1d64b9f55 100644 --- a/packages/deepagent-code/src/control-plane/workspace.ts +++ b/packages/deepagent-code/src/control-plane/workspace.ts @@ -879,9 +879,9 @@ export const layer = Layer.effect( .all() .pipe(Effect.orDie) const encoded = encodeReplayRequestPrefix(space.directory ?? "", events) - if (encoded.events.length === 0) + if (!encoded.valid || encoded.events.length === 0) return yield* new SessionWarpHistoryLimitError({ - message: `Sync event ${events[0]!.id} cannot fit the bounded replay request`, + message: `Sync event ${events[0]!.id} cannot fit the bounded replay protocol`, sessionID: input.sessionID, eventID: events[0]!.id, bytes: Buffer.byteLength( diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts index d9dfe43f1..d36ce314e 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts @@ -19,7 +19,7 @@ import { sql } from "drizzle-orm" import { Effect, Option, Schema, Scope } from "effect" import { HttpApiBuilder, HttpApiError } from "effect/unstable/httpapi" import { InstanceHttpApi } from "../api" -import { HistoryPayload, ReplayPayload, SessionPayload, SyncReplayLimits } from "../groups/sync" +import { HistoryPayload, ReplayPayload, SessionPayload } from "../groups/sync" import { encodeReplayRequestPrefix } from "@/sync/replay-protocol" import * as Log from "@deepagent-code/core/util/log" import { ServiceUnavailableError } from "../errors" diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts index 92aceaaf4..70ea8c5d8 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/middleware/sync-replay-body-limit.ts @@ -19,7 +19,9 @@ export const syncReplayBodyLimitLayer = HttpRouter.middleware<{ handles: unknown : request.text.pipe( Effect.provideService(HttpServerRequest.MaxBodySize, FileSystem.Size(SyncReplayLimits.requestBytes)), Effect.map((body) => new TextEncoder().encode(body)), - Effect.catch(() => Effect.succeed(undefined)), + Effect.catch((error) => + isBodyLimitError(error) ? Effect.succeed(undefined) : Effect.fail(error), + ), ) ) if (!buffered) return HttpServerResponse.empty({ status: 413 }) @@ -45,6 +47,12 @@ function pathOf(url: string) { return end === -1 ? url : url.slice(0, end) } +function isBodyLimitError(error: unknown): boolean { + if (error instanceof Error && error.message === "maxBytes exceeded") return true + if (!error || typeof error !== "object" || !("cause" in error)) return false + return isBodyLimitError(error.cause) +} + function bufferWebBody(request: Request) { if (!request.body) return Effect.succeed(new Uint8Array()) return Stream.fromReadableStream({ diff --git a/packages/deepagent-code/src/sync/replay-protocol.ts b/packages/deepagent-code/src/sync/replay-protocol.ts index a2b84563b..8ce6ada5e 100644 --- a/packages/deepagent-code/src/sync/replay-protocol.ts +++ b/packages/deepagent-code/src/sync/replay-protocol.ts @@ -7,7 +7,7 @@ export const SyncReplayLimits = { requestBytes: EventV2.AGGREGATE_READ_BATCH_BYTES + 1024 * 1024 + 64 * 1024, directoryCharacters: 4_096, eventIDCharacters: 200, - aggregateIDCharacters: 4_096, + aggregateIDCharacters: 200, typeCharacters: 256, } as const @@ -50,22 +50,23 @@ export function encodeReplayRequestPrefix(directory: string, events: readonly Re ) const eventsBytes = selected.contentBytes + 2 const requestBytes = prefixBytes + selected.contentBytes + suffixBytes + const valid = + directory.length <= SyncReplayLimits.directoryCharacters && + selected.events.every( + (event) => + event.id.length <= SyncReplayLimits.eventIDCharacters && + event.aggregateID.length > 0 && + event.aggregateID.length <= SyncReplayLimits.aggregateIDCharacters && + event.type.length > 0 && + event.type.length <= SyncReplayLimits.typeCharacters, + ) return { events: selected.events, json: prefix + selected.encoded.join(",") + suffix, dataBytes: selected.dataBytes, eventsBytes, requestBytes, - complete: - selected.events.length === events.length && - directory.length <= SyncReplayLimits.directoryCharacters && - selected.events.every( - (event) => - event.id.length <= SyncReplayLimits.eventIDCharacters && - event.aggregateID.length > 0 && - event.aggregateID.length <= SyncReplayLimits.aggregateIDCharacters && - event.type.length > 0 && - event.type.length <= SyncReplayLimits.typeCharacters, - ), + valid, + complete: selected.events.length === events.length && valid, } } diff --git a/packages/deepagent-code/test/control-plane/workspace.test.ts b/packages/deepagent-code/test/control-plane/workspace.test.ts index ca13c674b..c675daef7 100644 --- a/packages/deepagent-code/test/control-plane/workspace.test.ts +++ b/packages/deepagent-code/test/control-plane/workspace.test.ts @@ -17,7 +17,7 @@ import { AbsolutePath } from "@deepagent-code/core/schema" import { Session as SessionNs } from "@/session/session" import { SessionID } from "@/session/schema" import { SessionTable } from "@deepagent-code/core/session/sql" -import { EventSequenceTable } from "@deepagent-code/core/event/sql" +import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" import { SessionToolRequestReceiptTable } from "@/session/tool-request-receipt.sql" import { resetDatabase } from "../fixture/db" import { disposeAllInstances, provideTmpdirInstance, requireInstance, TestInstance } from "../fixture/fixture" @@ -35,6 +35,9 @@ import { Project } from "@/project/project" import { Vcs } from "@/project/vcs" import { RuntimeFlags } from "@/effect/runtime-flags" import { EventV2Bridge } from "@/event-v2-bridge" +import { EventV2 } from "@deepagent-code/core/event" +import { SessionV1 } from "@deepagent-code/core/v1/session" +import { SyncReplayLimits } from "@/sync/replay-protocol" void Log.init({ print: false }) @@ -1141,6 +1144,95 @@ describe("workspace CRUD", () => { }) }) + it.live("sessionWarp replays a maximum admitted event within the final request budget before steal", () => { + const calls: FetchCall[] = [] + return Effect.gen(function* () { + yield* HttpServer.serveEffect()( + Effect.gen(function* () { + const req = yield* HttpServerRequest.HttpServerRequest + const bodyText = yield* req.text + calls.push({ + url: new URL(req.url, "http://localhost"), + method: req.method, + headers: new Headers(req.headers), + bodyText, + json: bodyText ? JSON.parse(bodyText) : undefined, + }) + if (req.url.endsWith("/sync/replay")) return yield* HttpServerResponse.json({ sessionID: "ok" }) + if (req.url.endsWith("/sync/steal")) return yield* HttpServerResponse.json({ sessionID: "ok" }) + return HttpServerResponse.text("unexpected", { status: 500 }) + }), + ) + const url = yield* serverUrl() + yield* provideTmpdirInstance( + () => + Effect.gen(function* () { + const workspace = yield* Workspace.Service + const sessionSvc = yield* SessionNs.Service + const instance = yield* requireInstance + const type = unique("warp-max-event") + const target = workspaceInfo(instance.project.id, type, { directory: "remote-target-dir" }) + yield* insertWorkspace(target) + registerAdapter(instance.project.id, type, remoteAdapter(`${url}/warp-target`).adapter) + const session = yield* sessionSvc.create({}) + const { db } = yield* Database.Service + const sequence = (yield* sessionSequence(session.id))! + const base = { + sessionID: session.id, + info: { + id: SessionV1.MessageID.ascending(), + sessionID: session.id, + role: "user" as const, + time: { created: Date.now() }, + agent: "build", + model: { providerID: "test", modelID: "model" }, + system: "", + }, + } + const data = { + ...base, + info: { + ...base.info, + system: "x".repeat(EventV2.MAX_ENCODED_PAYLOAD_BYTES - Buffer.byteLength(JSON.stringify(base))), + }, + } + expect(Buffer.byteLength(JSON.stringify(data))).toBe(EventV2.MAX_ENCODED_PAYLOAD_BYTES) + yield* db + .insert(EventTable) + .values({ + id: EventV2.ID.make("evt_warp_max_admitted"), + aggregate_id: session.id, + seq: sequence + 1, + type: EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1), + data, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ seq: sequence + 1 }) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .run() + .pipe(Effect.orDie) + + yield* workspace.sessionWarp({ workspaceID: target.id, sessionID: session.id }) + + const replay = calls.filter((call) => call.url.pathname.endsWith("/sync/replay")) + expect(replay).toHaveLength(2) + expect(replay.flatMap((call) => (call.json as { events: Array<{ id: string }> }).events.map((e) => e.id))).toEqual([ + expect.stringMatching(/^evt_/), + "evt_warp_max_admitted", + ]) + expect(Buffer.byteLength(replay[1]!.bodyText!)).toBeGreaterThan(SyncReplayLimits.eventDataBytes) + expect(Buffer.byteLength(replay[1]!.bodyText!)).toBeLessThanOrEqual(SyncReplayLimits.requestBytes) + expect(calls.at(-1)?.url.pathname).toBe("/warp-target/sync/steal") + expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(target.id) + }), + { git: true }, + ) + }) + }, 30_000) + it.live("sessionWarp rejects an oversized chunked source patch before side effects or ownership changes", () => { const calls: FetchCall[] = [] return Effect.gen(function* () { diff --git a/packages/deepagent-code/test/server/httpapi-sync.test.ts b/packages/deepagent-code/test/server/httpapi-sync.test.ts index e4a07792c..d53374111 100644 --- a/packages/deepagent-code/test/server/httpapi-sync.test.ts +++ b/packages/deepagent-code/test/server/httpapi-sync.test.ts @@ -373,7 +373,20 @@ describe("sync HttpApi", () => { }) expect(tooMany.status).toBe(400) - const oversized = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { + const before = yield* Database.Service.use(({ db }) => + db.select().from(EventTable).where(eq(EventTable.aggregate_id, "aggregate-sync-replay-limit")).all(), + ) + const semanticOversized = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ + directory: tmp.directory, + events: [event(0, { value: "x".repeat(SyncReplayLimits.eventDataBytes) })], + }), + }) + expect(semanticOversized.status).toBe(400) + + const rawOversized = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { method: "POST", headers, body: JSON.stringify({ @@ -381,7 +394,12 @@ describe("sync HttpApi", () => { events: [event(0, { value: "x".repeat(SyncReplayLimits.requestBytes) })], }), }) - expect(oversized.status).toBe(413) + expect(rawOversized.status).toBe(413) + expect( + yield* Database.Service.use(({ db }) => + db.select().from(EventTable).where(eq(EventTable.aggregate_id, "aggregate-sync-replay-limit")).all(), + ), + ).toEqual(before) }), { git: true, config: { formatter: false, lsp: false } }, 15_000, From 1a9467c2038f9c0363f5aea63ef1ecabfae7bf10 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Wed, 12 Aug 2026 23:42:02 +0800 Subject: [PATCH 14/44] fix(session): filter legacy recovery authorities --- .../src/session/legacy-provider-resolution.ts | 17 +++++++++++------ .../deepagent-code/src/session/message-v2.ts | 3 +-- .../session/legacy-provider-resolution.test.ts | 8 ++++++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/packages/deepagent-code/src/session/legacy-provider-resolution.ts b/packages/deepagent-code/src/session/legacy-provider-resolution.ts index 10b4f1871..a92a862ee 100644 --- a/packages/deepagent-code/src/session/legacy-provider-resolution.ts +++ b/packages/deepagent-code/src/session/legacy-provider-resolution.ts @@ -204,9 +204,14 @@ const service = Effect.gen(function* () { Effect.gen(function* () { const receipt = item.receipt const request = receipt.final_request_hash ?? receipt.provider_request_hash ?? receipt.request_input_hash + const assistantMessageID = Option.getOrUndefined( + Schema.decodeUnknownOption(MessageID)(receipt.assistant_message_id), + ) + const promptEpoch = + receipt.prompt_epoch !== null && receipt.prompt_epoch >= 0 ? receipt.prompt_epoch : undefined if ( - !receipt.assistant_message_id || - receipt.prompt_epoch === null || + !assistantMessageID || + promptEpoch === undefined || !receipt.prompt_window_id || !receipt.effective_history_hash || !request @@ -229,7 +234,7 @@ const service = Effect.gen(function* () { .where( and( eq(SessionPromptEpochTable.session_id, sessionID), - eq(SessionPromptEpochTable.epoch, receipt.prompt_epoch), + eq(SessionPromptEpochTable.epoch, promptEpoch), ), ) .get() @@ -240,7 +245,7 @@ const service = Effect.gen(function* () { .where( and( eq(SessionWorldStateBaselineTable.session_id, sessionID), - eq(SessionWorldStateBaselineTable.prompt_epoch, receipt.prompt_epoch), + eq(SessionWorldStateBaselineTable.prompt_epoch, promptEpoch), ), ) .all() @@ -263,11 +268,11 @@ const service = Effect.gen(function* () { return { receiptID: receipt.receipt_id, sessionID, - assistantMessageID: MessageID.make(receipt.assistant_message_id), + assistantMessageID, providerID: receipt.provider_id, modelID: receipt.model_id, providerState: "indeterminate_after_crash" as const, - promptEpoch: receipt.prompt_epoch, + promptEpoch, promptWindowID: receipt.prompt_window_id, historyHash: receipt.effective_history_hash, requestHash: request, diff --git a/packages/deepagent-code/src/session/message-v2.ts b/packages/deepagent-code/src/session/message-v2.ts index dde9f3433..f943bb1de 100644 --- a/packages/deepagent-code/src/session/message-v2.ts +++ b/packages/deepagent-code/src/session/message-v2.ts @@ -1350,7 +1350,6 @@ export function promptHistoryCutoffAuthorityInTransaction( .select({ userMessageID: SessionToolRequestReceiptTable.user_message_id, assistantMessageID: SessionToolRequestReceiptTable.assistant_message_id, - providerAttemptID: SessionToolRequestReceiptTable.provider_attempt_id, promptEpoch: SessionToolRequestReceiptTable.prompt_epoch, promptWindowID: SessionToolRequestReceiptTable.prompt_window_id, effectiveHistoryHash: SessionToolRequestReceiptTable.effective_history_hash, @@ -1367,6 +1366,7 @@ export function promptHistoryCutoffAuthorityInTransaction( and( eq(SessionToolRequestReceiptTable.session_id, input.sessionID), eq(SessionToolRequestReceiptTable.provider_state, "indeterminate_after_crash"), + isNull(SessionToolRequestReceiptTable.provider_attempt_id), isNull(SessionToolRequestResolutionTable.resolution_id), ), ) @@ -1376,7 +1376,6 @@ export function promptHistoryCutoffAuthorityInTransaction( const userIndex = selected.messages.findIndex((message) => message.info.id === receipt.userMessageID) const assistantIndex = selected.messages.findIndex((message) => message.info.id === receipt.assistantMessageID) return ( - receipt.providerAttemptID === null && receipt.promptEpoch === epoch.epoch && receipt.promptWindowID === epoch.window_id && !!receipt.effectiveHistoryHash && diff --git a/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts b/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts index a95b4ca7d..9869b73c6 100644 --- a/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts +++ b/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts @@ -241,7 +241,9 @@ it.instance( reason: expect.stringContaining("fork cutoff is outside the recoverable history prefix"), }) - const [descriptor] = yield* recovery.describe(source.id) + const descriptors = yield* recovery.describe(source.id) + expect(descriptors).toHaveLength(1) + const [descriptor] = descriptors expect(descriptor).toMatchObject({ receiptID, sessionID: source.id, @@ -255,6 +257,8 @@ it.instance( worldStateBaselineHash: baselineHash, }) if (!descriptor) return + if (!descriptor.worldStateBaselineHash) + return yield* Effect.die("expected a resolvable provider recovery descriptor") const command = { sessionID: source.id, commandID: "resolve-ambiguous-provider-turn", @@ -266,7 +270,7 @@ it.instance( sessionMutationEpoch: descriptor.sessionMutationEpoch, requestHash: descriptor.requestHash, historyHash: descriptor.historyHash, - worldStateBaselineHash: descriptor.worldStateBaselineHash!, + worldStateBaselineHash: descriptor.worldStateBaselineHash, }, actorID: "test-user", } From f4a313cd3f312e56f2b551e1b63eca1ba0f8111f Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 00:05:12 +0800 Subject: [PATCH 15/44] fix(session): acknowledge implicit prompt admission --- packages/deepagent-code/src/session/prompt.ts | 86 ++++++++----------- .../test/session/prompt.test.ts | 51 +++++++++++ 2 files changed, 89 insertions(+), 48 deletions(-) diff --git a/packages/deepagent-code/src/session/prompt.ts b/packages/deepagent-code/src/session/prompt.ts index 63848eba0..a2b87a853 100644 --- a/packages/deepagent-code/src/session/prompt.ts +++ b/packages/deepagent-code/src/session/prompt.ts @@ -4531,60 +4531,52 @@ export const layer = Layer.effect( "SessionPrompt.promptAsync", )(function* (input: PromptInput) { const messageID = input.messageID ?? MessageID.ascending() - const claim = input.intentID - ? yield* SessionPromptIntent.claim({ - intentID: input.intentID, - sessionID: input.sessionID, - source: input.intentSource ?? "composer", - variant: - input.intentVariant ?? - (promptPipelineRequest(input.metadata).confirmedDraftID ? "rewritten" : "original"), - payloadHash: promptIntentPayloadHash(input), - messageID, - executionMode: input.noReply === true ? "deferred" : "run_now", - }).pipe(Effect.provideService(Database.Service, database)) - : undefined - if (claim?.kind === "admitted") + const claim = yield* SessionPromptIntent.claim({ + intentID: input.intentID ?? `legacy-prompt:${input.sessionID}:${messageID}`, + sessionID: input.sessionID, + source: input.intentSource ?? "composer", + variant: + input.intentVariant ?? (promptPipelineRequest(input.metadata).confirmedDraftID ? "rewritten" : "original"), + payloadHash: promptIntentPayloadHash(input), + messageID, + executionMode: input.noReply === true ? "deferred" : "run_now", + }).pipe(Effect.provideService(Database.Service, database)) + if (claim.kind === "admitted") return { messageID: claim.receipt.messageID, delivery: claim.receipt.delivery ?? "turn", } - const claimed = claim?.receipt - const admittedInput = claimed - ? { - ...input, - messageID: claimed.messageID, - parts: stableIntentParts(input.parts, claimed.intentID), - } - : { ...input, messageID } + const claimed = claim.receipt + const admittedInput = { + ...input, + messageID: claimed.messageID, + parts: stableIntentParts(input.parts, claimed.intentID), + } const admission = yield* Deferred.make< PromptAdmissionReceipt, Image.Error | SessionPromptIntent.Error | Session.BusyError >() - if (claimed) { - yield* Effect.suspend(() => - SessionPromptIntent.renew({ + yield* Effect.suspend(() => + SessionPromptIntent.renew({ + intentID: claimed.intentID, + ownerToken: claimed.ownerToken, + }).pipe( + Effect.provideService(Database.Service, database), + Effect.delay(Duration.seconds(10)), + Effect.flatMap((renewed) => (renewed ? Effect.void : Effect.interrupt)), + ), + ).pipe(Effect.forever, Effect.forkIn(scope)) + yield* promptOrSteer(admittedInput, { + intent: claimed, + ready: (receipt) => + SessionPromptIntent.complete({ intentID: claimed.intentID, ownerToken: claimed.ownerToken, + messageID: receipt.messageID, + delivery: receipt.delivery, }).pipe( Effect.provideService(Database.Service, database), - Effect.delay(Duration.seconds(10)), - Effect.flatMap((renewed) => (renewed ? Effect.void : Effect.interrupt)), - ), - ).pipe(Effect.forever, Effect.forkIn(scope)) - } - yield* promptOrSteer(admittedInput, { - intent: claimed, - ready: (receipt) => - (claimed - ? SessionPromptIntent.complete({ - intentID: claimed.intentID, - ownerToken: claimed.ownerToken, - messageID: receipt.messageID, - delivery: receipt.delivery, - }).pipe(Effect.provideService(Database.Service, database), Effect.asVoid) - : Effect.void - ).pipe( + Effect.asVoid, Effect.matchCauseEffect({ onFailure: (cause) => Deferred.failCause(admission, cause), onSuccess: () => Deferred.succeed(admission, receipt), @@ -4594,12 +4586,10 @@ export const layer = Layer.effect( }).pipe( Effect.catchCause((cause) => Effect.gen(function* () { - if (claimed) { - yield* SessionPromptIntent.fail({ - intentID: claimed.intentID, - ownerToken: claimed.ownerToken, - }).pipe(Effect.provideService(Database.Service, database)) - } + yield* SessionPromptIntent.fail({ + intentID: claimed.intentID, + ownerToken: claimed.ownerToken, + }).pipe(Effect.provideService(Database.Service, database)) yield* Effect.logError("prompt_async failed").pipe( Effect.annotateLogs({ sessionID: input.sessionID, cause }), ) diff --git a/packages/deepagent-code/test/session/prompt.test.ts b/packages/deepagent-code/test/session/prompt.test.ts index 96bcd4dd4..cf0ae1a61 100644 --- a/packages/deepagent-code/test/session/prompt.test.ts +++ b/packages/deepagent-code/test/session/prompt.test.ts @@ -3202,6 +3202,57 @@ it.instance( 15_000, ) +it.instance( + "promptAsync without an explicit intent acknowledges durable admission before the provider settles", + () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const gate = yield* Deferred.make() + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ title: "Implicit prompt intent" }) + const messageID = MessageID.ascending() + + yield* llm.hold("eventual answer", deferredAsPromise(gate)) + const receipt = yield* prompt.promptAsync({ + sessionID: chat.id, + messageID, + agent: "build", + model: ref, + parts: [{ type: "text", text: "hello" }], + }) + + expect(receipt).toEqual({ messageID, delivery: "turn" }) + yield* llm.wait(1) + const { db } = yield* Database.Service + expect( + yield* db + .select({ + state: SessionIntentTable.state, + executionState: SessionIntentTable.execution_state, + delivery: SessionIntentTable.delivery, + }) + .from(SessionIntentTable) + .where(eq(SessionIntentTable.intent_id, `legacy-prompt:${chat.id}:${messageID}`)) + .get() + .pipe(Effect.orDie), + ).toEqual({ state: "admitted", executionState: "claimed", delivery: "turn" }) + + yield* Deferred.succeed(gate, undefined) + yield* pollWithTimeout( + sessions.messages({ sessionID: chat.id }).pipe( + Effect.map((messages) => + messages.some((message) => message.info.role === "assistant" && message.info.time.completed) + ? messages + : undefined, + ), + ), + "provider turn did not settle after admission", + ) + }), + 15_000, +) + it.instance( "assertNotBusy fails with BusyError when loop running", () => From d990d9e1639d4bb9fc50b13d62c4a3128c2e7cdd Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 00:11:51 +0800 Subject: [PATCH 16/44] fix(control-plane): disable unsafe session transfers --- .../core/src/control-plane/move-session.ts | 106 +---- packages/core/test/move-session.test.ts | 357 ++++------------ .../src/control-plane/workspace.ts | 395 ++--------------- .../instance/httpapi/groups/control-plane.ts | 5 +- .../instance/httpapi/groups/workspace.ts | 6 +- .../httpapi/handlers/control-plane.ts | 14 +- .../routes/instance/httpapi/handlers/sync.ts | 22 +- .../instance/httpapi/handlers/workspace.ts | 4 + .../test/control-plane/workspace.test.ts | 403 +++++++++++------- .../test/server/httpapi-control-plane.test.ts | 33 +- .../test/server/httpapi-workspace.test.ts | 103 +++-- 11 files changed, 502 insertions(+), 946 deletions(-) diff --git a/packages/core/src/control-plane/move-session.ts b/packages/core/src/control-plane/move-session.ts index e1c2575dc..faa45121f 100644 --- a/packages/core/src/control-plane/move-session.ts +++ b/packages/core/src/control-plane/move-session.ts @@ -1,16 +1,10 @@ export * as MoveSession from "./move-session" -import { Context, DateTime, Effect, Layer, Schema } from "effect" -import { EventV2 } from "../event" -import { Git } from "../git" -import { Location } from "../location" +import { Context, Effect, Layer, Schema } from "effect" import { ProjectV2 } from "../project" import { SessionV2 } from "../session" -import { SessionExecution } from "../session/execution" -import { SessionEvent } from "../session/event" import { SessionSchema } from "../session/schema" -import { AbsolutePath, RelativePath } from "../schema" -import path from "path" +import { AbsolutePath } from "../schema" export const Destination = Schema.Struct({ directory: AbsolutePath, @@ -52,12 +46,23 @@ export class ResetSourceChangesError extends Schema.TaggedErrorClass()( + "MoveSession.TransferUnsupportedError", + { + sessionID: SessionSchema.ID, + source: AbsolutePath, + destination: AbsolutePath, + message: Schema.String, + }, +) {} + export type Error = | SessionV2.NotFoundError | DestinationProjectMismatchError | CaptureChangesError | ApplyChangesError | ResetSourceChangesError + | TransferUnsupportedError export interface Interface { readonly moveSession: (input: Input) => Effect.Effect @@ -68,93 +73,22 @@ export class Service extends Context.Service()("@deepagent-c export const layer = Layer.effect( Service, Effect.gen(function* () { - const git = yield* Git.Service - const events = yield* EventV2.Service - const project = yield* ProjectV2.Service const session = yield* SessionV2.Service const moveSession = Effect.fn("MoveSession.moveSession")(function* (input: Input) { const current = yield* session.get(input.sessionID) const directory = AbsolutePath.make(input.destination.directory) if (current.location.directory === directory) return - - const source = yield* project.resolve(current.location.directory) - const destination = yield* project.resolve(directory) - if (current.projectID !== destination.id) { - return yield* new DestinationProjectMismatchError({ expected: current.projectID, actual: destination.id }) - } - - const patch = - input.moveChanges && source.directory !== destination.directory - ? yield* git - .patch(current.location.directory) - .pipe(Effect.mapError((error) => new CaptureChangesError({ message: error.message }))) - : "" - if (patch) { - yield* git - .applyPatch({ directory, patch }) - .pipe(Effect.mapError((error) => new ApplyChangesError({ message: error.message }))) - } - - const sessions = isFilesystemRoot(current.location.directory) - ? conversationTree(current, yield* session.list({ directory: current.location.directory })) - : [current] - const timestamp = yield* DateTime.now - yield* Effect.forEach( - sessions, - (item) => - events.publish(SessionEvent.Moved, { - sessionID: item.id, - location: Location.Ref.make({ directory }), - subdirectory: RelativePath.make(path.relative(destination.directory, directory).replaceAll("\\", "/")), - timestamp, - }), - { discard: true }, - ) - - if (patch) { - yield* git.softResetChanges(current.location.directory).pipe( - Effect.mapError( - (error) => - new ResetSourceChangesError({ - directory: current.location.directory, - message: error.message, - cause: error.cause, - }), - ), - ) - } + return yield* new TransferUnsupportedError({ + sessionID: input.sessionID, + source: current.location.directory, + destination: directory, + message: "Session moves require durable transfer admission, execution fencing, and idempotent change receipts", + }) }) return Service.of({ moveSession }) }), ) -function isFilesystemRoot(directory: string) { - const resolved = path.resolve(directory) - return path.dirname(resolved) === resolved -} - -function conversationTree(current: SessionSchema.Info, sessions: SessionSchema.Info[]) { - const byID = new Map(sessions.map((session) => [session.id, session])) - const root = sessions.reduce( - (session) => (session.parentID ? (byID.get(session.parentID) ?? session) : session), - current, - ) - const result: SessionSchema.Info[] = [] - const append = (session: SessionSchema.Info) => { - if (result.some((item) => item.id === session.id)) return - result.push(session) - sessions.filter((item) => item.parentID === session.id).forEach(append) - } - append(root) - return result -} - -export const defaultLayer = layer.pipe( - Layer.provide(Git.defaultLayer), - Layer.provide(EventV2.defaultLayer), - Layer.provide(ProjectV2.defaultLayer), - Layer.provide(SessionExecution.noopLayer), - Layer.provide(SessionV2.defaultLayer), -) +export const defaultLayer = layer.pipe(Layer.provide(SessionV2.defaultLayer)) diff --git a/packages/core/test/move-session.test.ts b/packages/core/test/move-session.test.ts index 97ef070cb..66a38320e 100644 --- a/packages/core/test/move-session.test.ts +++ b/packages/core/test/move-session.test.ts @@ -1,311 +1,114 @@ import { describe, expect } from "bun:test" -import { $ } from "bun" -import fs from "fs/promises" -import path from "path" -import { eq } from "drizzle-orm" -import { Effect, Layer } from "effect" -import { MoveSession } from "@deepagent-code/core/control-plane/move-session" import { Database } from "@deepagent-code/core/database/database" -import { FSUtil } from "@deepagent-code/core/fs-util" -import { Git } from "@deepagent-code/core/git" -import { EventV2 } from "@deepagent-code/core/event" -import { Project } from "@deepagent-code/core/project" +import { MoveSession } from "@deepagent-code/core/control-plane/move-session" +import { EventTable } from "@deepagent-code/core/event/sql" +import { ProjectV2 } from "@deepagent-code/core/project" import { ProjectTable } from "@deepagent-code/core/project/sql" -import { AbsolutePath } from "@deepagent-code/core/schema" import { SessionV2 } from "@deepagent-code/core/session" -import { SessionExecution } from "@deepagent-code/core/session/execution" -import { SessionProjector } from "@deepagent-code/core/session/projector" import { SessionTable } from "@deepagent-code/core/session/sql" -import { SessionStore } from "@deepagent-code/core/session/store" -import { tmpdir } from "./fixture/tmpdir" +import { AbsolutePath } from "@deepagent-code/core/schema" +import { eq, sql } from "drizzle-orm" +import { Effect, Layer } from "effect" import { testEffect } from "./lib/effect" const database = Database.layerFromPath(":memory:") -const events = EventV2.layer.pipe(Layer.provide(database)) -const projector = SessionProjector.layer.pipe(Layer.provide(database), Layer.provide(events)) -const project = Project.layer.pipe( - Layer.provide(database), - Layer.provide(FSUtil.defaultLayer), - Layer.provide(Git.defaultLayer), -) -const store = SessionStore.layer.pipe(Layer.provide(database)) -const sessions = SessionV2.layer.pipe( - Layer.provide(database), - Layer.provide(events), - Layer.provide(project), - Layer.provide(store), - Layer.provide(SessionExecution.noopLayer), -) -const layer = MoveSession.layer.pipe( - Layer.provide(database), - Layer.provide(FSUtil.defaultLayer), - Layer.provide(Git.defaultLayer), - Layer.provide(events), - Layer.provide(project), - Layer.provide(sessions), -) -const it = testEffect( - Layer.mergeAll(layer, database, events, project, projector, store, SessionExecution.noopLayer, sessions), -) - -function abs(input: string) { - return AbsolutePath.make(input) -} - -async function initRepo(directory: string) { - await $`git init`.cwd(directory).quiet() - await $`git config core.autocrlf false`.cwd(directory).quiet() - await $`git config core.fsmonitor false`.cwd(directory).quiet() - await $`git config commit.gpgsign false`.cwd(directory).quiet() - await $`git config user.email test@deepagent-code.test`.cwd(directory).quiet() - await $`git config user.name Test`.cwd(directory).quiet() - await fs.writeFile(path.join(directory, "tracked.txt"), "initial\n") - await $`git add tracked.txt`.cwd(directory).quiet() - await $`git commit -m root`.cwd(directory).quiet() -} +const layer = Layer.mergeAll(database, SessionV2.defaultLayer, MoveSession.defaultLayer) +const it = testEffect(layer) + +const source = AbsolutePath.make("/source") +const destination = AbsolutePath.make("/destination") +const projectID = ProjectV2.ID.make("project") + +const insertSession = Effect.fn("MoveSessionTest.insertSession")(function* (sessionID: SessionV2.ID) { + const { db } = yield* Database.Service + yield* db + .insert(ProjectTable) + .values([{ id: projectID, worktree: source, sandboxes: [], time_created: 1, time_updated: 1 }]) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values([ + { + id: sessionID, + project_id: projectID, + slug: sessionID, + directory: source, + title: "move", + version: "test", + time_created: 1, + time_updated: 1, + }, + ]) + .run() + .pipe(Effect.orDie) +}) describe("MoveSession", () => { - it.live("moves session changes to another project directory", () => + it.effect("keeps an exact-location retry idempotent", () => Effect.gen(function* () { - const root = yield* Effect.acquireRelease( - Effect.promise(() => tmpdir()), - (dir) => Effect.promise(() => dir[Symbol.asyncDispose]()), - ) - yield* Effect.promise(() => initRepo(root.path)) - const source = abs(yield* Effect.promise(() => fs.realpath(root.path))) - const destination = abs(`${root.path}-move-destination`) - yield* Effect.addFinalizer(() => - Effect.promise(() => fs.rm(destination, { recursive: true, force: true })).pipe(Effect.ignore), - ) - yield* Effect.promise(() => $`git worktree add --detach ${destination} HEAD`.cwd(root.path).quiet()) - const moved = abs(yield* Effect.promise(() => fs.realpath(destination))) - yield* Effect.promise(() => fs.writeFile(path.join(source, "tracked.txt"), "changed\n")) - yield* Effect.promise(() => fs.writeFile(path.join(source, "untracked.txt"), "new\n")) - - const projectID = (yield* Project.Service.use((service) => service.resolve(source))).id - const sessionID = SessionV2.ID.make("ses_move") - const { db } = yield* Database.Service - yield* db - .insert(ProjectTable) - .values({ id: projectID, worktree: source, sandboxes: [], time_created: 1, time_updated: 1 }) - .run() - .pipe(Effect.orDie) - yield* db - .insert(SessionTable) - .values({ - id: sessionID, - project_id: projectID, - slug: "move", - directory: source, - title: "move", - version: "test", - time_created: 1, - time_updated: 1, - }) - .run() - .pipe(Effect.orDie) + const sessionID = SessionV2.ID.make("ses_move_same_location") + yield* insertSession(sessionID) yield* MoveSession.Service.use((service) => - service.moveSession({ sessionID, destination: { directory: moved }, moveChanges: true }), + service.moveSession({ sessionID, destination: { directory: source }, moveChanges: true }), ) - expect(yield* Effect.promise(() => fs.readFile(path.join(moved, "tracked.txt"), "utf8"))).toBe("changed\n") - expect(yield* Effect.promise(() => fs.readFile(path.join(moved, "untracked.txt"), "utf8"))).toBe("new\n") - expect(yield* Effect.promise(() => fs.readFile(path.join(source, "tracked.txt"), "utf8"))).toBe("initial\n") - expect(yield* Effect.promise(() => Bun.file(path.join(source, "untracked.txt")).exists())).toBe(false) - expect( - yield* db - .select({ directory: SessionTable.directory, path: SessionTable.path }) - .from(SessionTable) - .where(eq(SessionTable.id, sessionID)) - .get(), - ).toEqual({ directory: moved, path: "" }) + expect(yield* SessionV2.Service.use((service) => service.get(sessionID))).toMatchObject({ + id: sessionID, + location: { directory: source }, + }) }), ) - it.live("moves within a checkout without transferring existing changes", () => + it.effect("rejects placement changes before durable Session or Event side effects", () => Effect.gen(function* () { - const root = yield* Effect.acquireRelease( - Effect.promise(() => tmpdir()), - (dir) => Effect.promise(() => dir[Symbol.asyncDispose]()), - ) - yield* Effect.promise(() => initRepo(root.path)) - const source = abs(yield* Effect.promise(() => fs.realpath(root.path))) - const destination = abs(path.join(source, "packages")) - yield* Effect.promise(() => fs.mkdir(destination)) - yield* Effect.promise(() => fs.writeFile(path.join(source, "tracked.txt"), "changed\n")) - yield* Effect.promise(() => fs.writeFile(path.join(source, "untracked.txt"), "new\n")) - - const projectID = (yield* Project.Service.use((service) => service.resolve(source))).id - const sessionID = SessionV2.ID.make("ses_move_nested") + const sessionID = SessionV2.ID.make("ses_move_requires_transfer") + yield* insertSession(sessionID) const { db } = yield* Database.Service - yield* db - .insert(ProjectTable) - .values({ id: projectID, worktree: source, sandboxes: [], time_created: 1, time_updated: 1 }) - .run() - .pipe(Effect.orDie) - yield* db - .insert(SessionTable) - .values({ - id: sessionID, - project_id: projectID, - slug: "move-nested", - directory: source, - title: "move nested", - version: "test", - time_created: 1, - time_updated: 1, - }) - .run() - .pipe(Effect.orDie) + const before = { + session: yield* db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get(), + events: yield* db + .select({ count: sql`count(*)` }) + .from(EventTable) + .where(eq(EventTable.aggregate_id, sessionID)) + .get(), + } - yield* MoveSession.Service.use((service) => - service.moveSession({ sessionID, destination: { directory: destination }, moveChanges: true }), + const error = yield* MoveSession.Service.use((service) => + service + .moveSession({ sessionID, destination: { directory: destination }, moveChanges: true }) + .pipe(Effect.flip), ) - expect(yield* Effect.promise(() => fs.readFile(path.join(source, "tracked.txt"), "utf8"))).toBe("changed\n") - expect(yield* Effect.promise(() => fs.readFile(path.join(source, "untracked.txt"), "utf8"))).toBe("new\n") + expect(error).toEqual( + new MoveSession.TransferUnsupportedError({ + sessionID, + source, + destination, + message: + "Session moves require durable transfer admission, execution fencing, and idempotent change receipts", + }), + ) + expect(yield* db.select().from(SessionTable).where(eq(SessionTable.id, sessionID)).get()).toEqual(before.session) expect( yield* db - .select({ directory: SessionTable.directory, path: SessionTable.path }) - .from(SessionTable) - .where(eq(SessionTable.id, sessionID)) + .select({ count: sql`count(*)` }) + .from(EventTable) + .where(eq(EventTable.aggregate_id, sessionID)) .get(), - ).toEqual({ directory: destination, path: "packages" }) - }), - ) - - it.live("moves nested session changes without cleaning unrelated files", () => - Effect.gen(function* () { - const root = yield* Effect.acquireRelease( - Effect.promise(() => tmpdir()), - (dir) => Effect.promise(() => dir[Symbol.asyncDispose]()), - ) - yield* Effect.promise(() => initRepo(root.path)) - const source = abs(yield* Effect.promise(() => fs.realpath(root.path))) - const sourceDirectory = abs(path.join(source, "packages")) - yield* Effect.promise(() => fs.mkdir(sourceDirectory)) - yield* Effect.promise(() => fs.writeFile(path.join(sourceDirectory, "tracked.txt"), "initial\n")) - yield* Effect.promise(() => fs.writeFile(path.join(sourceDirectory, "staged.txt"), "initial\n")) - yield* Effect.promise(() => $`git add packages/tracked.txt packages/staged.txt`.cwd(source).quiet()) - yield* Effect.promise(() => $`git commit -m packages`.cwd(source).quiet()) - const destination = abs(`${root.path}-move-nested-destination`) - yield* Effect.addFinalizer(() => - Effect.promise(() => fs.rm(destination, { recursive: true, force: true })).pipe(Effect.ignore), - ) - yield* Effect.promise(() => $`git worktree add --detach ${destination} HEAD`.cwd(source).quiet()) - const moved = abs(path.join(yield* Effect.promise(() => fs.realpath(destination)), "packages")) - yield* Effect.promise(() => fs.writeFile(path.join(sourceDirectory, "tracked.txt"), "changed\n")) - yield* Effect.promise(() => fs.writeFile(path.join(sourceDirectory, "staged.txt"), "staged\n")) - yield* Effect.promise(() => $`git add packages/staged.txt`.cwd(source).quiet()) - yield* Effect.promise(() => fs.writeFile(path.join(sourceDirectory, "untracked.txt"), "new\n")) - yield* Effect.promise(() => fs.writeFile(path.join(source, "tracked.txt"), "unrelated\n")) - yield* Effect.promise(() => fs.writeFile(path.join(source, "untracked.txt"), "unrelated\n")) - - const projectID = (yield* Project.Service.use((service) => service.resolve(source))).id - const sessionID = SessionV2.ID.make("ses_move_nested_checkout") - const { db } = yield* Database.Service - yield* db - .insert(ProjectTable) - .values({ id: projectID, worktree: source, sandboxes: [], time_created: 1, time_updated: 1 }) - .run() - .pipe(Effect.orDie) - yield* db - .insert(SessionTable) - .values({ - id: sessionID, - project_id: projectID, - slug: "move-nested-checkout", - directory: sourceDirectory, - title: "move nested checkout", - version: "test", - time_created: 1, - time_updated: 1, - }) - .run() - .pipe(Effect.orDie) - - yield* MoveSession.Service.use((service) => - service.moveSession({ sessionID, destination: { directory: moved }, moveChanges: true }), - ) - - expect(yield* Effect.promise(() => fs.readFile(path.join(moved, "tracked.txt"), "utf8"))).toBe("changed\n") - expect(yield* Effect.promise(() => fs.readFile(path.join(moved, "staged.txt"), "utf8"))).toBe("staged\n") - expect(yield* Effect.promise(() => fs.readFile(path.join(moved, "untracked.txt"), "utf8"))).toBe("new\n") - expect(yield* Effect.promise(() => fs.readFile(path.join(sourceDirectory, "tracked.txt"), "utf8"))).toBe( - "initial\n", - ) - expect(yield* Effect.promise(() => Bun.file(path.join(sourceDirectory, "untracked.txt")).exists())).toBe(false) - expect(yield* Effect.promise(() => fs.readFile(path.join(sourceDirectory, "staged.txt"), "utf8"))).toBe( - "staged\n", - ) - expect(yield* Effect.promise(() => $`git status --porcelain -- packages/staged.txt`.cwd(source).text())).toBe( - "M packages/staged.txt\n", - ) - expect(yield* Effect.promise(() => fs.readFile(path.join(source, "tracked.txt"), "utf8"))).toBe("unrelated\n") - expect(yield* Effect.promise(() => fs.readFile(path.join(source, "untracked.txt"), "utf8"))).toBe("unrelated\n") + ).toEqual(before.events) }), ) - it.live("recovers a filesystem-root conversation tree without moving unrelated root sessions", () => + it.effect("preserves the typed not-found result", () => Effect.gen(function* () { - const root = yield* Effect.acquireRelease( - Effect.promise(() => tmpdir()), - (dir) => Effect.promise(() => dir[Symbol.asyncDispose]()), - ) - const destination = abs(path.join(root.path, "workspaces", "recovered")) - yield* Effect.promise(() => fs.mkdir(destination, { recursive: true })) - const projectID = Project.ID.global - const parent = SessionV2.ID.make("ses_root_parent") - const child = SessionV2.ID.make("ses_root_child") - const sibling = SessionV2.ID.make("ses_root_sibling") - const unrelated = SessionV2.ID.make("ses_root_unrelated") - const { db } = yield* Database.Service - yield* db - .insert(ProjectTable) - .values({ id: projectID, worktree: abs("/"), sandboxes: [], time_created: 1, time_updated: 1 }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie) - yield* db - .insert(SessionTable) - .values( - [ - { id: parent, parent_id: null, title: "parent" }, - { id: child, parent_id: parent, title: "child" }, - { id: sibling, parent_id: parent, title: "sibling" }, - { id: unrelated, parent_id: null, title: "unrelated" }, - ].map((session) => ({ - ...session, - project_id: projectID, - slug: session.id, - directory: abs("/"), - version: "test", - time_created: 1, - time_updated: 1, - })), - ) - .run() - .pipe(Effect.orDie) - - yield* MoveSession.Service.use((service) => - service.moveSession({ sessionID: child, destination: { directory: destination } }), - ) - - expect( - yield* db - .select({ id: SessionTable.id, directory: SessionTable.directory }) - .from(SessionTable) - .where(eq(SessionTable.project_id, projectID)) - .all(), - ).toEqual( - expect.arrayContaining([ - { id: parent, directory: destination }, - { id: child, directory: destination }, - { id: sibling, directory: destination }, - { id: unrelated, directory: abs("/") }, - ]), + const sessionID = SessionV2.ID.make("ses_move_missing") + const error = yield* MoveSession.Service.use((service) => + service.moveSession({ sessionID, destination: { directory: destination } }).pipe(Effect.flip), ) + expect(error).toBeInstanceOf(SessionV2.NotFoundError) }), ) }) diff --git a/packages/deepagent-code/src/control-plane/workspace.ts b/packages/deepagent-code/src/control-plane/workspace.ts index 1d64b9f55..3e044f07c 100644 --- a/packages/deepagent-code/src/control-plane/workspace.ts +++ b/packages/deepagent-code/src/control-plane/workspace.ts @@ -5,15 +5,13 @@ import { Database } from "@deepagent-code/core/database/database" import { asc } from "drizzle-orm" import { and } from "drizzle-orm" import { eq } from "drizzle-orm" -import { gt } from "drizzle-orm" import { inArray } from "drizzle-orm" -import { sql } from "drizzle-orm" import { Project } from "@/project/project" import { GlobalBus } from "@/bus/global" import { Auth } from "@/auth" import { EventV2 } from "@deepagent-code/core/event" import { EventV2Bridge } from "@/event-v2-bridge" -import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { EventSequenceTable } from "@deepagent-code/core/event/sql" import { FSUtil } from "@deepagent-code/core/fs-util" import * as Log from "@deepagent-code/core/util/log" import { RuntimeFlags } from "@/effect/runtime-flags" @@ -21,10 +19,9 @@ import { ProjectV2 } from "@deepagent-code/core/project" import { Slug } from "@deepagent-code/core/util/slug" import { WorkspaceTable } from "@deepagent-code/core/control-plane/workspace.sql" import { getAdapter, registeredAdapters } from "./adapters" -import { type Target, type WorkspaceInfo, WorkspaceInfo as WorkspaceInfoSchema } from "./types" +import { type WorkspaceInfo, WorkspaceInfo as WorkspaceInfoSchema } from "./types" import { WorkspaceV2 } from "@deepagent-code/core/workspace" import { Session } from "@/session/session" -import { SessionPrompt } from "@/session/prompt" import { SessionTable } from "@deepagent-code/core/session/sql" import { SessionID } from "@/session/schema" import { NotFoundError } from "@/storage/storage" @@ -32,11 +29,8 @@ import { errorData } from "@/util/error" import { waitEvent } from "./util" import { WorkspaceRef } from "@/effect/instance-ref" import { Vcs } from "@/project/vcs" -import { InstanceStore } from "@/project/instance-store" -import { InstanceBootstrap } from "@/project/bootstrap" import { WorkspaceAdapterRuntime } from "./workspace-adapter-runtime" import { SessionRecoveryTransferGuard } from "@/session/recovery-transfer-guard" -import { encodeReplayRequestPrefix, SyncReplayLimits } from "@/sync/replay-protocol" export const Info = Schema.Struct({ ...WorkspaceInfoSchema.fields, @@ -149,6 +143,15 @@ export class SessionWarpHistoryLimitError extends Schema.TaggedErrorClass()( + "WorkspaceSessionWarpTransferUnsupportedError", + { + message: Schema.String, + sessionID: SessionID, + reason: Schema.Literals(["placement_change", "copy_changes"]), + }, +) {} + export class SyncTimeoutError extends Schema.TaggedErrorClass()("WorkspaceSyncTimeoutError", { message: Schema.String, state: Schema.Record(Schema.String, Schema.Number), @@ -166,6 +169,7 @@ type SessionWarpError = | SessionWarpHttpError | SessionWarpRecoveryProjectionError | SessionWarpHistoryLimitError + | SessionWarpTransferUnsupportedError | Vcs.RawDiffError | Vcs.PatchApplyError | HttpClientError.HttpClientError @@ -199,10 +203,8 @@ export const layer = Layer.effect( Effect.gen(function* () { const auth = yield* Auth.Service const session = yield* Session.Service - const prompt = yield* SessionPrompt.Service const http = yield* HttpClient.HttpClient const events = yield* EventV2Bridge.Service - const vcs = yield* Vcs.Service const flags = yield* RuntimeFlags.Service const fs = yield* FSUtil.Service const { db } = yield* Database.Service @@ -294,148 +296,6 @@ export const layer = Layer.effect( ) }) - const runInWorkspace = (input: { - workspaceID?: WorkspaceV2.ID - local: () => Effect.Effect - remote: (input: { - workspace: Info - target: Extract - }) => HttpClientRequest.HttpClientRequest - fallback: A - response?: "json" | "text" - }) => - Effect.gen(function* () { - if (!input.workspaceID) return yield* input.local() - - const workspace = yield* get(input.workspaceID) - if (!workspace) return input.fallback - - const target = yield* WorkspaceAdapterRuntime.target(workspace) - - if (target.type === "local") { - const store = yield* InstanceStore.Service - return yield* store.provide({ directory: target.directory }, input.local()) - } - - const response = yield* http.execute(input.remote({ workspace, target })).pipe( - Effect.catch((error) => - Effect.sync(() => { - log.warn("workspace target request failed", { - workspaceID: workspace.id, - error: errorData(error), - }) - }), - ), - ) - if (!response) return input.fallback - if (response.status < 200 || response.status >= 300) { - const body = yield* response.text.pipe(Effect.catch(() => Effect.succeed(""))) - log.warn("workspace target request failed", { - workspaceID: workspace.id, - status: response.status, - body, - }) - return input.fallback - } - - const body = input.response === "text" ? response.text : response.json - return yield* body.pipe( - Effect.map((result) => result as A), - Effect.catch((error) => - Effect.sync(() => { - log.warn("workspace target response decode failed", { - workspaceID: workspace.id, - error: errorData(error), - }) - return input.fallback - }), - ), - ) - }) - - const loadRawPatch = Effect.fn("Workspace.loadRawPatch")(function* (workspaceID: WorkspaceV2.ID) { - const workspace = yield* get(workspaceID) - if (!workspace) - return yield* new Vcs.RawDiffError({ - message: `Raw patch source workspace not found: ${workspaceID}`, - reason: "remote-status", - }) - - const target = yield* WorkspaceAdapterRuntime.target(workspace) - if (target.type === "local") { - const store = yield* InstanceStore.Service - return yield* store.provide({ directory: target.directory }, vcs.diffRaw()) - } - - const response = yield* http - .execute( - HttpClientRequest.get(route(target.url, "/vcs/diff/raw"), { - headers: new Headers(target.headers), - }), - ) - .pipe( - Effect.mapError( - () => - new Vcs.RawDiffError({ - message: `Raw patch source request failed: ${workspaceID}`, - reason: "remote-status", - }), - ), - ) - if (response.status < 200 || response.status >= 300) - return yield* new Vcs.RawDiffError({ - message: `Raw patch source returned HTTP ${response.status}`, - reason: "remote-status", - actual: response.status, - }) - - const contentLength = Number(response.headers["content-length"]) - if (Number.isFinite(contentLength) && contentLength > Vcs.RawDiffLimits.patchBytes) - return yield* new Vcs.RawDiffError({ - message: "Remote raw patch exceeded the declared response budget", - reason: "remote-output", - limit: Vcs.RawDiffLimits.patchBytes, - actual: contentLength, - }) - - const result = yield* Stream.runFoldEffect( - response.stream, - () => ({ chunks: [] as Uint8Array[], bytes: 0 }), - (acc, chunk) => { - const bytes = acc.bytes + chunk.byteLength - if (bytes <= Vcs.RawDiffLimits.patchBytes) { - acc.chunks.push(chunk) - return Effect.succeed({ chunks: acc.chunks, bytes }) - } - return Effect.fail( - new Vcs.RawDiffError({ - message: "Remote raw patch exceeded the streamed response budget", - reason: "remote-output", - limit: Vcs.RawDiffLimits.patchBytes, - actual: bytes, - }), - ) - }, - ).pipe( - Effect.mapError((error) => - error instanceof Vcs.RawDiffError - ? error - : new Vcs.RawDiffError({ - message: "Remote raw patch response failed while streaming", - reason: "remote-output", - }), - ), - ) - return yield* Effect.try({ - try: () => new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(result.chunks, result.bytes)), - catch: () => - new Vcs.RawDiffError({ - message: "Remote raw patch response is not valid UTF-8", - reason: "remote-decode", - }), - }) - }) - const syncHistory = Effect.fn("Workspace.syncHistory")(function* ( space: Info, url: URL | string, @@ -726,226 +586,35 @@ export const layer = Layer.effect( .get() .pipe(Effect.orDie) - const recoveryAuthorityID = yield* SessionRecoveryTransferGuard.authorityID(db, input.sessionID) - if ( - recoveryAuthorityID && - (input.workspaceID ?? undefined) !== (current?.workspaceID ?? undefined) - ) - return yield* new SessionWarpRecoveryProjectionError({ - message: - "Provider recovery projection cannot move between workspaces until canonical recovery snapshot import is available", - sessionID: input.sessionID, - recoveryAuthorityID, - }) - - const sourcePatch = - input.copyChanges && current?.workspaceID - ? yield* loadRawPatch(current.workspaceID).pipe( - Effect.provide(InstanceStore.defaultLayer.pipe(Layer.provide(InstanceBootstrap.defaultLayer))), - ) - : "" - - if (sourcePatch) { - const applied = yield* runInWorkspace({ - workspaceID: input.workspaceID ?? undefined, - local: () => vcs.apply({ patch: sourcePatch }), - remote: ({ target }) => - HttpClientRequest.post(route(target.url, "/vcs/apply"), { - headers: new Headers(target.headers), - body: HttpBody.jsonUnsafe({ patch: sourcePatch }), - }), - fallback: { applied: false }, - }).pipe(Effect.provide(InstanceStore.defaultLayer.pipe(Layer.provide(InstanceBootstrap.defaultLayer)))) - if (!applied.applied) - return yield* new Vcs.PatchApplyError({ - message: "Patch can't be applied because the target workspace was unavailable", - reason: "not-clean", - }) - } - - if (current?.workspaceID) { - const previous = yield* get(current.workspaceID) - if (previous) { - const target = yield* WorkspaceAdapterRuntime.target(previous) - - if (target.type === "remote") { - yield* syncHistory(previous, target.url, target.headers).pipe( - Effect.catch((error) => - Effect.sync(() => { - log.warn("session warp final source sync failed", { - workspaceID: previous.id, - sessionID: input.sessionID, - error: errorData(error), - }) - }), - ), - ) - } else { - yield* prompt.cancel(input.sessionID) - } - - // "claim" this session so any future events coming from - // the old workspace are ignored - yield* events.claim(input.sessionID, input.workspaceID ?? previous.projectID) - } - } - - if (input.workspaceID === null) { - yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: undefined }) - - log.info("session warp complete", { - workspaceID: input.workspaceID, + if (!current) + return yield* new SessionEventsNotFoundError({ + message: `No session found for warp: ${input.sessionID}`, sessionID: input.sessionID, - target: "local", }) - return - } - - const workspaceID = input.workspaceID - const space = yield* get(workspaceID) - if (!space) + if ((input.workspaceID ?? undefined) === (current.workspaceID ?? undefined) && !input.copyChanges) return + if (input.workspaceID !== null && !(yield* get(input.workspaceID))) return yield* new WorkspaceNotFoundError({ - message: `Workspace not found: ${workspaceID}`, - workspaceID, - }) - - const target = yield* WorkspaceAdapterRuntime.target(space) - - if (target.type === "local") { - yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: input.workspaceID }) - - log.info("session warp complete", { + message: `Workspace not found: ${input.workspaceID}`, workspaceID: input.workspaceID, - sessionID: input.sessionID, - target: target.directory, }) - return - } - - const pageEvents = Math.min(10, SyncReplayLimits.events) - const pageBytes = SyncReplayLimits.eventDataBytes - let cursor = -1 - let total = 0 - while (true) { - const candidates = yield* db - .select({ - id: EventTable.id, - seq: EventTable.seq, - bytes: sql`length(CAST(${EventTable.data} AS BLOB))`, - }) - .from(EventTable) - .where(and(eq(EventTable.aggregate_id, input.sessionID), gt(EventTable.seq, cursor))) - .orderBy(asc(EventTable.seq)) - .limit(pageEvents) - .all() - .pipe(Effect.orDie) - if (candidates.length === 0) break - const page = candidates.reduce<{ - ids: EventV2.ID[] - bytes: number - closed: boolean - oversized?: (typeof candidates)[number] - }>( - (result, row) => { - if (result.closed) return result - if (row.bytes > pageBytes) - return result.ids.length === 0 - ? { ...result, closed: true, oversized: row } - : { ...result, closed: true } - if (result.bytes > 0 && result.bytes + row.bytes > pageBytes) return { ...result, closed: true } - return { ids: [...result.ids, row.id], bytes: result.bytes + row.bytes, closed: false } - }, - { ids: [], bytes: 0, closed: false }, - ) - if (page.oversized) - return yield* new SessionWarpHistoryLimitError({ - message: `Legacy sync event ${page.oversized.id} exceeds the bounded warp page and requires artifact migration`, - sessionID: input.sessionID, - eventID: page.oversized.id, - bytes: page.oversized.bytes, - }) - if (page.ids.length === 0) break - const events = yield* db - .select({ - id: EventTable.id, - aggregateID: EventTable.aggregate_id, - seq: EventTable.seq, - type: EventTable.type, - data: EventTable.data, - }) - .from(EventTable) - .where(inArray(EventTable.id, page.ids)) - .orderBy(asc(EventTable.seq)) - .all() - .pipe(Effect.orDie) - const encoded = encodeReplayRequestPrefix(space.directory ?? "", events) - if (!encoded.valid || encoded.events.length === 0) - return yield* new SessionWarpHistoryLimitError({ - message: `Sync event ${events[0]!.id} cannot fit the bounded replay protocol`, - sessionID: input.sessionID, - eventID: events[0]!.id, - bytes: Buffer.byteLength( - JSON.stringify({ - directory: space.directory ?? "", - events: [events[0]], - }), - ), - }) - const response = yield* http.execute( - HttpClientRequest.post(route(target.url, "/sync/replay"), { - headers: new Headers(target.headers), - body: HttpBody.text(encoded.json, "application/json"), - }), - ) - if (response.status < 200 || response.status >= 300) { - const body = yield* response.text - return yield* new SessionWarpHttpError({ - message: `Failed to warp session ${input.sessionID} into workspace ${workspaceID}: HTTP ${response.status} ${body}`, - workspaceID, - sessionID: input.sessionID, - status: response.status, - body, - }) - } - cursor = encoded.events.at(-1)!.seq - total += encoded.events.length - if (encoded.complete && page.ids.length === candidates.length && candidates.length < pageEvents) break - } - if (total === 0) - return yield* new SessionEventsNotFoundError({ - message: `No events found for session: ${input.sessionID}`, - sessionID: input.sessionID, - }) - - const response = yield* http.execute( - HttpClientRequest.post(route(target.url, "/sync/steal"), { - headers: new Headers(target.headers), - body: HttpBody.jsonUnsafe({ sessionID: input.sessionID }), - }), - ) - if (response.status < 200 || response.status >= 300) { - const body = yield* response.text - log.error("session warp steal failed", { - workspaceID: input.workspaceID, + const recoveryAuthorityID = yield* SessionRecoveryTransferGuard.authorityID(db, input.sessionID) + if (recoveryAuthorityID) + return yield* new SessionWarpRecoveryProjectionError({ + message: + "Provider recovery projection cannot move between workspaces until canonical recovery snapshot import is available", sessionID: input.sessionID, - status: response.status, - body, + recoveryAuthorityID, }) - return yield* new SessionWarpHttpError({ - message: `Failed to steal session ${input.sessionID} into workspace ${workspaceID}: HTTP ${response.status} ${body}`, - workspaceID, + if (input.copyChanges) + return yield* new SessionWarpTransferUnsupportedError({ + message: "Copying workspace changes requires an idempotent patch transfer receipt", sessionID: input.sessionID, - status: response.status, - body, + reason: "copy_changes", }) - } - - yield* session.setWorkspace({ sessionID: input.sessionID, workspaceID: input.workspaceID }) - - log.info("session warp complete", { - workspaceID: input.workspaceID, + return yield* new SessionWarpTransferUnsupportedError({ + message: "Session placement changes require durable transfer admission, source fencing, and target receipts", sessionID: input.sessionID, - events: total, + reason: "placement_change", }) }).pipe( Effect.tapError((err) => @@ -1145,9 +814,7 @@ export const layer = Layer.effect( export const defaultLayer = layer.pipe( Layer.provide(Auth.defaultLayer), Layer.provide(Session.defaultLayer), - Layer.provide(SessionPrompt.defaultLayer), Layer.provide(Project.defaultLayer), - Layer.provide(Vcs.defaultLayer), Layer.provide(FSUtil.defaultLayer), Layer.provide(Database.defaultLayer), Layer.provide(EventV2Bridge.defaultLayer), diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/control-plane.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/control-plane.ts index 33c7e4d75..ba8048afe 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/control-plane.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/control-plane.ts @@ -1,6 +1,7 @@ import { MoveSession } from "@deepagent-code/core/control-plane/move-session" import { Schema } from "effect" import { HttpApi, HttpApiEndpoint, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi" +import { ConflictError } from "../errors" import { described } from "./metadata" const root = "/experimental/control-plane" @@ -22,12 +23,12 @@ export const ControlPlaneApi = HttpApi.make("controlPlane").add( HttpApiEndpoint.post("moveSession", `${root}/move-session`, { payload: MoveSessionPayload, success: described(HttpApiSchema.NoContent, "Session moved"), - error: ApiMoveSessionError, + error: [ApiMoveSessionError, ConflictError], }).annotateMerge( OpenApi.annotations({ identifier: "experimental.controlPlane.moveSession", summary: "Move session", - description: "Move a session to another project directory, optionally transferring local changes.", + description: "Move a session when a durable transfer implementation is available.", }), ), ) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/workspace.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/workspace.ts index c509af99c..b227e3fb2 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/workspace.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/workspace.ts @@ -3,7 +3,7 @@ import { WorkspaceAdapterEntry } from "@/control-plane/types" import { Schema, Struct } from "effect" import { HttpApi, HttpApiEndpoint, HttpApiError, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi" import { ApiVcsApplyError, ApiVcsRawDiffError } from "./instance" -import { ApiNotFoundError } from "../errors" +import { ApiNotFoundError, ConflictError } from "../errors" import { Authorization } from "../middleware/authorization" import { InstanceContextMiddleware } from "../middleware/instance-context" import { WorkspaceRoutingMiddleware, WorkspaceRoutingQuery } from "../middleware/workspace-routing" @@ -118,12 +118,12 @@ export const WorkspaceApi = HttpApi.make("workspace") query: WorkspaceRoutingQuery, payload: WarpPayload, success: described(HttpApiSchema.NoContent, "Session warped"), - error: [ApiWorkspaceWarpError, ApiVcsApplyError, ApiVcsRawDiffError, ApiNotFoundError], + error: [ApiWorkspaceWarpError, ApiVcsApplyError, ApiVcsRawDiffError, ApiNotFoundError, ConflictError], }).annotateMerge( OpenApi.annotations({ identifier: "experimental.workspace.warp", summary: "Warp session into workspace", - description: "Move a session's sync history into the target workspace, or detach it to the local project.", + description: "Move a session when a durable workspace transfer implementation is available.", }), ), ) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/control-plane.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/control-plane.ts index ee3c0a3fe..da0232d93 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/control-plane.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/control-plane.ts @@ -3,6 +3,7 @@ import { SessionV2 } from "@deepagent-code/core/session" import { Effect } from "effect" import { HttpApiBuilder } from "effect/unstable/httpapi" import { RootHttpApi } from "../api" +import { ConflictError } from "../errors" import { ApiMoveSessionError, MoveSessionPayload } from "../groups/control-plane" export const controlPlaneHandlers = HttpApiBuilder.group(RootHttpApi, "controlPlane", (handlers) => @@ -13,12 +14,13 @@ export const controlPlaneHandlers = HttpApiBuilder.group(RootHttpApi, "controlPl payload: typeof MoveSessionPayload.Type }) { yield* service.moveSession(ctx.payload).pipe( - Effect.mapError( - (error) => - new ApiMoveSessionError({ - name: "MoveSessionError", - data: { message: message(error) }, - }), + Effect.mapError((error) => + error instanceof MoveSession.TransferUnsupportedError + ? new ConflictError({ message: error.message, resource: `session:${error.sessionID}` }) + : new ApiMoveSessionError({ + name: "MoveSessionError", + data: { message: message(error) }, + }), ), ) }) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts index d36ce314e..d8399cd85 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts @@ -1,13 +1,11 @@ import { Workspace } from "@/control-plane/workspace" import * as InstanceState from "@/effect/instance-state" -import { Session } from "@/session/session" import { MessageV2 } from "@/session/message-v2" import { SessionV1 } from "@deepagent-code/core/v1/session" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" import { EventV2Bridge } from "@/event-v2-bridge" import { EventTable } from "@deepagent-code/core/event/sql" -import { SessionRecoveryTransferGuard } from "@/session/recovery-transfer-guard" import { asc } from "drizzle-orm" import { and } from "drizzle-orm" import { eq } from "drizzle-orm" @@ -59,7 +57,6 @@ END` export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handlers) => Effect.gen(function* () { const workspace = yield* Workspace.Service - const session = yield* Session.Service const scope = yield* Scope.Scope const events = yield* EventV2Bridge.Service const { db } = yield* Database.Service @@ -97,23 +94,10 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl const steal = Effect.fn("SyncHttpApi.steal")(function* (ctx: { payload: typeof SessionPayload.Type }) { const workspaceID = yield* InstanceState.workspaceID if (!workspaceID) return yield* new HttpApiError.BadRequest({}) - - const recoveryAuthorityID = yield* SessionRecoveryTransferGuard.authorityID(db, ctx.payload.sessionID) - if (recoveryAuthorityID) - return yield* new ConflictError({ - message: - "Provider recovery projection cannot move between workspaces until canonical snapshot import is available", - resource: recoveryAuthorityID, - }) - - yield* session.setWorkspace({ sessionID: ctx.payload.sessionID, workspaceID }) - - log.info("sync session stolen", { - sessionID: ctx.payload.sessionID, - workspaceID, + return yield* new ConflictError({ + message: "Session steal requires a durable transfer operation receipt and is disabled in this release", + resource: `${workspaceID}:${ctx.payload.sessionID}`, }) - - return { sessionID: ctx.payload.sessionID } }) const history = Effect.fn("SyncHttpApi.history")(function* (ctx: { payload: typeof HistoryPayload.Type }) { diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/workspace.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/workspace.ts index abacf5632..7cc74547f 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/workspace.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/workspace.ts @@ -8,6 +8,7 @@ import { InstanceHttpApi } from "../api" import { notFound } from "../errors" import { ApiVcsApplyError, ApiVcsRawDiffError } from "../groups/instance" import { ApiWorkspaceCreateError, ApiWorkspaceWarpError, CreatePayload, WarpPayload } from "../groups/workspace" +import { ConflictError } from "../errors" export const workspaceHandlers = HttpApiBuilder.group(InstanceHttpApi, "workspace", (handlers) => Effect.gen(function* () { @@ -71,6 +72,9 @@ export const workspaceHandlers = HttpApiBuilder.group(InstanceHttpApi, "workspac .pipe( Effect.mapError((error) => { if (error instanceof Workspace.WorkspaceNotFoundError) return notFound(error.message) + if (error instanceof Workspace.SessionWarpTransferUnsupportedError) { + return new ConflictError({ message: error.message, resource: `session:${error.sessionID}` }) + } if (error instanceof Vcs.PatchApplyError) { return new ApiVcsApplyError({ name: "VcsApplyError", diff --git a/packages/deepagent-code/test/control-plane/workspace.test.ts b/packages/deepagent-code/test/control-plane/workspace.test.ts index c675daef7..9709d934f 100644 --- a/packages/deepagent-code/test/control-plane/workspace.test.ts +++ b/packages/deepagent-code/test/control-plane/workspace.test.ts @@ -836,7 +836,7 @@ describe("workspace CRUD", () => { ) it.instance( - "sessionWarp moves a session into a local workspace and claims ownership", + "sessionWarp rejects a local placement change before ownership or workspace changes", () => { return Effect.gen(function* () { const { directory: dir } = yield* TestInstance @@ -854,20 +854,72 @@ describe("workspace CRUD", () => { const session = yield* sessionSvc.create({}) yield* attachSessionToWorkspace(session.id, previous.id) - yield* workspace.sessionWarp({ workspaceID: target.id, sessionID: session.id }) + const owner = yield* sessionSequenceOwner(session.id) + const error = yield* workspace.sessionWarp({ workspaceID: target.id, sessionID: session.id }).pipe(Effect.flip) + + expect(error).toBeInstanceOf(Workspace.SessionWarpTransferUnsupportedError) + if (!(error instanceof Workspace.SessionWarpTransferUnsupportedError)) return + expect(error.reason).toBe("placement_change") + expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(previous.id) + expect(yield* sessionSequenceOwner(session.id)).toBe(owner) + }) + }, + { git: true }, + ) + it.instance( + "sessionWarp treats an exact placement retry as a zero-write no-op", + () => + Effect.gen(function* () { + const instance = yield* requireInstance + const workspace = yield* Workspace.Service + const sessionSvc = yield* SessionNs.Service + const current = workspaceInfo(instance.project.id, unique("warp-no-op")) + yield* insertWorkspace(current) + const session = yield* sessionSvc.create({}) + yield* attachSessionToWorkspace(session.id, current.id) const { db } = yield* Database.Service + const beforeSession = yield* db + .select() + .from(SessionTable) + .where(eq(SessionTable.id, session.id)) + .get() + .pipe(Effect.orDie) + const beforeSequence = yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + const beforeEvents = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie) + + yield* workspace.sessionWarp({ workspaceID: current.id, sessionID: session.id }) + expect( - (yield* db - .select({ workspaceID: SessionTable.workspace_id }) - .from(SessionTable) - .where(eq(SessionTable.id, session.id)) + yield* db.select().from(SessionTable).where(eq(SessionTable.id, session.id)).get().pipe(Effect.orDie), + ).toEqual(beforeSession) + expect( + yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) .get() - .pipe(Effect.orDie))?.workspaceID, - ).toBe(target.id) - expect(yield* sessionSequenceOwner(session.id)).toBe(target.id) - }) - }, + .pipe(Effect.orDie), + ).toEqual(beforeSequence) + expect( + yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie), + ).toEqual(beforeEvents) + }), { git: true }, ) @@ -935,10 +987,11 @@ describe("workspace CRUD", () => { }) }, { git: true }, + 30_000, ) it.instance( - "sessionWarp applies source workspace patch to local target workspace", + "sessionWarp rejects local copyChanges before target files, ownership, or workspace change", () => { return Effect.gen(function* () { const { directory: dir } = yield* TestInstance @@ -962,18 +1015,34 @@ describe("workspace CRUD", () => { registerAdapter(instance.project.id, targetType, localAdapter(targetDir, { createDir: false }).adapter) const session = yield* sessionSvc.create({}) yield* attachSessionToWorkspace(session.id, previous.id) + const owner = yield* sessionSequenceOwner(session.id) - yield* workspace.sessionWarp({ workspaceID: target.id, sessionID: session.id, copyChanges: true }) + const error = yield* workspace + .sessionWarp({ workspaceID: target.id, sessionID: session.id, copyChanges: true }) + .pipe(Effect.flip) - expect(yield* Effect.promise(() => fs.readFile(path.join(targetDir, "tracked.txt"), "utf8"))).toBe("changed\n") - expect(yield* Effect.promise(() => fs.readFile(path.join(targetDir, "new.txt"), "utf8"))).toBe("new\n") + expect(error).toBeInstanceOf(Workspace.SessionWarpTransferUnsupportedError) + if (!(error instanceof Workspace.SessionWarpTransferUnsupportedError)) return + expect(error.reason).toBe("copy_changes") + expect(yield* Effect.promise(() => fs.readFile(path.join(targetDir, "tracked.txt"), "utf8"))).toBe("base\n") + expect( + yield* Effect.promise(() => + fs.stat(path.join(targetDir, "new.txt")).then( + () => true, + () => false, + ), + ), + ).toBe(false) + expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(previous.id) + expect(yield* sessionSequenceOwner(session.id)).toBe(owner) }) }, { git: true }, + 30_000, ) it.instance( - "sessionWarp detaches a session to the local project and claims project ownership", + "sessionWarp rejects detaching a session before ownership or workspace changes", () => { return Effect.gen(function* () { const { directory: dir } = yield* TestInstance @@ -987,18 +1056,14 @@ describe("workspace CRUD", () => { const session = yield* sessionSvc.create({}) yield* attachSessionToWorkspace(session.id, previous.id) - yield* workspace.sessionWarp({ workspaceID: null, sessionID: session.id }) + const owner = yield* sessionSequenceOwner(session.id) + const error = yield* workspace.sessionWarp({ workspaceID: null, sessionID: session.id }).pipe(Effect.flip) - const { db } = yield* Database.Service - expect( - (yield* db - .select({ workspaceID: SessionTable.workspace_id }) - .from(SessionTable) - .where(eq(SessionTable.id, session.id)) - .get() - .pipe(Effect.orDie))?.workspaceID, - ).toBeNull() - expect(yield* sessionSequenceOwner(session.id)).toBe(instance.project.id) + expect(error).toBeInstanceOf(Workspace.SessionWarpTransferUnsupportedError) + if (!(error instanceof Workspace.SessionWarpTransferUnsupportedError)) return + expect(error.reason).toBe("placement_change") + expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(previous.id) + expect(yield* sessionSequenceOwner(session.id)).toBe(owner) }) }, { git: true }, @@ -1006,7 +1071,7 @@ describe("workspace CRUD", () => { const itCrossInstance = process.platform === "win32" ? it.instance.skip : it.instance itCrossInstance( - "sessionWarp detaches to the source project when invoked from a workspace instance", + "sessionWarp rejects detach from a workspace instance before ownership changes", () => Effect.gen(function* () { const instance = yield* requireInstance @@ -1019,34 +1084,28 @@ describe("workspace CRUD", () => { const session = yield* sessionSvc.create({}) yield* attachSessionToWorkspace(session.id, previous.id) - const workspaceProjectID = yield* provideTmpdirInstance( + const owner = yield* sessionSequenceOwner(session.id) + yield* provideTmpdirInstance( (workspaceDir) => Effect.gen(function* () { registerAdapter(projectID, previousType, localAdapter(workspaceDir, { createDir: false }).adapter) const workspaceCtx = yield* requireInstance expect(workspaceCtx.project.id).not.toBe(projectID) - yield* workspace.sessionWarp({ workspaceID: null, sessionID: session.id }) - return workspaceCtx.project.id + const error = yield* workspace.sessionWarp({ workspaceID: null, sessionID: session.id }).pipe(Effect.flip) + expect(error).toBeInstanceOf(Workspace.SessionWarpTransferUnsupportedError) + if (!(error instanceof Workspace.SessionWarpTransferUnsupportedError)) return + expect(error.reason).toBe("placement_change") }), { git: true }, ) - const { db } = yield* Database.Service - expect( - (yield* db - .select({ workspaceID: SessionTable.workspace_id }) - .from(SessionTable) - .where(eq(SessionTable.id, session.id)) - .get() - .pipe(Effect.orDie))?.workspaceID, - ).toBeNull() - expect(yield* sessionSequenceOwner(session.id)).toBe(projectID) - expect(yield* sessionSequenceOwner(session.id)).not.toBe(workspaceProjectID) + expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(previous.id) + expect(yield* sessionSequenceOwner(session.id)).toBe(owner) }), { git: true }, ) - it.live("sessionWarp syncs previous remote history, replays it, steals, and claims the sequence", () => { + it.live("sessionWarp rejects a remote source before patch, sync, replay, steal, or ownership side effects", () => { const calls: FetchCall[] = [] let historySessionID: SessionID | undefined let historySession: SessionNs.Info | undefined @@ -1107,133 +1166,170 @@ describe("workspace CRUD", () => { historySession = { ...session, workspaceID: previous.id, title: "from source history" } historyNextSeq = ((yield* sessionSequence(session.id)) ?? -1) + 1 - yield* workspace.sessionWarp({ workspaceID: target.id, sessionID: session.id, copyChanges: true }) + const owner = yield* sessionSequenceOwner(session.id) + const error = yield* workspace + .sessionWarp({ workspaceID: target.id, sessionID: session.id, copyChanges: true }) + .pipe(Effect.flip) - expect(calls.map((call) => `${call.method} ${call.url.pathname}`)).toEqual([ - "GET /warp-source/vcs/diff/raw", - "POST /warp-target/vcs/apply", - "POST /warp-source/sync/history", - "POST /warp-source/sync/history", - "POST /warp-target/sync/replay", - "POST /warp-target/sync/steal", - ]) - expect(calls[1].json).toEqual({ patch: "remote patch" }) - expect(calls[2].json).toEqual({ [session.id]: historyNextSeq - 1 }) - expect(calls[3].json).toEqual({ [session.id]: historyNextSeq }) - expect(calls[4].json).toMatchObject({ - directory: "remote-target-dir", - events: [ - { - aggregateID: session.id, - seq: 0, - type: "session.created.1", - }, - { - aggregateID: session.id, - seq: historyNextSeq, - type: "session.updated.1", - }, - ], - }) - expect(calls[5].json).toEqual({ sessionID: session.id }) - expect((yield* sessionSvc.get(session.id)).title).toBe("from source history") - expect(yield* sessionSequenceOwner(session.id)).toBe(target.id) + expect(error).toBeInstanceOf(Workspace.SessionWarpTransferUnsupportedError) + if (!(error instanceof Workspace.SessionWarpTransferUnsupportedError)) return + expect(error.reason).toBe("copy_changes") + expect(calls).toEqual([]) + expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(previous.id) + expect((yield* sessionSvc.get(session.id)).title).not.toBe("from source history") + expect(yield* sessionSequenceOwner(session.id)).toBe(owner) }), { git: true }, ) }) }) - it.live("sessionWarp replays a maximum admitted event within the final request budget before steal", () => { + it.live("sessionWarp rejects a remote target before replay, steal, ownership, or workspace side effects", () => { const calls: FetchCall[] = [] return Effect.gen(function* () { yield* HttpServer.serveEffect()( Effect.gen(function* () { const req = yield* HttpServerRequest.HttpServerRequest - const bodyText = yield* req.text calls.push({ url: new URL(req.url, "http://localhost"), method: req.method, headers: new Headers(req.headers), - bodyText, - json: bodyText ? JSON.parse(bodyText) : undefined, }) - if (req.url.endsWith("/sync/replay")) return yield* HttpServerResponse.json({ sessionID: "ok" }) - if (req.url.endsWith("/sync/steal")) return yield* HttpServerResponse.json({ sessionID: "ok" }) return HttpServerResponse.text("unexpected", { status: 500 }) }), ) const url = yield* serverUrl() yield* provideTmpdirInstance( - () => + (dir) => Effect.gen(function* () { const workspace = yield* Workspace.Service const sessionSvc = yield* SessionNs.Service const instance = yield* requireInstance - const type = unique("warp-max-event") - const target = workspaceInfo(instance.project.id, type, { directory: "remote-target-dir" }) + const previousType = unique("warp-local-source") + const targetType = unique("warp-remote-target") + const previous = workspaceInfo(instance.project.id, previousType) + const target = workspaceInfo(instance.project.id, targetType, { directory: "remote-target-dir" }) + yield* insertWorkspace(previous) yield* insertWorkspace(target) - registerAdapter(instance.project.id, type, remoteAdapter(`${url}/warp-target`).adapter) + registerAdapter( + instance.project.id, + previousType, + localAdapter(path.join(dir, "warp-local-source")).adapter, + ) + registerAdapter(instance.project.id, targetType, remoteAdapter(`${url}/warp-target`).adapter) const session = yield* sessionSvc.create({}) - const { db } = yield* Database.Service - const sequence = (yield* sessionSequence(session.id))! - const base = { - sessionID: session.id, - info: { - id: SessionV1.MessageID.ascending(), - sessionID: session.id, - role: "user" as const, - time: { created: Date.now() }, - agent: "build", - model: { providerID: "test", modelID: "model" }, - system: "", - }, - } - const data = { - ...base, - info: { - ...base.info, - system: "x".repeat(EventV2.MAX_ENCODED_PAYLOAD_BYTES - Buffer.byteLength(JSON.stringify(base))), - }, - } - expect(Buffer.byteLength(JSON.stringify(data))).toBe(EventV2.MAX_ENCODED_PAYLOAD_BYTES) - yield* db - .insert(EventTable) - .values({ - id: EventV2.ID.make("evt_warp_max_admitted"), - aggregate_id: session.id, - seq: sequence + 1, - type: EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1), - data, - }) - .run() - .pipe(Effect.orDie) - yield* db - .update(EventSequenceTable) - .set({ seq: sequence + 1 }) - .where(eq(EventSequenceTable.aggregate_id, session.id)) - .run() - .pipe(Effect.orDie) - - yield* workspace.sessionWarp({ workspaceID: target.id, sessionID: session.id }) - - const replay = calls.filter((call) => call.url.pathname.endsWith("/sync/replay")) - expect(replay).toHaveLength(2) - expect(replay.flatMap((call) => (call.json as { events: Array<{ id: string }> }).events.map((e) => e.id))).toEqual([ - expect.stringMatching(/^evt_/), - "evt_warp_max_admitted", - ]) - expect(Buffer.byteLength(replay[1]!.bodyText!)).toBeGreaterThan(SyncReplayLimits.eventDataBytes) - expect(Buffer.byteLength(replay[1]!.bodyText!)).toBeLessThanOrEqual(SyncReplayLimits.requestBytes) - expect(calls.at(-1)?.url.pathname).toBe("/warp-target/sync/steal") - expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(target.id) + yield* attachSessionToWorkspace(session.id, previous.id) + const owner = yield* sessionSequenceOwner(session.id) + + const error = yield* workspace + .sessionWarp({ workspaceID: target.id, sessionID: session.id }) + .pipe(Effect.flip) + + expect(error).toBeInstanceOf(Workspace.SessionWarpTransferUnsupportedError) + if (!(error instanceof Workspace.SessionWarpTransferUnsupportedError)) return + expect(error.reason).toBe("placement_change") + expect(calls).toEqual([]) + expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(previous.id) + expect(yield* sessionSequenceOwner(session.id)).toBe(owner) }), { git: true }, ) }) - }, 30_000) + }) - it.live("sessionWarp rejects an oversized chunked source patch before side effects or ownership changes", () => { + it.live( + "sessionWarp rejects a remote target before reading or replaying maximum admitted events", + () => { + const calls: FetchCall[] = [] + return Effect.gen(function* () { + yield* HttpServer.serveEffect()( + Effect.gen(function* () { + const req = yield* HttpServerRequest.HttpServerRequest + const bodyText = yield* req.text + calls.push({ + url: new URL(req.url, "http://localhost"), + method: req.method, + headers: new Headers(req.headers), + bodyText, + json: bodyText ? JSON.parse(bodyText) : undefined, + }) + if (req.url.endsWith("/sync/replay")) return yield* HttpServerResponse.json({ sessionID: "ok" }) + if (req.url.endsWith("/sync/steal")) return yield* HttpServerResponse.json({ sessionID: "ok" }) + return HttpServerResponse.text("unexpected", { status: 500 }) + }), + ) + const url = yield* serverUrl() + yield* provideTmpdirInstance( + () => + Effect.gen(function* () { + const workspace = yield* Workspace.Service + const sessionSvc = yield* SessionNs.Service + const instance = yield* requireInstance + const type = unique("warp-max-event") + const target = workspaceInfo(instance.project.id, type, { directory: "remote-target-dir" }) + yield* insertWorkspace(target) + registerAdapter(instance.project.id, type, remoteAdapter(`${url}/warp-target`).adapter) + const session = yield* sessionSvc.create({}) + const { db } = yield* Database.Service + const sequence = (yield* sessionSequence(session.id))! + const base = { + sessionID: session.id, + info: { + id: SessionV1.MessageID.ascending(), + sessionID: session.id, + role: "user" as const, + time: { created: Date.now() }, + agent: "build", + model: { providerID: "test", modelID: "model" }, + system: "", + }, + } + const data = { + ...base, + info: { + ...base.info, + system: "x".repeat(EventV2.MAX_ENCODED_PAYLOAD_BYTES - Buffer.byteLength(JSON.stringify(base))), + }, + } + expect(Buffer.byteLength(JSON.stringify(data))).toBe(EventV2.MAX_ENCODED_PAYLOAD_BYTES) + yield* db + .insert(EventTable) + .values({ + id: EventV2.ID.make("evt_warp_max_admitted"), + aggregate_id: session.id, + seq: sequence + 1, + type: EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1), + data, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ seq: sequence + 1 }) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .run() + .pipe(Effect.orDie) + + const owner = yield* sessionSequenceOwner(session.id) + const error = yield* workspace + .sessionWarp({ workspaceID: target.id, sessionID: session.id }) + .pipe(Effect.flip) + + expect(error).toBeInstanceOf(Workspace.SessionWarpTransferUnsupportedError) + if (!(error instanceof Workspace.SessionWarpTransferUnsupportedError)) return + expect(error.reason).toBe("placement_change") + expect(calls).toEqual([]) + expect((yield* sessionSvc.get(session.id)).workspaceID).toBeUndefined() + expect(yield* sessionSequenceOwner(session.id)).toBe(owner) + }), + { git: true }, + ) + }) + }, + 30_000, + ) + + it.live("sessionWarp rejects a remote source before reading an oversized patch or changing ownership", () => { const calls: FetchCall[] = [] return Effect.gen(function* () { yield* HttpServer.serveEffect()( @@ -1250,10 +1346,7 @@ describe("workspace CRUD", () => { calls.push(call) if (call.url.pathname === "/raw-limit-source/vcs/diff/raw") return HttpServerResponse.stream( - Stream.make( - new Uint8Array(Vcs.RawDiffLimits.patchBytes), - new Uint8Array([120]), - ), + Stream.make(new Uint8Array(Vcs.RawDiffLimits.patchBytes), new Uint8Array([120])), { contentType: "text/x-diff" }, ) return HttpServerResponse.text("unexpected", { status: 500 }) @@ -1278,20 +1371,14 @@ describe("workspace CRUD", () => { yield* attachSessionToWorkspace(session.id, previous.id) const owner = yield* sessionSequenceOwner(session.id) - const exit = yield* workspace + const error = yield* workspace .sessionWarp({ workspaceID: target.id, sessionID: session.id, copyChanges: true }) - .pipe(Effect.exit) - - expect(exit._tag).toBe("Failure") - if (exit._tag === "Failure") - expect(Cause.squash(exit.cause)).toMatchObject({ - _tag: "VcsRawDiffError", - reason: "remote-output", - limit: Vcs.RawDiffLimits.patchBytes, - }) - expect(calls.map((call) => `${call.method} ${call.url.pathname}`)).toEqual([ - "GET /raw-limit-source/vcs/diff/raw", - ]) + .pipe(Effect.flip) + + expect(error).toBeInstanceOf(Workspace.SessionWarpTransferUnsupportedError) + if (!(error instanceof Workspace.SessionWarpTransferUnsupportedError)) return + expect(error.reason).toBe("copy_changes") + expect(calls).toEqual([]) expect((yield* sessionSvc.get(session.id)).workspaceID).toBe(previous.id) expect(yield* sessionSequenceOwner(session.id)).toBe(owner) }), @@ -1589,8 +1676,7 @@ describe("workspace sync state", () => { if (url.pathname === "/history/sync/history") { const body = bodyText ? JSON.parse(bodyText) : undefined historyBodies.push(body) - if (body?.[historySessionID!] === historyNextSeq) - return HttpServerResponse.fromWeb(Response.json([])) + if (body?.[historySessionID!] === historyNextSeq) return HttpServerResponse.fromWeb(Response.json([])) return HttpServerResponse.fromWeb( Response.json([ { @@ -1632,10 +1718,7 @@ describe("workspace sync state", () => { expect((yield* sessionSvc.get(session.id).pipe(Effect.orDie)).title).toBe("from history") }), ) - expect(historyBodies).toEqual([ - { [session.id]: historyNextSeq - 1 }, - { [session.id]: historyNextSeq }, - ]) + expect(historyBodies).toEqual([{ [session.id]: historyNextSeq - 1 }, { [session.id]: historyNextSeq }]) expect( captured.events.some( (event) => diff --git a/packages/deepagent-code/test/server/httpapi-control-plane.test.ts b/packages/deepagent-code/test/server/httpapi-control-plane.test.ts index ac9f809f5..71676e77d 100644 --- a/packages/deepagent-code/test/server/httpapi-control-plane.test.ts +++ b/packages/deepagent-code/test/server/httpapi-control-plane.test.ts @@ -25,6 +25,11 @@ const input = MoveSession.Input.make({ destination: { directory: AbsolutePath.make("/destination") }, moveChanges: true, }) +const unsupported = MoveSession.Input.make({ + sessionID: SessionV2.ID.make("ses_move_unsupported"), + destination: { directory: AbsolutePath.make("/unsupported") }, + moveChanges: true, +}) const called = Ref.makeUnsafe(undefined) const apiLayer = HttpRouter.serve( @@ -43,7 +48,17 @@ const apiLayer = HttpRouter.serve( Layer.provide(Layer.mock(Installation.Service)({})), Layer.provide( Layer.mock(MoveSession.Service)({ - moveSession: (value) => Ref.set(called, value), + moveSession: (value) => + value.destination.directory === unsupported.destination.directory + ? Effect.fail( + new MoveSession.TransferUnsupportedError({ + sessionID: value.sessionID, + source: AbsolutePath.make("/source"), + destination: value.destination.directory, + message: "durable transfer is unavailable", + }), + ) + : Ref.set(called, value), }), ), Layer.provide(ServerAuth.Config.layer({ password: Option.none(), username: "deepagent-code" })), @@ -64,4 +79,20 @@ describe("control-plane HttpApi", () => { expect(yield* Ref.get(called)).toEqual(input) }), ) + + it.live("returns a declared conflict while durable session transfer is unavailable", () => + Effect.gen(function* () { + const response = yield* HttpClientRequest.post("/experimental/control-plane/move-session").pipe( + HttpClientRequest.setBody(HttpBody.jsonUnsafe(unsupported)), + HttpClient.execute, + ) + + expect(response.status).toBe(409) + expect(yield* response.json).toEqual({ + _tag: "ConflictError", + message: "durable transfer is unavailable", + resource: `session:${unsupported.sessionID}`, + }) + }), + ) }) diff --git a/packages/deepagent-code/test/server/httpapi-workspace.test.ts b/packages/deepagent-code/test/server/httpapi-workspace.test.ts index 26cb57bb3..0f6345b7b 100644 --- a/packages/deepagent-code/test/server/httpapi-workspace.test.ts +++ b/packages/deepagent-code/test/server/httpapi-workspace.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, mock } from "bun:test" import { mkdir } from "node:fs/promises" import path from "node:path" import { Effect, Layer, Stream } from "effect" +import { eq } from "drizzle-orm" import { Flag } from "@deepagent-code/core/flag/flag" import { registerAdapter } from "../../src/control-plane/adapters" import { WorkspaceV2 } from "@deepagent-code/core/workspace" @@ -11,6 +12,8 @@ import { WorkspacePaths } from "../../src/server/routes/instance/httpapi/groups/ import { EventPaths } from "../../src/server/routes/instance/httpapi/groups/event" import { Session } from "@/session/session" import { Database } from "@deepagent-code/core/database/database" +import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { SessionTable } from "@deepagent-code/core/session/sql" import * as Log from "@deepagent-code/core/util/log" import { Server } from "../../src/server/server" import { resetDatabase } from "../fixture/db" @@ -21,6 +24,7 @@ import { Project } from "../../src/project/project" import { InstancePaths } from "../../src/server/routes/instance/httpapi/groups/instance" import { testEffect } from "../lib/effect" import { httpApiLayer, requestInDirectory } from "./httpapi-layer" +import { SyncPaths } from "../../src/server/routes/instance/httpapi/groups/sync" void Log.init({ print: false }) @@ -226,12 +230,69 @@ describe("workspace HttpApi", () => { expect(workspace).toMatchObject({ type: "local-test", name: "local-test" }) const session = yield* Session.use.create({}).pipe(provideInstance(dir)) + const { db } = yield* Database.Service + const beforeSession = yield* db + .select() + .from(SessionTable) + .where(eq(SessionTable.id, session.id)) + .get() + .pipe(Effect.orDie) + const beforeSequence = yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + const beforeEvents = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie) const warped = yield* request(WorkspacePaths.warp, dir, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ id: workspace.id, sessionID: session.id }), }) - expect(warped.status).toBe(204) + expect(warped.status).toBe(409) + expect(yield* warped.json).toMatchObject({ + _tag: "ConflictError", + message: expect.stringContaining("durable transfer admission"), + resource: `session:${session.id}`, + }) + + const stealURL = new URL(`http://localhost${SyncPaths.steal}`) + stealURL.searchParams.set("workspace", workspace.id) + const stolen = yield* request(stealURL.toString(), dir, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sessionID: session.id }), + }) + expect(stolen.status).toBe(409) + expect(yield* stolen.json).toMatchObject({ + _tag: "ConflictError", + message: expect.stringContaining("durable transfer operation receipt"), + resource: `${workspace.id}:${session.id}`, + }) + expect( + yield* db.select().from(SessionTable).where(eq(SessionTable.id, session.id)).get().pipe(Effect.orDie), + ).toEqual(beforeSession) + expect( + yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie), + ).toEqual(beforeSequence) + expect( + yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie), + ).toEqual(beforeEvents) const removed = yield* request(WorkspacePaths.remove.replace(":id", workspace.id), dir, { method: "DELETE" }) expect(removed.status).toBe(200) @@ -446,7 +507,7 @@ describe("workspace HttpApi", () => { }), ) - it.live("proxies remote workspace requests selected from session ownership", () => + it.live("rejects remote workspace placement before proxy or ownership side effects", () => Effect.gen(function* () { Flag.DEEPAGENT_CODE_EXPERIMENTAL_WORKSPACES = true const dir = yield* tmpdirScoped({ git: true }) @@ -478,34 +539,20 @@ describe("workspace HttpApi", () => { headers: { "content-type": "application/json" }, body: JSON.stringify({ id: workspace.id, sessionID: session.id }), }) - expect(warped.status).toBe(204) + expect(warped.status).toBe(409) + expect(yield* warped.json).toMatchObject({ + _tag: "ConflictError", + message: expect.stringContaining("durable transfer admission"), + resource: `session:${session.id}`, + }) try { - const response = yield* requestDefault(`http://localhost/session/${session.id}/message`, dir, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ parts: [{ type: "text", text: "hello" }] }), - }) - - const responseBody = yield* response.text - expect({ status: response.status, body: responseBody }).toMatchObject({ status: 200 }) - expect(JSON.parse(responseBody)).toEqual({ proxied: true, path: `/base/session/${session.id}/message` }) - expect(proxied.filter((item) => new URL(item.url).pathname === `/base/session/${session.id}/message`)).toEqual([ - expect.objectContaining({ - url: `http://127.0.0.1:${remote.port}/base/session/${session.id}/message`, - method: "POST", - }), - ]) - - const aborted = yield* request(`http://localhost/session/${session.id}/abort`, dir, { method: "POST" }) - expect(aborted.status).toBe(200) - expect(proxied.filter((item) => new URL(item.url).pathname === `/base/session/${session.id}/abort`)).toEqual([ - expect.objectContaining({ - url: `http://127.0.0.1:${remote.port}/base/session/${session.id}/abort`, - method: "POST", - body: "", - }), - ]) + expect( + proxied.filter((item) => + ["/base/vcs/apply", "/base/sync/replay", "/base/sync/steal"].includes(new URL(item.url).pathname), + ), + ).toEqual([]) + expect((yield* Session.use.get(session.id)).workspaceID).toBeUndefined() } finally { void remote.stop(true) yield* requestDefault(WorkspacePaths.remove.replace(":id", workspace.id), dir, { method: "DELETE" }) From 9e078049a5d9346fbc3e6d004da64e349eaeff36 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 00:12:23 +0800 Subject: [PATCH 17/44] chore(sdk): regenerate transfer conflict types --- packages/sdk/js/src/gen/sdk.gen.ts | 4 ++-- packages/sdk/js/src/gen/types.gen.ts | 20 ++++++++++++++------ packages/sdk/js/src/v2/gen/sdk.gen.ts | 4 ++-- packages/sdk/js/src/v2/gen/types.gen.ts | 20 ++++++++++++++------ 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/packages/sdk/js/src/gen/sdk.gen.ts b/packages/sdk/js/src/gen/sdk.gen.ts index 35c290b0c..2a33b6a15 100644 --- a/packages/sdk/js/src/gen/sdk.gen.ts +++ b/packages/sdk/js/src/gen/sdk.gen.ts @@ -782,7 +782,7 @@ export class ControlPlane extends HeyApiClient { /** * Move session * - * Move a session to another project directory, optionally transferring local changes. + * Move a session when a durable transfer implementation is available. */ public moveSession( parameters: { @@ -1403,7 +1403,7 @@ export class Workspace extends HeyApiClient { /** * Warp session into workspace * - * Move a session's sync history into the target workspace, or detach it to the local project. + * Move a session when a durable workspace transfer implementation is available. */ public warp( parameters: { diff --git a/packages/sdk/js/src/gen/types.gen.ts b/packages/sdk/js/src/gen/types.gen.ts index ad320bc40..b853e4d88 100644 --- a/packages/sdk/js/src/gen/types.gen.ts +++ b/packages/sdk/js/src/gen/types.gen.ts @@ -160,6 +160,12 @@ export type MoveSessionError = { } } +export type ConflictError = { + _tag: "ConflictError" + message: string + resource?: string +} + export type SnapshotFileDiff = { file?: string patch?: string @@ -3823,12 +3829,6 @@ export type Session5 = { preview?: string } -export type ConflictError = { - _tag: "ConflictError" - message: string - resource?: string -} - export type Session6 = { id: string slug: string @@ -7132,6 +7132,10 @@ export type ExperimentalControlPlaneMoveSessionErrors = { * MoveSessionError | InvalidRequestError */ 400: MoveSessionError | InvalidRequestError + /** + * ConflictError + */ + 409: ConflictError } export type ExperimentalControlPlaneMoveSessionError = @@ -15865,6 +15869,10 @@ export type ExperimentalWorkspaceWarpErrors = { * NotFoundError */ 404: NotFoundError + /** + * ConflictError + */ + 409: ConflictError /** * VcsRawDiffError */ diff --git a/packages/sdk/js/src/v2/gen/sdk.gen.ts b/packages/sdk/js/src/v2/gen/sdk.gen.ts index 35c290b0c..2a33b6a15 100644 --- a/packages/sdk/js/src/v2/gen/sdk.gen.ts +++ b/packages/sdk/js/src/v2/gen/sdk.gen.ts @@ -782,7 +782,7 @@ export class ControlPlane extends HeyApiClient { /** * Move session * - * Move a session to another project directory, optionally transferring local changes. + * Move a session when a durable transfer implementation is available. */ public moveSession( parameters: { @@ -1403,7 +1403,7 @@ export class Workspace extends HeyApiClient { /** * Warp session into workspace * - * Move a session's sync history into the target workspace, or detach it to the local project. + * Move a session when a durable workspace transfer implementation is available. */ public warp( parameters: { diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index ad320bc40..b853e4d88 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -160,6 +160,12 @@ export type MoveSessionError = { } } +export type ConflictError = { + _tag: "ConflictError" + message: string + resource?: string +} + export type SnapshotFileDiff = { file?: string patch?: string @@ -3823,12 +3829,6 @@ export type Session5 = { preview?: string } -export type ConflictError = { - _tag: "ConflictError" - message: string - resource?: string -} - export type Session6 = { id: string slug: string @@ -7132,6 +7132,10 @@ export type ExperimentalControlPlaneMoveSessionErrors = { * MoveSessionError | InvalidRequestError */ 400: MoveSessionError | InvalidRequestError + /** + * ConflictError + */ + 409: ConflictError } export type ExperimentalControlPlaneMoveSessionError = @@ -15865,6 +15869,10 @@ export type ExperimentalWorkspaceWarpErrors = { * NotFoundError */ 404: NotFoundError + /** + * ConflictError + */ + 409: ConflictError /** * VcsRawDiffError */ From 5c81417fcca99dcf9f6c7095fca367b4bde11b7c Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 01:07:11 +0800 Subject: [PATCH 18/44] fix(session): close activity crash restart matrix --- packages/deepagent-code/src/effect/runner.ts | 19 +- .../src/session/activity-crash-test.ts | 6 +- .../src/session/prompt-intent.ts | 127 +++++- packages/deepagent-code/src/session/prompt.ts | 84 ++-- .../deepagent-code/src/session/run-state.ts | 25 +- .../deepagent-code/test/effect/runner.test.ts | 32 +- .../test/session/prompt-intent.test.ts | 199 ++++++++ .../live-llm/activity-progress-restart.ts | 428 +++++++++++++++--- 8 files changed, 820 insertions(+), 100 deletions(-) diff --git a/packages/deepagent-code/src/effect/runner.ts b/packages/deepagent-code/src/effect/runner.ts index d8c96b04d..40f946192 100644 --- a/packages/deepagent-code/src/effect/runner.ts +++ b/packages/deepagent-code/src/effect/runner.ts @@ -5,6 +5,8 @@ export interface Runner { readonly busy: boolean readonly ensureRunning: (work: Effect.Effect, onRunning?: Effect.Effect) => Effect.Effect readonly startRunning: (work: Effect.Effect) => Effect.Effect + readonly markFinalizing: Effect.Effect + readonly markRunning: Effect.Effect readonly startShell: (work: Effect.Effect, ready?: Latch.Latch) => Effect.Effect readonly cancel: Effect.Effect } @@ -34,6 +36,7 @@ interface PendingHandle { export type State = | { readonly _tag: "Idle" } | { readonly _tag: "Running"; readonly run: RunHandle } + | { readonly _tag: "Finalizing"; readonly run: RunHandle } | { readonly _tag: "Shell"; readonly shell: ShellHandle } | { readonly _tag: "ShellThenRun"; readonly shell: ShellHandle; readonly run: PendingHandle } @@ -74,10 +77,10 @@ export const make = ( (st) => [ Effect.gen(function* () { - if (st._tag === "Running" && st.run.id === id) yield* idle + if ((st._tag === "Running" || st._tag === "Finalizing") && st.run.id === id) yield* idle yield* complete(done, exit) }), - st._tag === "Running" && st.run.id === id ? ({ _tag: "Idle" } as const) : st, + (st._tag === "Running" || st._tag === "Finalizing") && st.run.id === id ? ({ _tag: "Idle" } as const) : st, ] as const, ).pipe(Effect.flatten) @@ -121,6 +124,7 @@ export const make = ( onRunning ? onRunning.pipe(Effect.andThen(awaitDone(done))) : awaitDone(done) switch (st._tag) { case "Running": + case "Finalizing": case "ShellThenRun": return [awaitRunning(st.run.done), st] as const case "Shell": { @@ -140,6 +144,14 @@ export const make = ( }), ).pipe(Effect.flatten) + const markFinalizing = SynchronizedRef.update(ref, (st) => + st._tag === "Running" ? ({ _tag: "Finalizing", run: st.run } as const) : st, + ) + + const markRunning = SynchronizedRef.update(ref, (st) => + st._tag === "Finalizing" ? ({ _tag: "Running", run: st.run } as const) : st, + ) + const startRunning = (work: Effect.Effect): Effect.Effect => SynchronizedRef.modifyEffect( ref, @@ -191,6 +203,7 @@ export const make = ( case "Idle": return [Effect.void, st] as const case "Running": + case "Finalizing": return [ Effect.gen(function* () { yield* Fiber.interrupt(st.run.fiber) @@ -228,6 +241,8 @@ export const make = ( }, ensureRunning, startRunning, + markFinalizing, + markRunning, startShell, cancel, } diff --git a/packages/deepagent-code/src/session/activity-crash-test.ts b/packages/deepagent-code/src/session/activity-crash-test.ts index ff7279363..4602d9efe 100644 --- a/packages/deepagent-code/src/session/activity-crash-test.ts +++ b/packages/deepagent-code/src/session/activity-crash-test.ts @@ -3,10 +3,14 @@ import path from "node:path" import { Effect } from "effect" export type Point = + | "after_coordinator_reserve" | "after_admit_and_bind" + | "after_provider_prepared" | "after_provider_streaming" | "after_provider_receipt_terminal" - | "after_progress_settled" + | "inside_revision_terminal_transaction" + | "after_terminal_commit_before_publish" + | "while_finalizing_before_follow_up_drain" export function pause(point: Point) { if (process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT !== point) return Effect.void diff --git a/packages/deepagent-code/src/session/prompt-intent.ts b/packages/deepagent-code/src/session/prompt-intent.ts index 387844633..037bf9e40 100644 --- a/packages/deepagent-code/src/session/prompt-intent.ts +++ b/packages/deepagent-code/src/session/prompt-intent.ts @@ -23,6 +23,7 @@ import { } from "./activity-sql" import { SessionToolRequestReceiptTable } from "./tool-request-receipt.sql" import { SessionActivityOwner } from "./activity-owner" +import { pause as pauseAtActivityCrashPoint } from "./activity-crash-test" export type Source = "composer" | "intelligence" | "followup" | "rewrite" export type Variant = "original" | "rewritten" @@ -230,7 +231,7 @@ export const claim = Effect.fn("SessionPromptIntent.claim")(function* (input: { }) { const { db } = yield* Database.Service const now = Date.now() - const ownerToken = randomUUID() + const ownerToken = `${SessionActivityOwner.processOwnerToken}:${randomUUID()}` const executionMode = input.executionMode ?? "run_now" return yield* db .transaction( @@ -356,7 +357,12 @@ export const claim = Effect.fn("SessionPromptIntent.claim")(function* (input: { const receipt = fromRow(admitted) return { kind: "admitted" as const, receipt: { ...receipt, state: "admitted" as const, messageID } } } - if (existing.state === "admitting" && existing.lease_expires_at !== null && existing.lease_expires_at > now) { + if ( + existing.state === "admitting" && + existing.lease_expires_at !== null && + existing.lease_expires_at > now && + claimOwnerMayStillBeAlive(existing.owner_token) + ) { return yield* Effect.fail(new InProgress({ intentID: input.intentID })) } @@ -395,6 +401,25 @@ export const claim = Effect.fn("SessionPromptIntent.claim")(function* (input: { .pipe(Effect.catchTag("SqlError", Effect.die)) }) +function claimOwnerMayStillBeAlive(ownerToken: string | null) { + if (!ownerToken) return true + const [pid, processToken, claimToken, ...rest] = ownerToken.split(":") + if ( + rest.length > 0 || + !/^\d+$/.test(pid) || + !/^[0-9a-f-]{36}$/.test(processToken) || + !/^[0-9a-f-]{36}$/.test(claimToken) + ) + return true + if (`${pid}:${processToken}` === SessionActivityOwner.processOwnerToken) return true + try { + process.kill(Number(pid), 0) + return true + } catch (error) { + return !(error instanceof Error && "code" in error && error.code === "ESRCH") + } +} + export const complete = Effect.fn("SessionPromptIntent.complete")(function* (input: { readonly intentID: string readonly ownerToken: string @@ -1197,6 +1222,102 @@ export const claimActiveActivityRun = Effect.fn("SessionPromptIntent.claimActive .pipe(Effect.catchTag("SqlError", Effect.die)) }) +export const markRunFinalizing = Effect.fn("SessionPromptIntent.markRunFinalizing")(function* (run: RunIdentity) { + const { db } = yield* Database.Service + return yield* db + .transaction( + (tx) => + Effect.gen(function* () { + const current = yield* tx + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, run.runID)) + .get() + if ( + !current || + current.activity_id !== run.activityID || + current.session_id !== run.sessionID || + current.mutation_epoch !== run.mutationEpoch || + current.generation !== run.generation || + current.owner_token !== run.ownerToken || + !["running", "finalizing"].includes(current.state) + ) + return yield* Effect.fail( + new Conflict({ intentID: run.runID, reason: "activity run cannot enter finalizing" }), + ) + if (current.state === "finalizing") return + const updated = yield* tx + .update(SessionLegacyActivityRunTable) + .set({ state: "finalizing" }) + .where( + and( + eq(SessionLegacyActivityRunTable.run_id, run.runID), + eq(SessionLegacyActivityRunTable.owner_token, run.ownerToken), + eq(SessionLegacyActivityRunTable.state, "running"), + ), + ) + .returning({ runID: SessionLegacyActivityRunTable.run_id }) + .get() + if (!updated) + return yield* Effect.fail( + new Conflict({ intentID: run.runID, reason: "activity run finalizing CAS lost" }), + ) + }), + { behavior: "immediate" }, + ) + .pipe( + Effect.catchTag("EffectDrizzleQueryError", Effect.die), + Effect.catchTag("SqlError", Effect.die), + ) +}) + +export const markRunRunning = Effect.fn("SessionPromptIntent.markRunRunning")(function* (run: RunIdentity) { + const { db } = yield* Database.Service + return yield* db + .transaction( + (tx) => + Effect.gen(function* () { + const current = yield* tx + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, run.runID)) + .get() + if ( + !current || + current.activity_id !== run.activityID || + current.session_id !== run.sessionID || + current.mutation_epoch !== run.mutationEpoch || + current.generation !== run.generation || + current.owner_token !== run.ownerToken || + !["running", "finalizing"].includes(current.state) + ) + return yield* Effect.fail( + new Conflict({ intentID: run.runID, reason: "activity run cannot resume running" }), + ) + if (current.state === "running") return + const updated = yield* tx + .update(SessionLegacyActivityRunTable) + .set({ state: "running" }) + .where( + and( + eq(SessionLegacyActivityRunTable.run_id, run.runID), + eq(SessionLegacyActivityRunTable.owner_token, run.ownerToken), + eq(SessionLegacyActivityRunTable.state, "finalizing"), + ), + ) + .returning({ runID: SessionLegacyActivityRunTable.run_id }) + .get() + if (!updated) + return yield* Effect.fail(new Conflict({ intentID: run.runID, reason: "activity run resume CAS lost" })) + }), + { behavior: "immediate" }, + ) + .pipe( + Effect.catchTag("EffectDrizzleQueryError", Effect.die), + Effect.catchTag("SqlError", Effect.die), + ) +}) + export const activityForMessage = Effect.fn("SessionPromptIntent.activityForMessage")(function* (input: { readonly sessionID: SessionID readonly messageID: MessageID @@ -1662,6 +1783,7 @@ export const finalizeActivityWithRevision = Effect.fn("SessionPromptIntent.final kind: "follow_up_required" as const, membershipOrdinal: latestMembership, } satisfies FinalizeResult + yield* pauseAtActivityCrashPoint("inside_revision_terminal_transaction") const runState = decision.state === "settled" ? "completed" @@ -1843,6 +1965,7 @@ export const finalizeActivityWithoutRevision = Effect.fn("SessionPromptIntent.fi .get() if (input.decision.source !== "cancel" && (membershipOrdinal > input.membershipOrdinal || pendingSteer)) return { kind: "follow_up_required" as const, membershipOrdinal } satisfies FinalizeResult + yield* pauseAtActivityCrashPoint("inside_revision_terminal_transaction") const runState = terminalRunState(input.decision.state) const terminalRun = yield* tx .update(SessionLegacyActivityRunTable) diff --git a/packages/deepagent-code/src/session/prompt.ts b/packages/deepagent-code/src/session/prompt.ts index a2b87a853..ba3a74deb 100644 --- a/packages/deepagent-code/src/session/prompt.ts +++ b/packages/deepagent-code/src/session/prompt.ts @@ -2391,6 +2391,7 @@ export const layer = Layer.effect( input.sessionID, lastAssistant(input.sessionID), Effect.gen(function* () { + yield* pauseAtActivityCrashPoint("after_coordinator_reserve") yield* reconcileInactiveActivity(input.sessionID) const admitted = yield* admit if (!admitted.materialized || !("run" in admitted.materialized)) @@ -2945,6 +2946,31 @@ export const layer = Layer.effect( : undefined let providerBoundary: SessionPromptIntent.ProviderInputBoundary | undefined let activityTerminalCommitted = false + const enterFinalizing = (run: SessionPromptIntent.RunIdentity) => + SessionPromptIntent.markRunFinalizing(run).pipe( + Effect.provideService(Database.Service, database), + Effect.andThen(state.markFinalizing(sessionID)), + ) + const resumeRunning = (run: SessionPromptIntent.RunIdentity) => + SessionPromptIntent.markRunRunning(run).pipe( + Effect.provideService(Database.Service, database), + Effect.andThen(state.markRunning(sessionID)), + ) + const finishFinalization = ( + run: SessionPromptIntent.RunIdentity, + result: SessionPromptIntent.FinalizeResult, + ) => + Effect.gen(function* () { + if (result.kind === "follow_up_required") { + yield* pauseAtActivityCrashPoint("while_finalizing_before_follow_up_drain") + yield* resumeRunning(run) + return result + } + activityTerminalCommitted = true + yield* pauseAtActivityCrashPoint("after_terminal_commit_before_publish") + yield* publishActivityProjection(result.invalidation) + return result + }) const terminalDecision = ( state: SessionPromptIntent.ActivityTerminalState, reasonCode: string, @@ -2967,27 +2993,19 @@ export const layer = Layer.effect( ) => { const decision = terminalDecision(state, reasonCode, source === "provider_final" ? source : "host_stop") if (!currentActivityRun || !providerBoundary || !decision) return Effect.succeed(undefined) - const finalized = assistantMessageID - ? SessionPromptIntent.finalizeActivityWithRevision({ - run: currentActivityRun, - assistantMessageID, - decision, - }) - : SessionPromptIntent.finalizeActivityWithoutRevision({ - run: currentActivityRun, - membershipOrdinal: providerBoundary.membershipOrdinal, - decision: { ...decision, source }, - }) - return finalized.pipe( - Effect.provideService(Database.Service, database), - Effect.tap((result) => - Effect.sync(() => { - activityTerminalCommitted = result.kind !== "follow_up_required" - }), - ), - Effect.tap((result) => - result.kind === "follow_up_required" ? Effect.void : publishActivityProjection(result.invalidation), + const run = currentActivityRun + return enterFinalizing(run).pipe( + Effect.andThen( + assistantMessageID + ? SessionPromptIntent.finalizeActivityWithRevision({ run, assistantMessageID, decision }) + : SessionPromptIntent.finalizeActivityWithoutRevision({ + run, + membershipOrdinal: providerBoundary.membershipOrdinal, + decision: { ...decision, source }, + }), ), + Effect.provideService(Database.Service, database), + Effect.flatMap((result) => finishFinalization(run, result)), ) } const publishActivityProgress = (progress: SessionPromptIntent.Progress) => @@ -4019,22 +4037,20 @@ export const layer = Layer.effect( yield* pauseAtActivityCrashPoint("after_provider_receipt_terminal") const finalized = decision && currentActivityRun && progressActivity - ? yield* SessionPromptIntent.finalizeActivityWithRevision({ - run: currentActivityRun, - assistantMessageID: handle.message.id, - decision, - }).pipe( - Effect.provideService(Database.Service, database), - Effect.tap((result) => - result.kind === "follow_up_required" - ? Effect.void - : publishActivityProjection(result.invalidation), + ? yield* enterFinalizing(currentActivityRun).pipe( + Effect.andThen( + SessionPromptIntent.finalizeActivityWithRevision({ + run: currentActivityRun, + assistantMessageID: handle.message.id, + decision, + }), ), + Effect.provideService(Database.Service, database), + Effect.flatMap((result) => finishFinalization(currentActivityRun!, result)), ) : undefined if (!finalized && activityProgressFinalizer.value) yield* activityProgressFinalizer.value() activityProgressFinalizer.value = undefined - yield* pauseAtActivityCrashPoint("after_progress_settled") turnSettled.value = true // Summary diffs mutate user-message metadata. Run them only after the Provider // receipt is terminal so cancellation cannot strand an admitted request. @@ -4130,7 +4146,10 @@ export const layer = Layer.effect( .run() .pipe(Effect.orDie, Effect.asVoid) }, - adapterPrepared: prepareAdapterReceipt, + adapterPrepared: (input) => + prepareAdapterReceipt(input).pipe( + Effect.tap(() => pauseAtActivityCrashPoint("after_provider_prepared")), + ), dispatched: () => Effect.gen(function* () { const transitioned = yield* transitionReceipt({ @@ -4636,6 +4655,7 @@ export const layer = Layer.effect( input.sessionID, lastAssistant(input.sessionID), Effect.gen(function* () { + yield* pauseAtActivityCrashPoint("after_coordinator_reserve") yield* reconcileInactiveActivity(input.sessionID) const deferredRun = yield* SessionPromptIntent.claimDeferredActivity({ sessionID: input.sessionID }).pipe( Effect.provideService(Database.Service, database), diff --git a/packages/deepagent-code/src/session/run-state.ts b/packages/deepagent-code/src/session/run-state.ts index bb7d23377..7bbd3e75d 100644 --- a/packages/deepagent-code/src/session/run-state.ts +++ b/packages/deepagent-code/src/session/run-state.ts @@ -30,6 +30,8 @@ export interface Interface { onInterrupt: Effect.Effect, work: Effect.Effect, ) => Effect.Effect + readonly markFinalizing: (sessionID: SessionID) => Effect.Effect + readonly markRunning: (sessionID: SessionID) => Effect.Effect readonly startShell: ( sessionID: SessionID, onInterrupt: Effect.Effect, @@ -140,7 +142,28 @@ export const layer = Layer.effect( .pipe(Effect.catchTag("RunnerBusy", () => Effect.fail(busyError(sessionID)))) }) - return Service.of({ assertNotBusy, isBusy, cancel, ensureRunning, startRunning, startShell }) + const markFinalizing = Effect.fn("SessionRunState.markFinalizing")(function* (sessionID: SessionID) { + const data = yield* InstanceState.get(state) + const existing = data.runners.get(sessionID) + if (existing) yield* existing.markFinalizing + }) + + const markRunning = Effect.fn("SessionRunState.markRunning")(function* (sessionID: SessionID) { + const data = yield* InstanceState.get(state) + const existing = data.runners.get(sessionID) + if (existing) yield* existing.markRunning + }) + + return Service.of({ + assertNotBusy, + isBusy, + cancel, + ensureRunning, + startRunning, + markFinalizing, + markRunning, + startShell, + }) }), ) diff --git a/packages/deepagent-code/test/effect/runner.test.ts b/packages/deepagent-code/test/effect/runner.test.ts index dfc504a7e..a5afe79f2 100644 --- a/packages/deepagent-code/test/effect/runner.test.ts +++ b/packages/deepagent-code/test/effect/runner.test.ts @@ -64,10 +64,7 @@ describe("Runner", () => { const runner = Runner.make(s) const ready = yield* Deferred.make() const fiber = yield* runner - .ensureRunning( - Effect.never.pipe(Effect.as("never")), - Deferred.succeed(ready, undefined).pipe(Effect.asVoid), - ) + .ensureRunning(Effect.never.pipe(Effect.as("never")), Deferred.succeed(ready, undefined).pipe(Effect.asVoid)) .pipe(Effect.forkChild) yield* Deferred.await(ready).pipe(Effect.timeout("250 millis")) @@ -78,6 +75,33 @@ describe("Runner", () => { }), ) + it.live( + "keeps one busy owner while a run moves through finalizing and back to running", + Effect.gen(function* () { + const s = yield* Scope.Scope + const runner = Runner.make(s) + const started = yield* Deferred.make() + const release = yield* Deferred.make() + const fiber = yield* runner + .startRunning( + Deferred.succeed(started, undefined).pipe(Effect.andThen(Deferred.await(release)), Effect.as("done")), + ) + .pipe(Effect.forkChild) + + yield* Deferred.await(started) + yield* runner.markFinalizing + expect(runner.state._tag).toBe("Finalizing") + expect(runner.busy).toBe(true) + yield* runner.markRunning + expect(runner.state._tag).toBe("Running") + expect(runner.busy).toBe(true) + + yield* Deferred.succeed(release, undefined) + expect(yield* Fiber.join(fiber)).toBe("done") + expect(runner.state._tag).toBe("Idle") + }), + ) + it.live( "concurrent callers share the same run", Effect.gen(function* () { diff --git a/packages/deepagent-code/test/session/prompt-intent.test.ts b/packages/deepagent-code/test/session/prompt-intent.test.ts index 982fdb276..3ec29d589 100644 --- a/packages/deepagent-code/test/session/prompt-intent.test.ts +++ b/packages/deepagent-code/test/session/prompt-intent.test.ts @@ -5,6 +5,8 @@ import { ProjectTable } from "@deepagent-code/core/project/sql" import { ModelV2 } from "@deepagent-code/core/model" import { ProviderV2 } from "@deepagent-code/core/provider" import { AbsolutePath } from "@deepagent-code/core/schema" +import { SessionMessage } from "@deepagent-code/core/session/message" +import { Prompt } from "@deepagent-code/core/session/prompt" import { MessageTable, PartTable, @@ -128,6 +130,36 @@ describe("SessionPromptIntent", () => { }), ) + it.effect("an exact retry takes over an unexpired claim owned by a dead process", () => + Effect.gen(function* () { + yield* setup + const first = yield* claim({ + intentID: "intent_process_restart", + messageID: MessageID.make("msg_process_restart"), + }) + expect(first.kind).toBe("claimed") + const { db } = yield* Database.Service + yield* db + .update(SessionIntentTable) + .set({ + owner_token: "2147483647:00000000-0000-4000-8000-000000000001:00000000-0000-4000-8000-000000000002", + lease_expires_at: Date.now() + 30_000, + }) + .where(eq(SessionIntentTable.intent_id, "intent_process_restart")) + .run() + .pipe(Effect.orDie) + + const retry = yield* claim({ + intentID: "intent_process_restart", + messageID: MessageID.make("msg_process_restart_retry"), + }) + expect(retry.kind).toBe("claimed") + if (retry.kind !== "claimed") return + expect(retry.receipt.messageID).toBe(MessageID.make("msg_process_restart")) + expect(retry.receipt.ownerToken).not.toBe(first.kind === "claimed" ? first.receipt.ownerToken : undefined) + }), + ) + it.effect("variant or payload changes conflict instead of creating a second admission", () => Effect.gen(function* () { yield* setup @@ -564,6 +596,173 @@ describe("SessionPromptIntent", () => { }), ) + it.effect("a follow-up arriving after the provider boundary resumes the same finalizing run", () => + Effect.gen(function* () { + yield* setup + const trigger = yield* claim({ + intentID: "intent_finalizing_followup", + messageID: MessageID.make("msg_finalizing_followup_user"), + }) + if (trigger.kind !== "claimed") return + const materialized = yield* SessionPromptIntent.materializeTurn({ + receipt: trigger.receipt, + message: message(trigger.receipt.messageID), + }) + if (!("run" in materialized) || !materialized.run) return + const boundary = yield* SessionPromptIntent.freezeProviderInputBoundary(materialized.run) + if (boundary.kind !== "ready") return + + const { db } = yield* Database.Service + const assistantID = MessageID.make("msg_finalizing_followup_assistant") + yield* db + .insert(MessageTable) + .values({ + id: assistantID, + session_id: sessionID, + time_created: 2, + data: { + role: "assistant", + parentID: trigger.receipt.messageID, + mode: "build", + agent: "build", + path: { cwd: "/project", root: "/project" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + modelID: ModelV2.ID.make("test"), + providerID: ProviderV2.ID.make("test"), + time: { created: 2, completed: 3 }, + finish: "stop", + } as typeof MessageTable.$inferInsert.data, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: "receipt-finalizing-followup", + request_ordinal: 1, + session_id: sessionID, + user_message_id: trigger.receipt.messageID, + assistant_message_id: assistantID, + provider_id: "test", + model_id: "test", + registry_tool_ids: [], + permission_filtered_tool_ids: [], + final_offered_tool_ids: [], + call_ids: [], + provider_state: "settled", + terminal_at: 3, + response_fingerprint: "response-finalizing-followup", + request_state: "dispatched", + created_at: 2, + }) + .run() + .pipe(Effect.orDie) + yield* SessionPromptIntent.beginProgress({ + activityID: materialized.run.activityID, + assistantMessageID: assistantID, + providerReceiptID: "receipt-finalizing-followup", + membershipOrdinal: boundary.boundary.membershipOrdinal, + }) + + const steerID = SessionMessage.ID.make("msg_finalizing_followup_steer") + const admissionID = "admission-finalizing-followup-steer" + const steerClaim = yield* claim({ + intentID: "intent_finalizing_followup_steer", + messageID: MessageID.make(steerID), + source: "followup", + }) + if (steerClaim.kind !== "claimed") return + yield* db + .insert(SessionSteerTable) + .values({ + id: steerID, + session_id: sessionID, + correlation_id: steerID, + prompt: Prompt.fromUserMessage({ text: "follow up" }), + delivery: "steer", + mutation_epoch: materialized.run.mutationEpoch, + time_created: 3, + }) + .run() + .pipe(Effect.orDie) + yield* SessionPromptIntent.complete({ + intentID: steerClaim.receipt.intentID, + ownerToken: steerClaim.receipt.ownerToken, + messageID: MessageID.make(steerID), + delivery: "steer", + }) + yield* db + .insert(SessionActivityAdmissionTable) + .values({ + admission_id: admissionID, + session_id: sessionID, + source_kind: "legacy_intent", + legacy_intent_id: steerClaim.receipt.intentID, + admitted_message_id: steerID, + delivery: "steer", + payload_fingerprint_kind: "payload_hash", + payload_fingerprint: "payload-a", + execution_mode: "run_now", + created_at: 3, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionLegacyActivityAdmissionTable) + .values({ + activity_id: materialized.run.activityID, + admission_id: admissionID, + ordinal: boundary.boundary.membershipOrdinal + 1, + role: "steer", + attached_at: 3, + }) + .run() + .pipe(Effect.orDie) + + yield* SessionPromptIntent.markRunFinalizing(materialized.run) + expect( + yield* SessionPromptIntent.finalizeActivityWithRevision({ + run: materialized.run, + assistantMessageID: assistantID, + decision: { + state: "settled", + reasonCode: "provider_final", + source: "provider_final", + operationID: `${materialized.run.runID}:terminal`, + ownerToken: materialized.run.ownerToken, + }, + }), + ).toEqual({ kind: "follow_up_required", membershipOrdinal: boundary.boundary.membershipOrdinal + 1 }) + expect( + yield* db + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, materialized.run.runID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ state: "finalizing", activity_id: materialized.run.activityID }) + expect( + yield* db + .select() + .from(SessionLegacyActivityTerminalTable) + .where(eq(SessionLegacyActivityTerminalTable.activity_id, materialized.run.activityID)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + + yield* SessionPromptIntent.markRunRunning(materialized.run) + expect( + yield* db + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.run_id, materialized.run.runID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ state: "running", activity_id: materialized.run.activityID }) + }), + ) + it.effect("terminal provider receipts settle provisional progress deterministically after restart", () => Effect.gen(function* () { yield* setup diff --git a/packages/desktop/scripts/live-llm/activity-progress-restart.ts b/packages/desktop/scripts/live-llm/activity-progress-restart.ts index e656e5ecb..3364f844a 100644 --- a/packages/desktop/scripts/live-llm/activity-progress-restart.ts +++ b/packages/desktop/scripts/live-llm/activity-progress-restart.ts @@ -1,6 +1,6 @@ import { strict as assert } from "node:assert" import { randomUUID } from "node:crypto" -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { mkdtemp, readFile, rm } from "node:fs/promises" import os from "node:os" import path from "node:path" import { DatabaseSync } from "node:sqlite" @@ -13,10 +13,12 @@ import { loadLiveConfig, messages, preflight, - startPrompt, + request, waitFor, + waitForPrompt, writeArtifact, type Runtime, + type Status, } from "./runtime.ts" const suite = "activity-progress-restart" @@ -28,38 +30,192 @@ const preflightResult = await preflight(config) const startedAt = Date.now() const results: RestartResult[] = [] -const cases = [ +const cases: readonly RestartCase[] = [ { - name: "streaming-recovery-required", + name: "coordinator-reservation-exact-retry", + point: "after_coordinator_reserve", + before: { + dispatches: 0, + activity: undefined, + run: undefined, + receipt: undefined, + progress: undefined, + terminals: 0, + }, + after: { + dispatches: 0, + activity: undefined, + run: undefined, + receipt: undefined, + progress: undefined, + terminals: 0, + }, + promptOutcome: "disconnected", + exactRetry: true, + nextAdmission: "accepted", + }, + { + name: "bound-run-pre-dispatch-recovery", + point: "after_admit_and_bind", + before: { + dispatches: 0, + activity: "active", + run: "running", + receipt: undefined, + progress: undefined, + terminals: 0, + }, + after: { dispatches: 0, activity: "failed", run: "failed", receipt: undefined, progress: undefined, terminals: 1 }, + promptOutcome: "disconnected", + nextAdmission: "accepted", + }, + { + name: "prepared-provider-never-dispatched", + point: "after_provider_prepared", + before: { + dispatches: 0, + activity: "active", + run: "running", + receipt: "prepared", + progress: "provisional", + terminals: 0, + }, + after: { + dispatches: 0, + activity: "failed", + run: "failed", + receipt: "failed", + progress: "recovery_required", + terminals: 1, + }, + promptOutcome: "acknowledged", + nextAdmission: "accepted", + }, + { + name: "streaming-outcome-indeterminate", point: "after_provider_streaming", - prompt: (marker: string) => `Reply with exactly ${marker}. Do not call any tool.`, - expectedBefore: { activity: "active", progress: "provisional", receipt: "streaming" }, - expectedAfter: { activity: "recovery_required", progress: "recovery_required" }, + before: { + dispatches: 1, + activity: "active", + run: "running", + receipt: "streaming", + progress: "provisional", + terminals: 0, + }, + after: { + dispatches: 1, + activity: "recovery_required", + run: "recovery_required", + receipt: "indeterminate_after_crash", + progress: "recovery_required", + terminals: 1, + }, + promptOutcome: "acknowledged", + nextAdmission: "rejected", }, { - name: "terminal-receipt-final-reconciliation", + name: "terminal-receipt-revision-reconciliation", point: "after_provider_receipt_terminal", - prompt: (marker: string) => `Reply with exactly ${marker}. Do not call any tool.`, - expectedBefore: { activity: "active", progress: "provisional", receipt: "settled" }, - expectedAfter: { activity: "settled", progress: "final" }, + before: { + dispatches: 1, + activity: "active", + run: "running", + receipt: "settled", + progress: "provisional", + terminals: 0, + }, + after: { + dispatches: 1, + activity: "recovery_required", + run: "recovery_required", + receipt: "settled", + progress: "final", + terminals: 1, + }, + promptOutcome: "acknowledged", + nextAdmission: "accepted", + }, + { + name: "revision-terminal-transaction-rollback", + point: "inside_revision_terminal_transaction", + before: { + dispatches: 1, + activity: "active", + run: "finalizing", + receipt: "settled", + progress: "provisional", + terminals: 0, + }, + after: { + dispatches: 1, + activity: "recovery_required", + run: "recovery_required", + receipt: "settled", + progress: "final", + terminals: 1, + }, + promptOutcome: "acknowledged", + nextAdmission: "accepted", + }, + { + name: "terminal-commit-survives-lost-publish", + point: "after_terminal_commit_before_publish", + before: { + dispatches: 1, + activity: "settled", + run: "completed", + receipt: "settled", + progress: "final", + terminals: 1, + }, + after: { + dispatches: 1, + activity: "settled", + run: "completed", + receipt: "settled", + progress: "final", + terminals: 1, + }, + promptOutcome: "acknowledged", + nextAdmission: "accepted", }, { - name: "settled-tool-progress-recovery-required", - point: "after_progress_settled", - prompt: (_marker: string) => - "Read restart-fact.txt exactly once. After the tool result, return the exact file content and do not call any other tool.", - file: "restart-fact.txt", - expectedBefore: { activity: "active", progress: "progress", receipt: "settled" }, - expectedAfter: { activity: "recovery_required", progress: "progress" }, + name: "finalizing-follow-up-fails-closed", + point: "while_finalizing_before_follow_up_drain", + before: { + dispatches: 1, + activity: "active", + run: "finalizing", + receipt: "settled", + progress: "progress", + terminals: 0, + }, + after: { + dispatches: 1, + activity: "recovery_required", + run: "recovery_required", + receipt: "settled", + progress: "progress", + terminals: 1, + }, + promptOutcome: "acknowledged", + followUp: true, + nextAdmission: "accepted", }, -] as const +] try { for (const testCase of cases) { console.log(`${suite}: starting ${testCase.name}`) const root = await mkdtemp(path.join(os.tmpdir(), `deepagent-code-${suite}-${testCase.name}-`)) const markerFile = path.join(root, "crash-marker.json") - const marker = `restart-${randomUUID()}` + const responseMarker = `restart-${randomUUID()}` + const requestID = randomUUID().replaceAll("-", "") + const body = promptBody( + `msg_restart_${requestID}`, + `intent-restart-${requestID}`, + `Reply with exactly ${responseMarker}. Do not call any tool.`, + ) let initial: Runtime | undefined let restarted: Runtime | undefined try { @@ -71,66 +227,105 @@ try { DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER: markerFile, }, }) - if ("file" in testCase) await writeFile(path.join(initial.workspace, testCase.file), `${marker}\n`) const session = await createSession(initial, `DeepSeek restart ${testCase.name}`, "live-ui") - const promptRequest = startPrompt(initial, session.id, testCase.prompt(marker), "live-ui").then( + const beforeMessages = new Set((await messages(initial, session.id)).map((message) => message.info.id)) + const promptRequest = sendPrompt(initial, session.id, body).then( () => ({ state: "acknowledged" as const }), (error) => ({ state: "disconnected" as const, error: error instanceof Error ? error.message : String(error) }), ) + let followUpOutcome: "acknowledged" | undefined + if (testCase.followUp) { + await waitFor( + async () => { + const snapshot = inspect(path.join(root, "deepagent.sqlite"), session.id) + if (["dispatching", "streaming"].includes(snapshot.receipts.at(-1)?.provider_state ?? "")) return true + }, + `${testCase.name} provider dispatch`, + config.timeoutMs, + ) + const followUpID = randomUUID().replaceAll("-", "") + await sendPrompt( + initial, + session.id, + promptBody( + `msg_restart_followup_${followUpID}`, + `intent-restart-followup-${followUpID}`, + `After the current reply, reply with exactly follow-up-${followUpID}.`, + ), + ) + followUpOutcome = "acknowledged" + } + const crash = await waitFor( async () => { const value = await readFile(markerFile, "utf8").catch(() => undefined) - if (value) return JSON.parse(value) as { point: string; pid: number; reachedAt: number } + if (value) return JSON.parse(value) as CrashMarker }, `${testCase.name} crash point`, config.timeoutMs, ) - console.log(`${suite}: reached ${testCase.point} in sidecar ${crash.pid}`) assert.equal(crash.point, testCase.point) + const statuses = await request>(initial, "/session/status") + assert.equal(statuses[session.id]?.type, "busy") const before = inspect(path.join(root, "deepagent.sqlite"), session.id) - console.log(`${suite}: captured durable pre-kill state for ${session.id}`) - assert.equal(before.activity?.state, testCase.expectedBefore.activity) - assert.equal(before.progress.at(-1)?.state, testCase.expectedBefore.progress) - assert.equal(before.receipts.at(-1)?.provider_state, testCase.expectedBefore.receipt) + assertSnapshot(before, testCase.before) const initialAppPID = initial.app.process().pid - console.log(`${suite}: sending SIGKILL to Electron ${initialAppPID}`) await hardKill(initial) initial = undefined const promptOutcome = await Promise.race([ promptRequest, new Promise<{ state: "timed_out" }>((resolve) => setTimeout(() => resolve({ state: "timed_out" }), 10_000)), ]) - assert.notEqual(promptOutcome.state, "timed_out") + assert.equal(promptOutcome.state, testCase.promptOutcome) - console.log(`${suite}: restarting from ${root}`) restarted = await launch(suite, config, { root, cleanupRoot: false }) const restartedAppPID = restarted.app.process().pid assert.notEqual(restartedAppPID, initialAppPID) const after = await waitFor( async () => { - const value = inspect(path.join(root, "deepagent.sqlite"), session.id) - if ( - value.activity?.state === testCase.expectedAfter.activity && - value.progress.at(-1)?.state === testCase.expectedAfter.progress - ) - return value + const snapshot = inspect(path.join(root, "deepagent.sqlite"), session.id) + if (matchesSnapshot(snapshot, testCase.after)) return snapshot }, `${testCase.name} restart reconciliation`, 30_000, ) - console.log(`${suite}: reconciled ${testCase.name} in Electron ${restartedAppPID}`) + assertSnapshot(after, testCase.after) assert.equal(after.receipts.length, before.receipts.length) - assert.equal(after.messageCount, before.messageCount) - assert.equal(after.toolPartCount, before.toolPartCount) - assert.equal(after.userMessageCount, 1) - if (testCase.point === "after_provider_receipt_terminal") { + assert.equal(after.counts.message_count, before.counts.message_count) + assert.equal(after.counts.part_count, before.counts.part_count) + assert.equal(after.counts.tool_part_count, before.counts.tool_part_count) + assert.equal(after.dispatchCount, before.dispatchCount) + const restartedStatuses = await request>(restarted, "/session/status") + assert.notEqual(restartedStatuses[session.id]?.type, "busy") + + let retry: Snapshot | undefined + if (testCase.exactRetry) { + await sendPrompt(restarted, session.id, body) + await waitForPrompt(restarted, session.id, beforeMessages, `${testCase.name} exact retry`) + retry = inspect(path.join(root, "deepagent.sqlite"), session.id) + assert.equal(retry.dispatchCount, 1) + assert.equal(retry.counts.user_message_count, 1) assert.equal( (await messages(restarted, session.id)).some((message) => - message.parts.some((part) => part.type === "text" && part.text.includes(marker)), + message.parts.some((part) => part.type === "text" && part.text.includes(responseMarker)), ), true, ) } + + const nextID = randomUUID().replaceAll("-", "") + const nextAdmission = await sendPrompt(restarted, session.id, { + ...promptBody(`msg_restart_next_${nextID}`, `intent-restart-next-${nextID}`, "deferred restart probe"), + noReply: true, + }).then( + () => "accepted" as const, + () => "rejected" as const, + ) + assert.equal(nextAdmission, testCase.nextAdmission) + const afterAdmission = inspect(path.join(root, "deepagent.sqlite"), session.id) + assert.equal(afterAdmission.dispatchCount, retry?.dispatchCount ?? after.dispatchCount) + assert.equal(afterAdmission.activities.length, retry?.activities.length ?? after.activities.length) + results.push({ name: testCase.name, point: testCase.point, @@ -139,9 +334,16 @@ try { initialAppPID, restartedAppPID, promptOutcome, + followUpOutcome, + runnerBefore: statuses[session.id]?.type, + runnerAfter: restartedStatuses[session.id]?.type ?? "idle", + nextAdmission, before, after, + retry, + afterAdmission, }) + console.log(`${suite}: passed ${testCase.name}`) } finally { if (initial) await close(initial).catch(() => undefined) if (restarted) await close(restarted).catch(() => undefined) @@ -171,71 +373,181 @@ try { await closeAll() } +type CrashPoint = + | "after_coordinator_reserve" + | "after_admit_and_bind" + | "after_provider_prepared" + | "after_provider_streaming" + | "after_provider_receipt_terminal" + | "inside_revision_terminal_transaction" + | "after_terminal_commit_before_publish" + | "while_finalizing_before_follow_up_drain" + +type SnapshotExpectation = { + dispatches: number + activity: string | undefined + run: string | undefined + receipt: string | undefined + progress: string | undefined + terminals: number +} + +type RestartCase = { + name: string + point: CrashPoint + before: SnapshotExpectation + after: SnapshotExpectation + promptOutcome: "acknowledged" | "disconnected" + exactRetry?: true + followUp?: true + nextAdmission: "accepted" | "rejected" +} + +type CrashMarker = { point: string; pid: number; reachedAt: number } type Snapshot = ReturnType type RestartResult = { name: string - point: string + point: CrashPoint sessionID: string - crash: { point: string; pid: number; reachedAt: number } + crash: CrashMarker initialAppPID?: number restartedAppPID?: number promptOutcome: { state: "acknowledged" } | { state: "disconnected"; error: string } + followUpOutcome?: "acknowledged" + runnerBefore: string | undefined + runnerAfter: string + nextAdmission: "accepted" | "rejected" before: Snapshot after: Snapshot + retry?: Snapshot + afterAdmission: Snapshot +} + +function promptBody(messageID: string, intentID: string, text: string) { + return { + messageID, + intentID, + model: { providerID: "live-deepseek", modelID: config.modelID }, + agent: "live-ui", + parts: [{ type: "text", text }], + } +} + +function sendPrompt(runtime: Runtime, sessionID: string, body: ReturnType & { noReply?: boolean }) { + return request(runtime, `/session/${sessionID}/prompt_async`, { + method: "POST", + body: JSON.stringify(body), + }) +} + +function matchesSnapshot(snapshot: Snapshot, expected: SnapshotExpectation) { + return ( + snapshot.dispatchCount === expected.dispatches && + snapshot.activities.at(-1)?.state === expected.activity && + snapshot.runs.at(-1)?.state === expected.run && + snapshot.receipts.at(-1)?.provider_state === expected.receipt && + snapshot.progress.at(-1)?.state === expected.progress && + snapshot.terminals.length === expected.terminals + ) +} + +function assertSnapshot(snapshot: Snapshot, expected: SnapshotExpectation) { + assert.equal(snapshot.dispatchCount, expected.dispatches) + assert.equal(snapshot.activities.at(-1)?.state, expected.activity) + assert.equal(snapshot.runs.at(-1)?.state, expected.run) + assert.equal(snapshot.receipts.at(-1)?.provider_state, expected.receipt) + assert.equal(snapshot.progress.at(-1)?.state, expected.progress) + assert.equal(snapshot.terminals.length, expected.terminals) } function inspect(databasePath: string, sessionID: string) { const database = new DatabaseSync(databasePath, { readOnly: true }) try { database.exec("PRAGMA busy_timeout = 5000") - const activity = database + const activities = database + .prepare( + "SELECT activity_id, owner_token, state, terminal_reason FROM session_legacy_activity WHERE session_id = ? ORDER BY ordinal", + ) + .all(sessionID) as Array<{ + activity_id: string + owner_token: string + state: string + terminal_reason: string | null + }> + const runs = database .prepare( - "SELECT activity_id, owner_token, state, terminal_reason FROM session_legacy_activity WHERE session_id = ? ORDER BY ordinal DESC LIMIT 1", + "SELECT run_id, activity_id, generation, owner_token, state, terminal_reason FROM session_legacy_activity_run WHERE session_id = ? ORDER BY generation", ) - .get(sessionID) as - | { activity_id: string; owner_token: string; state: string; terminal_reason: string | null } - | undefined + .all(sessionID) as Array<{ + run_id: string + activity_id: string + generation: number + owner_token: string + state: string + terminal_reason: string | null + }> const progress = database .prepare( - "SELECT revision, assistant_message_id, provider_receipt_id, state, finish_observed FROM session_activity_progress WHERE activity_id = ? ORDER BY revision", + "SELECT activity_id, revision, assistant_message_id, provider_receipt_id, input_membership_ordinal, state, finish_observed FROM session_activity_progress WHERE activity_id IN (SELECT activity_id FROM session_legacy_activity WHERE session_id = ?) ORDER BY activity_id, revision", ) - .all(activity?.activity_id ?? "") as Array<{ + .all(sessionID) as Array<{ + activity_id: string revision: number assistant_message_id: string provider_receipt_id: string + input_membership_ordinal: number state: string finish_observed: string | null }> const receipts = database .prepare( - "SELECT receipt_id, assistant_message_id, provider_state, request_state, call_ids FROM session_tool_request_receipt WHERE session_id = ? ORDER BY request_ordinal", + "SELECT receipt_id, assistant_message_id, provider_state, request_state, request_error_code, call_ids FROM session_tool_request_receipt WHERE session_id = ? ORDER BY request_ordinal", ) .all(sessionID) as Array<{ receipt_id: string assistant_message_id: string provider_state: string request_state: string + request_error_code: string | null call_ids: string }> + const terminals = database + .prepare( + "SELECT activity_id, run_id, state, reason_code, source, assistant_message_id, progress_revision, membership_ordinal FROM session_legacy_activity_terminal WHERE session_id = ? ORDER BY created_at", + ) + .all(sessionID) + const intents = database + .prepare( + "SELECT intent_id, state, delivery, execution_mode, execution_state, admitted_message_id FROM session_intent WHERE session_id = ? ORDER BY time_created", + ) + .all(sessionID) + const steers = database + .prepare("SELECT id, delivery, consumed_seq, superseded_at FROM session_steer WHERE session_id = ? ORDER BY seq") + .all(sessionID) const counts = database .prepare( `SELECT (SELECT COUNT(*) FROM message WHERE session_id = ?) AS message_count, (SELECT COUNT(*) FROM message WHERE session_id = ? AND json_extract(data, '$.role') = 'user') AS user_message_count, + (SELECT COUNT(*) FROM part WHERE session_id = ?) AS part_count, (SELECT COUNT(*) FROM part WHERE session_id = ? AND json_extract(data, '$.type') = 'tool') AS tool_part_count`, ) - .get(sessionID, sessionID, sessionID) as { + .get(sessionID, sessionID, sessionID, sessionID) as { message_count: number user_message_count: number + part_count: number tool_part_count: number } return { - activity, + activities, + runs, progress, receipts: receipts.map((receipt) => ({ ...receipt, call_ids: JSON.parse(receipt.call_ids) as string[] })), - messageCount: counts.message_count, - userMessageCount: counts.user_message_count, - toolPartCount: counts.tool_part_count, + terminals, + intents, + steers, + counts, + dispatchCount: receipts.filter((receipt) => receipt.request_state === "dispatched").length, } } finally { database.close() From 537350ba0966ad2fd19676c2e8918e47d190c340 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 10:46:05 +0800 Subject: [PATCH 19/44] fix(session): close provider recovery and sync gates --- packages/app/src/i18n/en.ts | 2 + packages/app/src/i18n/zh.ts | 1 + .../session-provider-recovery-dock.test.ts | 54 +- .../session-provider-recovery-dock.tsx | 60 +- .../core/bug407-artifact-backfill-one.gate.ts | 32 + .../core/bug407-artifact-backfill.gate.ts | 32 + packages/core/bug407-migration-only.gate.ts | 13 + .../src/context-federation/session-sql.ts | 13 +- packages/core/src/database/migration.gen.ts | 3 + packages/core/src/database/migration.ts | 4 + ...20260813100000_event_snapshot_authority.ts | 165 ++++ ...0000_provider_recovery_authority_bridge.ts | 186 +++++ ...00_legacy_provider_receipt_supersession.ts | 39 + packages/core/src/event.ts | 709 +++++++++++++++++- packages/core/src/event/sql.ts | 103 ++- packages/core/src/npm.ts | 20 +- packages/core/src/session/input.ts | 1 + packages/core/src/session/projector.ts | 82 ++ packages/core/src/v1/session.ts | 8 + packages/core/test/database-migration.test.ts | 46 ++ packages/core/test/event.test.ts | 129 +++- ...der-receipt-supersession-migration.test.ts | 92 +++ packages/core/test/npm.test.ts | 41 +- packages/core/test/session-create.test.ts | 276 ++++++- packages/core/test/session-projector.test.ts | 115 ++- .../test/session-runner-tool-events.test.ts | 5 + packages/deepagent-code/src/config/config.ts | 2 +- .../src/context-federation/diagnostics.ts | 197 +++-- .../src/control-plane/workspace.ts | 113 ++- packages/deepagent-code/src/effect/runner.ts | 11 +- .../routes/instance/httpapi/groups/sync.ts | 56 +- .../routes/instance/httpapi/handlers/sync.ts | 288 ++++++- .../src/server/shared/workspace-routing.ts | 2 + .../src/session/legacy-provider-resolution.ts | 384 ++++++++-- .../deepagent-code/src/session/message-v2.ts | 31 +- .../deepagent-code/src/session/session.ts | 35 +- .../deepagent-code/test/config/config.test.ts | 35 +- .../context-federation/diagnostics.test.ts | 362 +++++---- .../test/control-plane/workspace.test.ts | 319 +++++++- .../deepagent-code/test/effect/runner.test.ts | 40 + .../server/httpapi-provider-recovery.test.ts | 165 +++- .../test/server/httpapi-sync.test.ts | 457 +++++++++-- .../test/server/httpapi-workspace.test.ts | 86 +++ .../test/server/workspace-routing.test.ts | 8 + .../legacy-provider-resolution.test.ts | 296 +++++++- .../test/session/prompt-intent.test.ts | 4 +- .../test/session/prompt.test.ts | 106 +++ .../test/session/session.test.ts | 63 ++ packages/sdk/js/src/gen/sdk.gen.ts | 55 +- packages/sdk/js/src/gen/types.gen.ts | 159 ++-- packages/sdk/js/src/v2/gen/sdk.gen.ts | 55 +- packages/sdk/js/src/v2/gen/types.gen.ts | 159 ++-- 52 files changed, 4966 insertions(+), 753 deletions(-) create mode 100644 packages/core/bug407-artifact-backfill-one.gate.ts create mode 100644 packages/core/bug407-artifact-backfill.gate.ts create mode 100644 packages/core/bug407-migration-only.gate.ts create mode 100644 packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts create mode 100644 packages/core/src/database/migration/20260813110000_provider_recovery_authority_bridge.ts create mode 100644 packages/core/src/database/migration/20260813120000_legacy_provider_receipt_supersession.ts create mode 100644 packages/core/test/legacy-provider-receipt-supersession-migration.test.ts diff --git a/packages/app/src/i18n/en.ts b/packages/app/src/i18n/en.ts index 80d048693..1882e11de 100644 --- a/packages/app/src/i18n/en.ts +++ b/packages/app/src/i18n/en.ts @@ -1648,4 +1648,6 @@ export const dict = { "This shared workspace requires coordinated recovery and cannot be resolved from this app instance.", "session.providerRecovery.baselineUnsupported": "The source World State baseline is missing or invalid. Sending remains locked until maintenance recovery verifies it.", + "session.providerRecovery.maintenanceUnsupported": + "This recovery record is incomplete or belongs to another recovery authority. Sending remains locked until maintenance repair completes.", } diff --git a/packages/app/src/i18n/zh.ts b/packages/app/src/i18n/zh.ts index ceaf5a905..7baf438ae 100644 --- a/packages/app/src/i18n/zh.ts +++ b/packages/app/src/i18n/zh.ts @@ -1585,4 +1585,5 @@ export const dict = { "session.providerRecovery.unsupported": "该 continuation 需要维护恢复，当前不能在这里放弃。", "session.providerRecovery.workspaceUnsupported": "共享工作区需要协调恢复，当前应用实例不能直接处理。", "session.providerRecovery.baselineUnsupported": "源 World State baseline 缺失或无效；维护恢复验证完成前，发送将保持锁定。", + "session.providerRecovery.maintenanceUnsupported": "该恢复记录不完整或属于其他恢复权威；维护修复完成前，发送将保持锁定。", } satisfies Partial> diff --git a/packages/app/src/pages/session/composer/session-provider-recovery-dock.test.ts b/packages/app/src/pages/session/composer/session-provider-recovery-dock.test.ts index 0c839cbda..03717348a 100644 --- a/packages/app/src/pages/session/composer/session-provider-recovery-dock.test.ts +++ b/packages/app/src/pages/session/composer/session-provider-recovery-dock.test.ts @@ -1,14 +1,15 @@ import { describe, expect, test } from "bun:test" -import type { ProviderRecovery } from "./session-provider-recovery-dock" +import type { ResolvableProviderRecovery } from "./session-provider-recovery-dock" import { providerRecoveryCommandID, providerRecoveryFingerprint, providerRecoveryPending, providerRecoveryResolveInput, + isResolvableProviderRecovery, resolveProviderRecovery, } from "./session-provider-recovery-dock" -const recovery = (overrides: Partial = {}): ProviderRecovery => ({ +const recovery = (overrides: Partial = {}): ResolvableProviderRecovery => ({ receiptID: "receipt-1", sessionID: "session-1", assistantMessageID: "message-1", @@ -24,6 +25,8 @@ const recovery = (overrides: Partial = {}): ProviderRecovery = workspaceRecoverySupported: true, sourceWorldStateBaselineStatus: "available", worldStateBaselineHash: "wsb1-baseline", + resolutionSupported: true, + unsupportedReasons: [], ...overrides, }) @@ -69,6 +72,25 @@ describe("provider recovery command", () => { }) describe("provider recovery admission", () => { + test("allows command construction only for a complete resolvable descriptor", () => { + expect(isResolvableProviderRecovery(recovery())).toBe(true) + expect( + isResolvableProviderRecovery({ + receiptID: "orphan", + sessionID: "session-1", + providerID: "provider", + modelID: "model", + providerState: "indeterminate_after_crash", + sessionMutationEpoch: 7, + resolutionSupported: false, + unsupportedReasons: ["legacy_receipt_authority_incomplete"], + continuationRecoverySupported: true, + workspaceRecoverySupported: true, + sourceWorldStateBaselineStatus: "missing", + }), + ).toBe(false) + }) + test("fails closed while loading or after a list failure", () => { expect(providerRecoveryPending({ loading: true, error: undefined, recoveries: undefined })).toBe(true) expect(providerRecoveryPending({ loading: false, error: new Error("offline"), recoveries: undefined })).toBe(true) @@ -84,6 +106,34 @@ describe("provider recovery admission", () => { }) describe("provider recovery settlement", () => { + test("never dispatches a maintenance-only recovery descriptor", async () => { + let dispatched = false + const result = await resolveProviderRecovery({ + sessionID: "session-1", + recovery: { + receiptID: "orphan", + sessionID: "session-1", + providerID: "provider", + modelID: "model", + providerState: "indeterminate_after_crash", + sessionMutationEpoch: 7, + resolutionSupported: false, + unsupportedReasons: ["legacy_receipt_authority_incomplete"], + continuationRecoverySupported: true, + workspaceRecoverySupported: true, + sourceWorldStateBaselineStatus: "missing", + }, + resolve: async () => { + dispatched = true + return {} + }, + refresh: async () => "retained", + }) + + expect(result).toEqual({ status: "unsupported" }) + expect(dispatched).toBe(false) + }) + test("uses the exact idempotent request and refreshes after success", async () => { const requests: unknown[] = [] let refreshes = 0 diff --git a/packages/app/src/pages/session/composer/session-provider-recovery-dock.tsx b/packages/app/src/pages/session/composer/session-provider-recovery-dock.tsx index 6adc93721..ff4fcfc61 100644 --- a/packages/app/src/pages/session/composer/session-provider-recovery-dock.tsx +++ b/packages/app/src/pages/session/composer/session-provider-recovery-dock.tsx @@ -13,20 +13,40 @@ import { formatServerError } from "@/utils/server-errors" import { showToast } from "@/utils/toast" export type ProviderRecovery = SessionProviderResolutionListResponse[number] +export type ResolvableProviderRecovery = Extract -export function providerRecoveryFingerprint(item: ProviderRecovery) { - return JSON.stringify([ - item.receiptID, - item.providerState, - item.promptEpoch, - item.sessionMutationEpoch, - item.requestHash, - item.historyHash, - item.worldStateBaselineHash, - ]) +export function isResolvableProviderRecovery(item: ProviderRecovery): item is ResolvableProviderRecovery { + return ( + item.resolutionSupported === true && + item.unsupportedReasons.length === 0 && + item.sourceWorldStateBaselineStatus === "available" && + typeof item.assistantMessageID === "string" && + typeof item.promptEpoch === "number" && + typeof item.promptWindowID === "string" && + typeof item.historyHash === "string" && + typeof item.requestHash === "string" && + typeof item.worldStateBaselineHash === "string" + ) +} + +export function providerRecoveryFingerprint(item: ResolvableProviderRecovery) { + return JSON.stringify({ + version: 1, + sessionID: item.sessionID, + receiptID: item.receiptID, + decision: "abandoned", + expected: { + providerState: item.providerState, + promptEpoch: item.promptEpoch, + sessionMutationEpoch: item.sessionMutationEpoch, + requestHash: item.requestHash, + historyHash: item.historyHash, + worldStateBaselineHash: item.worldStateBaselineHash, + }, + }) } -export async function providerRecoveryCommandID(item: ProviderRecovery) { +export async function providerRecoveryCommandID(item: ResolvableProviderRecovery) { return `provider-recovery-${await hash(providerRecoveryFingerprint(item))}` } @@ -39,7 +59,7 @@ export function providerRecoveryPending(input: { return input.loading || input.error !== undefined || input.settlementFailed === true || (input.recoveries?.length ?? 0) > 0 } -export async function providerRecoveryResolveInput(item: ProviderRecovery) { +export async function providerRecoveryResolveInput(item: ResolvableProviderRecovery) { return { commandID: await providerRecoveryCommandID(item), receiptID: item.receiptID, @@ -50,7 +70,7 @@ export async function providerRecoveryResolveInput(item: ProviderRecovery) { sessionMutationEpoch: item.sessionMutationEpoch, requestHash: item.requestHash, historyHash: item.historyHash, - worldStateBaselineHash: item.worldStateBaselineHash!, + worldStateBaselineHash: item.worldStateBaselineHash, }, } } @@ -64,6 +84,7 @@ export async function resolveProviderRecovery(input: { }> refresh: () => Promise<"cleared" | "retained" | "failed"> }) { + if (!isResolvableProviderRecovery(input.recovery)) return { status: "unsupported" as const } const request = await providerRecoveryResolveInput(input.recovery) const outcome = await input .resolve({ sessionID: input.sessionID, ...request }) @@ -144,7 +165,7 @@ export function SessionProviderRecoveryDock(props: { } const abandon = async (item: ProviderRecovery) => { - if (store.resolving) return + if (store.resolving || !isResolvableProviderRecovery(item)) return const sessionID = props.sessionID setStore({ resolving: item.receiptID, status: "idle", detail: undefined }) const outcome = await resolveProviderRecovery({ @@ -195,6 +216,7 @@ export function SessionProviderRecoveryDock(props: { } const confirm = (item: ProviderRecovery) => + isResolvableProviderRecovery(item) && dialog.show(() => (
@@ -249,10 +271,7 @@ export function SessionProviderRecoveryDock(props: { } > {(item) => { - const supported = () => - item.continuationRecoverySupported && - item.workspaceRecoverySupported && - item.sourceWorldStateBaselineStatus === "available" + const supported = () => isResolvableProviderRecovery(item) return (
+ +
+ {language.t("session.providerRecovery.maintenanceUnsupported")} +
+
{language.t( diff --git a/packages/core/bug407-artifact-backfill-one.gate.ts b/packages/core/bug407-artifact-backfill-one.gate.ts new file mode 100644 index 000000000..5bf7ef35c --- /dev/null +++ b/packages/core/bug407-artifact-backfill-one.gate.ts @@ -0,0 +1,32 @@ +const database = process.env.BUG407_GATE_DATABASE +if (!database) throw new Error("BUG407_GATE_DATABASE is required") + +process.env.DEEPAGENT_CODE_DB = database + +const { Effect } = await import("effect") +const { EventV2 } = await import("./src/event.ts") + +const started = performance.now() +const result = await Effect.runPromise( + Effect.gen(function* () { + const events = yield* EventV2.Service + let cursor: EventV2.ID | undefined + let batches = 0 + let processed = 0 + + while (true) { + const page = yield* events.canonicalizeLegacyArtifacts({ + ...(cursor ? { afterID: cursor } : {}), + limit: 1, + }) + batches += 1 + processed += page.processed + if (!page.next) break + cursor = page.next + } + + return { batches, processed } + }).pipe(Effect.provide(EventV2.defaultLayer)), +) + +process.stdout.write(`${JSON.stringify({ ...result, elapsedMs: Math.round(performance.now() - started) })}\n`) diff --git a/packages/core/bug407-artifact-backfill.gate.ts b/packages/core/bug407-artifact-backfill.gate.ts new file mode 100644 index 000000000..bb5fbbe06 --- /dev/null +++ b/packages/core/bug407-artifact-backfill.gate.ts @@ -0,0 +1,32 @@ +const database = process.env.BUG407_GATE_DATABASE +if (!database) throw new Error("BUG407_GATE_DATABASE is required") + +process.env.DEEPAGENT_CODE_DB = database + +const { Effect } = await import("effect") +const { EventV2 } = await import("./src/event.ts") + +const started = performance.now() +const result = await Effect.runPromise( + Effect.gen(function* () { + const events = yield* EventV2.Service + let cursor: EventV2.ID | undefined + let batches = 0 + let processed = 0 + + while (true) { + const page = yield* events.canonicalizeLegacyArtifacts({ + ...(cursor ? { afterID: cursor } : {}), + limit: EventV2.LEGACY_ARTIFACT_BATCH_EVENTS, + }) + batches += 1 + processed += page.processed + if (!page.next) break + cursor = page.next + } + + return { batches, processed } + }).pipe(Effect.provide(EventV2.defaultLayer)), +) + +process.stdout.write(`${JSON.stringify({ ...result, elapsedMs: Math.round(performance.now() - started) })}\n`) diff --git a/packages/core/bug407-migration-only.gate.ts b/packages/core/bug407-migration-only.gate.ts new file mode 100644 index 000000000..3c0d5a929 --- /dev/null +++ b/packages/core/bug407-migration-only.gate.ts @@ -0,0 +1,13 @@ +const database = process.env.BUG407_GATE_DATABASE +if (!database) throw new Error("BUG407_GATE_DATABASE is required") + +process.env.DEEPAGENT_CODE_DB = database + +const { Effect } = await import("effect") +const { EventV2 } = await import("./src/event.ts") + +const started = performance.now() +await Effect.runPromise(Effect.gen(function* () { + yield* EventV2.Service +}).pipe(Effect.provide(EventV2.defaultLayer))) +process.stdout.write(`${JSON.stringify({ elapsedMs: Math.round(performance.now() - started) })}\n`) diff --git a/packages/core/src/context-federation/session-sql.ts b/packages/core/src/context-federation/session-sql.ts index 6785fdd6d..c7afe9f48 100644 --- a/packages/core/src/context-federation/session-sql.ts +++ b/packages/core/src/context-federation/session-sql.ts @@ -178,7 +178,7 @@ export const SessionProviderAttemptResolutionTable = sqliteTable( resolution_id: text().primaryKey(), attempt_id: text() .notNull() - .references(() => SessionProviderAttemptTable.attempt_id), + .references(() => SessionProviderAttemptTable.attempt_id, { onDelete: "cascade" }), actor_type: text().$type<"user" | "administrator" | "system">().notNull(), actor_id: text().notNull(), decision: text().$type<"abandoned" | "settled" | "replayed">().notNull(), @@ -190,6 +190,17 @@ export const SessionProviderAttemptResolutionTable = sqliteTable( (table) => [uniqueIndex("session_provider_attempt_resolution_attempt_idx").on(table.attempt_id)], ) +export const SessionProviderAttemptRecoveryBridgeTable = sqliteTable("session_provider_attempt_recovery_bridge", { + resolution_id: text().primaryKey(), + attempt_id: text() + .notNull() + .unique() + .references(() => SessionProviderAttemptTable.attempt_id, { onDelete: "cascade" }), + receipt_id: text().notNull().unique(), + command_id: text().notNull().unique(), + created_at: integer().notNull(), +}) + export const ContextArtifactTable = sqliteTable( "context_artifact", { diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index d0345dbb9..b0a93104a 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -83,5 +83,8 @@ export const migrations = ( import("./migration/20260812120000_legacy_provider_recovery"), import("./migration/20260812130000_legacy_activity_lifecycle_expand"), import("./migration/20260812140000_session_diff_manifest"), + import("./migration/20260813100000_event_snapshot_authority"), + import("./migration/20260813110000_provider_recovery_authority_bridge"), + import("./migration/20260813120000_legacy_provider_receipt_supersession"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration.ts b/packages/core/src/database/migration.ts index 55f835a50..ba2170f40 100644 --- a/packages/core/src/database/migration.ts +++ b/packages/core/src/database/migration.ts @@ -20,6 +20,10 @@ export function apply(db: Database) { export function applyOnly(db: Database, input: Migration[]) { return Effect.gen(function* () { + const duplicate = input.find( + (migration, index) => input.findIndex((candidate) => candidate.id === migration.id) !== index, + ) + if (duplicate) return yield* Effect.die(new Error(`duplicate database migration id: ${duplicate.id}`)) yield* db.run( sql`CREATE TABLE IF NOT EXISTS ${sql.identifier("migration")} (id TEXT PRIMARY KEY, time_completed INTEGER NOT NULL)`, ) diff --git a/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts b/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts new file mode 100644 index 000000000..09d2d5dba --- /dev/null +++ b/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts @@ -0,0 +1,165 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813100000_event_snapshot_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run("ALTER TABLE event_sequence ADD COLUMN retention_floor_seq INTEGER") + yield* tx.run("ALTER TABLE event_sequence ADD COLUMN snapshot_id TEXT") + yield* tx.run("ALTER TABLE event ADD COLUMN sync_seq INTEGER") + yield* tx.run("UPDATE event SET sync_seq = rowid WHERE sync_seq IS NULL") + yield* tx.run("CREATE UNIQUE INDEX event_sync_seq_idx ON event(sync_seq)") + yield* tx.run(` + CREATE TABLE event_sync_sequence ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + seq INTEGER NOT NULL CHECK (seq >= -1), + generation TEXT NOT NULL CHECK (length(generation) = 32), + cursor_secret TEXT NOT NULL CHECK (length(cursor_secret) = 64) + ) + `) + yield* tx.run("INSERT INTO event_sync_sequence(id, seq, generation, cursor_secret) SELECT 1, COALESCE(MAX(sync_seq), -1), lower(hex(randomblob(16))), lower(hex(randomblob(32))) FROM event") + yield* tx.run(` + CREATE TABLE workspace_sync_cursor ( + workspace_id TEXT NOT NULL, + remote_fingerprint TEXT NOT NULL CHECK (length(remote_fingerprint) = 64), + cursor TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (workspace_id, remote_fingerprint) + ) + `) + yield* tx.run(` + CREATE TABLE event_snapshot ( + snapshot_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + through_seq INTEGER NOT NULL CHECK (through_seq >= 0), + sync_seq INTEGER NOT NULL UNIQUE CHECK (sync_seq >= 0), + codec TEXT NOT NULL CHECK (length(codec) > 0), + schema_version INTEGER NOT NULL CHECK (schema_version >= 1), + snapshot_hash TEXT NOT NULL CHECK (length(snapshot_hash) = 64), + body TEXT NOT NULL CHECK (json_valid(body)), + owner_id TEXT, + created_at INTEGER NOT NULL, + UNIQUE (aggregate_id, through_seq) + ) + `) + yield* tx.run( + "CREATE INDEX event_snapshot_aggregate_created_idx ON event_snapshot(aggregate_id, created_at)", + ) + yield* tx.run(` + CREATE TABLE event_dedupe ( + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + seq INTEGER NOT NULL CHECK (seq >= 0), + event_id TEXT NOT NULL UNIQUE, + type TEXT NOT NULL, + data_hash TEXT NOT NULL CHECK (length(data_hash) = 64), + compacted_at INTEGER NOT NULL, + UNIQUE (aggregate_id, seq) + ) + `) + yield* tx.run(` + CREATE TABLE event_artifact ( + artifact_id TEXT NOT NULL PRIMARY KEY, + event_id TEXT NOT NULL UNIQUE, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + seq INTEGER NOT NULL CHECK (seq >= 0), + kind TEXT NOT NULL CHECK (kind IN ('legacy_message_diff')), + original_data_hash TEXT NOT NULL CHECK (length(original_data_hash) = 64), + canonical_data_hash TEXT NOT NULL CHECK (length(canonical_data_hash) = 64), + canonical_data TEXT NOT NULL CHECK (json_valid(canonical_data)), + body_hash TEXT NOT NULL CHECK (length(body_hash) = 64), + body_bytes INTEGER NOT NULL CHECK (body_bytes >= 0), + chunk_count INTEGER NOT NULL CHECK (chunk_count >= 1), + codec_version INTEGER NOT NULL CHECK (codec_version >= 1), + created_at INTEGER NOT NULL, + UNIQUE (aggregate_id, seq) + ) + `) + yield* tx.run(` + CREATE TABLE event_artifact_chunk ( + artifact_id TEXT NOT NULL REFERENCES event_artifact(artifact_id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL CHECK (chunk_index >= 0), + data BLOB NOT NULL, + chunk_hash TEXT NOT NULL CHECK (length(chunk_hash) = 64), + PRIMARY KEY (artifact_id, chunk_index) + ) + `) + yield* tx.run(` + CREATE TRIGGER event_sync_seq_legacy_allocator + AFTER INSERT ON event + WHEN NEW.sync_seq IS NULL + BEGIN + UPDATE event_sync_sequence SET seq = seq + 1 WHERE id = 1; + UPDATE event SET sync_seq = (SELECT seq FROM event_sync_sequence WHERE id = 1) WHERE id = NEW.id; + END + `) + yield* tx.run(` + CREATE TRIGGER event_sequence_snapshot_validate_insert + BEFORE INSERT ON event_sequence + WHEN NEW.retention_floor_seq IS NOT NULL OR NEW.snapshot_id IS NOT NULL + BEGIN + SELECT RAISE(ABORT, 'event_sequence_snapshot_requires_checkpoint_update'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_sequence_snapshot_validate_update + BEFORE UPDATE OF retention_floor_seq, snapshot_id ON event_sequence + BEGIN + SELECT CASE + WHEN NEW.retention_floor_seq IS NULL OR NEW.snapshot_id IS NULL + THEN RAISE(ABORT, 'event_sequence_snapshot_pointer_incomplete') + WHEN NEW.retention_floor_seq < 0 OR NEW.retention_floor_seq > NEW.seq + THEN RAISE(ABORT, 'event_sequence_snapshot_floor_invalid') + WHEN OLD.retention_floor_seq IS NOT NULL AND NEW.retention_floor_seq < OLD.retention_floor_seq + THEN RAISE(ABORT, 'event_sequence_snapshot_floor_regressed') + WHEN NOT EXISTS ( + SELECT 1 FROM event_snapshot snapshot + WHERE snapshot.snapshot_id = NEW.snapshot_id + AND snapshot.aggregate_id = NEW.aggregate_id + AND snapshot.through_seq = NEW.retention_floor_seq + ) THEN RAISE(ABORT, 'event_sequence_snapshot_missing') + END; + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_delete_active_guard + BEFORE DELETE ON event_snapshot + WHEN EXISTS (SELECT 1 FROM event_sequence sequence WHERE sequence.snapshot_id = OLD.snapshot_id) + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_is_active'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_artifact_binding_guard + BEFORE INSERT ON event_artifact + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM event source + WHERE source.id = NEW.event_id + AND source.aggregate_id = NEW.aggregate_id + AND source.seq = NEW.seq + ) THEN RAISE(ABORT, 'event_artifact_source_mismatch') END; + END + `) + yield* tx.run(` + CREATE TRIGGER event_delete_requires_dedupe + BEFORE DELETE ON event + WHEN EXISTS ( + SELECT 1 FROM event_sequence sequence + WHERE sequence.aggregate_id = OLD.aggregate_id + AND sequence.retention_floor_seq IS NOT NULL + AND OLD.seq <= sequence.retention_floor_seq + ) AND NOT EXISTS ( + SELECT 1 FROM event_dedupe dedupe + WHERE dedupe.aggregate_id = OLD.aggregate_id + AND dedupe.seq = OLD.seq + AND dedupe.event_id = OLD.id + AND dedupe.type = OLD.type + ) + BEGIN + SELECT RAISE(ABORT, 'event_compaction_dedupe_missing'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813110000_provider_recovery_authority_bridge.ts b/packages/core/src/database/migration/20260813110000_provider_recovery_authority_bridge.ts new file mode 100644 index 000000000..660d898d0 --- /dev/null +++ b/packages/core/src/database/migration/20260813110000_provider_recovery_authority_bridge.ts @@ -0,0 +1,186 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813110000_provider_recovery_authority_bridge", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE session_provider_attempt_resolution_rebuilt ( + resolution_id TEXT PRIMARY KEY, + attempt_id TEXT NOT NULL UNIQUE + REFERENCES session_provider_attempt(attempt_id) ON DELETE CASCADE, + actor_type TEXT NOT NULL CHECK (actor_type IN ('user', 'administrator', 'system')), + actor_id TEXT NOT NULL, + decision TEXT NOT NULL CHECK (decision IN ('abandoned', 'settled', 'replayed')), + provider_evidence TEXT, + risk_acknowledged INTEGER NOT NULL CHECK (risk_acknowledged IN (0, 1)), + reason TEXT NOT NULL, + created_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + INSERT INTO session_provider_attempt_resolution_rebuilt + SELECT * FROM session_provider_attempt_resolution + `) + yield* tx.run("DROP TABLE session_provider_attempt_resolution") + yield* tx.run(` + ALTER TABLE session_provider_attempt_resolution_rebuilt + RENAME TO session_provider_attempt_resolution + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_resolution_immutable + BEFORE UPDATE ON session_provider_attempt_resolution + BEGIN + SELECT RAISE(ABORT, 'session_provider_attempt_resolution is append-only'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_resolution_immutable_delete + BEFORE DELETE ON session_provider_attempt_resolution + WHEN EXISTS ( + SELECT 1 + FROM session_provider_attempt attempt + JOIN session ON session.id = attempt.session_id + WHERE attempt.attempt_id = OLD.attempt_id + ) + BEGIN + SELECT RAISE(ABORT, 'session_provider_attempt_resolution is append-only'); + END + `) + yield* tx.run(` + CREATE TABLE session_provider_attempt_recovery_bridge ( + resolution_id TEXT NOT NULL PRIMARY KEY, + attempt_id TEXT NOT NULL UNIQUE + REFERENCES session_provider_attempt(attempt_id) ON DELETE CASCADE, + receipt_id TEXT NOT NULL UNIQUE + REFERENCES session_tool_request_receipt(receipt_id) ON DELETE CASCADE, + command_id TEXT NOT NULL UNIQUE + REFERENCES session_tool_request_resolution_command(command_id) ON DELETE CASCADE, + created_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_recovery_bridge_validate_insert + BEFORE INSERT ON session_provider_attempt_recovery_bridge + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 + FROM session_tool_request_receipt receipt + JOIN session_provider_attempt attempt + ON attempt.attempt_id = receipt.provider_attempt_id + JOIN session_tool_request_resolution_command command + ON command.command_id = NEW.command_id + WHERE receipt.receipt_id = NEW.receipt_id + AND receipt.provider_attempt_id = NEW.attempt_id + AND receipt.provider_state = 'indeterminate_after_crash' + AND attempt.session_id = receipt.session_id + AND attempt.state = 'indeterminate_after_crash' + AND command.session_id = receipt.session_id + AND command.receipt_id = receipt.receipt_id + AND command.result_resolution_id IS NULL + ) THEN RAISE(ABORT, 'provider_recovery_bridge_authority_invalid') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM session_provider_attempt_resolution + WHERE attempt_id = NEW.attempt_id + ) THEN RAISE(ABORT, 'provider_recovery_bridge_attempt_already_resolved') END; + SELECT CASE WHEN EXISTS ( + SELECT 1 FROM session_tool_request_resolution + WHERE receipt_id = NEW.receipt_id + ) THEN RAISE(ABORT, 'provider_recovery_bridge_receipt_already_resolved') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_recovery_bridge_immutable_update + BEFORE UPDATE ON session_provider_attempt_recovery_bridge + BEGIN + SELECT RAISE(ABORT, 'provider_recovery_bridge_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_recovery_bridge_immutable_delete + BEFORE DELETE ON session_provider_attempt_recovery_bridge + WHEN EXISTS ( + SELECT 1 + FROM session_tool_request_receipt receipt + JOIN session ON session.id = receipt.session_id + WHERE receipt.receipt_id = OLD.receipt_id + ) + BEGIN + SELECT RAISE(ABORT, 'provider_recovery_bridge_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_resolution_legacy_guard + BEFORE INSERT ON session_provider_attempt_resolution + WHEN EXISTS ( + SELECT 1 FROM session_tool_request_receipt receipt + WHERE receipt.provider_attempt_id = NEW.attempt_id + AND receipt.provider_state = 'indeterminate_after_crash' + ) + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 + FROM session_provider_attempt_recovery_bridge bridge + JOIN session_tool_request_resolution resolution + ON resolution.resolution_id = bridge.resolution_id + AND resolution.receipt_id = bridge.receipt_id + WHERE bridge.resolution_id = NEW.resolution_id + AND bridge.attempt_id = NEW.attempt_id + AND resolution.decision = NEW.decision + ) THEN RAISE(ABORT, 'provider_attempt_requires_unified_legacy_recovery') END; + SELECT CASE WHEN NEW.decision != 'abandoned' + THEN RAISE(ABORT, 'provider_attempt_legacy_recovery_only_supports_abandoned') END; + END + `) + yield* tx.run("DROP TRIGGER session_tool_request_resolution_validate_insert") + yield* tx.run(` + CREATE TRIGGER session_tool_request_resolution_validate_insert + BEFORE INSERT ON session_tool_request_resolution + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_tool_request_receipt receipt + WHERE receipt.receipt_id = NEW.receipt_id + AND receipt.session_id = NEW.session_id + AND receipt.assistant_message_id = NEW.assistant_message_id + AND receipt.provider_state = NEW.expected_provider_state + AND receipt.prompt_epoch = NEW.source_prompt_epoch + AND receipt.prompt_window_id = NEW.source_window_id + AND receipt.effective_history_hash = NEW.source_effective_history_hash + AND COALESCE(receipt.final_request_hash, receipt.provider_request_hash, receipt.request_input_hash) = NEW.source_request_hash + AND ( + receipt.provider_attempt_id IS NULL OR EXISTS ( + SELECT 1 + FROM session_provider_attempt_recovery_bridge bridge + WHERE bridge.resolution_id = NEW.resolution_id + AND bridge.receipt_id = NEW.receipt_id + AND bridge.attempt_id = receipt.provider_attempt_id + ) + ) + ) THEN RAISE(ABORT, 'legacy_provider_resolution_receipt_binding_invalid') END; + SELECT CASE WHEN NEW.legacy_activity_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_activity_progress progress + JOIN session_legacy_activity activity ON activity.activity_id = progress.activity_id + WHERE progress.provider_receipt_id = NEW.receipt_id + AND progress.activity_id = NEW.legacy_activity_id + AND activity.session_id = NEW.session_id + AND progress.state = 'recovery_required' + AND activity.state = 'recovery_required' + ) THEN RAISE(ABORT, 'legacy_provider_resolution_activity_binding_invalid') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_prompt_epoch source + WHERE source.session_id = NEW.session_id + AND source.epoch = NEW.source_prompt_epoch + AND source.window_id = NEW.source_window_id + AND source.state = 'retired' + AND source.authority_state = 'recovery_required' + ) THEN RAISE(ABORT, 'legacy_provider_resolution_source_binding_invalid') END; + SELECT CASE WHEN NEW.successor_prompt_epoch <= NEW.source_prompt_epoch + THEN RAISE(ABORT, 'legacy_provider_resolution_successor_not_newer') END; + SELECT CASE WHEN NEW.successor_mutation_epoch != NEW.source_mutation_epoch + 1 + THEN RAISE(ABORT, 'legacy_provider_resolution_mutation_epoch_invalid') END; + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813120000_legacy_provider_receipt_supersession.ts b/packages/core/src/database/migration/20260813120000_legacy_provider_receipt_supersession.ts new file mode 100644 index 000000000..43614e1f9 --- /dev/null +++ b/packages/core/src/database/migration/20260813120000_legacy_provider_receipt_supersession.ts @@ -0,0 +1,39 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813120000_legacy_provider_receipt_supersession", + up(tx) { + return Effect.gen(function* () { + yield* tx.run("DROP TRIGGER session_tool_request_receipt_provider_transition") + yield* tx.run(` + UPDATE session_tool_request_receipt AS receipt + SET provider_state = 'failed', + terminal_at = COALESCE(receipt.terminal_at, receipt.created_at), + request_error_code = 'legacy_request_superseded_by_later_request' + WHERE receipt.provider_state = 'indeterminate_after_crash' + AND receipt.request_error_code = 'legacy_dispatch_outcome_unknown' + AND receipt.provider_attempt_id IS NULL + AND EXISTS ( + SELECT 1 + FROM session_tool_request_receipt AS later + WHERE later.session_id = receipt.session_id + AND later.request_ordinal > receipt.request_ordinal + ) + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_provider_transition + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_state != OLD.provider_state AND NOT ( + (OLD.provider_state = 'preparing' AND NEW.provider_state IN ('prepared', 'failed')) OR + (OLD.provider_state = 'prepared' AND NEW.provider_state IN ('dispatching', 'failed')) OR + (OLD.provider_state = 'dispatching' AND NEW.provider_state IN ('streaming', 'settled', 'failed', 'indeterminate_after_crash')) OR + (OLD.provider_state = 'streaming' AND NEW.provider_state IN ('settled', 'failed', 'indeterminate_after_crash')) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal provider receipt transition'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index 9a5307fb5..e47a646d7 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -1,14 +1,23 @@ export * as EventV2 from "./event" import { Cause, Context, Effect, Layer, Option, PubSub, Schema, Stream } from "effect" -import { and, eq, gt, inArray, sql } from "drizzle-orm" +import { and, asc, eq, gt, inArray, lte, sql } from "drizzle-orm" import { Database } from "./database/database" -import { EventSequenceTable, EventTable } from "./event/sql" +import { + EventArtifactTable, + EventArtifactChunkTable, + EventDedupeTable, + EventSequenceTable, + EventSnapshotTable, + EventSyncSequenceTable, + EventTable, +} from "./event/sql" import { Location } from "./location" import { makeGlobalNode } from "./effect/app-node" import { externalID, type ExternalID, NonNegativeInt, withStatics } from "./schema" import { Identifier } from "./util/identifier" import { isDeepStrictEqual } from "node:util" +import { Hash } from "./util/hash" export const ID = Schema.String.check(Schema.isStartsWith("evt_")).pipe( Schema.brand("Event.ID"), @@ -48,6 +57,10 @@ export type Payload = { readonly metadata?: Record /** Internal replay marker for projectors that own non-replicated operational state. */ readonly replay?: boolean + /** Internal exact-replay marker set only after the durable event identity and payload are verified. */ + readonly replayExact?: boolean + /** Internal owner authority supplied by a replay ingress. It is never serialized into the event payload. */ + readonly replayOwnerID?: string } export type Projector = (event: Payload) => Effect.Effect @@ -65,6 +78,19 @@ export type SerializedEvent = { readonly data: Record } +export type SerializedSnapshot = { + readonly snapshotID: string + readonly aggregateID: string + readonly throughSeq: number + readonly syncSeq: number + readonly codec: string + readonly schemaVersion: number + readonly snapshotHash: string + readonly body: Record + readonly ownerID?: string + readonly createdAt: number +} + export type CursorEvent = { readonly cursor: Cursor readonly event: E @@ -81,6 +107,10 @@ export class InvalidSyncEventError extends Schema.TaggedErrorClass()( "EventV2.EncodedPayloadTooLarge", @@ -92,6 +122,18 @@ export class EncodedPayloadTooLargeError extends Schema.TaggedErrorClass()( + "EventV2.ResyncRequired", + { + aggregateID: Schema.String, + requestedAfter: Schema.Number, + snapshotID: Schema.String, + baseSeq: NonNegativeInt, + snapshotHash: Schema.String, + message: Schema.String, + }, +) {} + export function versionedType(type: string, version: number) { return `${type}.${version}` } @@ -119,6 +161,62 @@ function admitEncodedPayload(definition: SyncDefinition, data: unknown) { return { encoded } as const } +function stableJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]` + if (value && typeof value === "object") + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`) + .join(",")}}` + return JSON.stringify(value) +} + +function hashJson(value: unknown) { + return Hash.sha256(stableJson(value)) +} + +function canonicalizeLegacyMessageDiff(eventID: ID, type: string, data: Record) { + if (type !== versionedType("message.updated", 1)) return + if (!data.info || typeof data.info !== "object" || Array.isArray(data.info)) return + const info = data.info as Record + if (!info.summary || typeof info.summary !== "object" || Array.isArray(info.summary)) return + const summary = info.summary as Record + if (!Array.isArray(summary.diffs) || summary.diffs.length === 0) return + const originalDataHash = hashJson(data) + const body = { diffs: summary.diffs } + const bodyHash = hashJson(body) + const artifactID = `evtart_${Hash.sha256(`legacy-message-diff.v1:${eventID}:${originalDataHash}:${bodyHash}`)}` + const diffs = summary.diffs.slice(0, LEGACY_DIFF_DESCRIPTOR_FILES).flatMap((item) => { + if (!item || typeof item !== "object" || Array.isArray(item)) return [] + const { patch: _, ...descriptor } = item as Record + return [descriptor] + }) + const canonicalData = { + ...data, + info: { + ...info, + summary: { + ...summary, + diffs, + diffArtifact: { + id: artifactID, + hash: bodyHash, + codec: "legacy-message-diff.v1", + fileCount: summary.diffs.length, + }, + }, + }, + } + return { + kind: "legacy_message_diff" as const, + artifactID, + originalDataHash, + canonicalDataHash: hashJson(canonicalData), + canonicalData, + body, + } +} + // Synchronized events cross a JSON boundary, so their data schemas must encode and decode without services. const syncCodec = (definition: Definition) => definition.data as Schema.Codec @@ -203,6 +301,29 @@ export interface Interface { events: SerializedEvent[], options?: { readonly publish?: boolean; readonly ownerID?: string; readonly strictOwner?: boolean }, ) => Effect.Effect + readonly snapshot: (aggregateID: string) => Effect.Effect + readonly checkpoint: (input: { + readonly aggregateID: string + readonly throughSeq: Cursor + readonly codec: string + readonly schemaVersion: number + readonly body: Record + readonly expectedLatest: Cursor + readonly ownerID?: string + readonly now?: number + }) => Effect.Effect + readonly importSnapshot: (snapshot: SerializedSnapshot) => Effect.Effect + readonly compact: (input: { + readonly aggregateID: string + readonly throughSeq: Cursor + readonly limit?: number + readonly now?: number + }) => Effect.Effect<{ readonly deleted: number; readonly complete: boolean }> + readonly canonicalizeLegacyArtifacts: (input?: { + readonly afterID?: ID + readonly limit?: number + readonly now?: number + }) => Effect.Effect<{ readonly processed: number; readonly next?: ID }> readonly remove: (aggregateID: string) => Effect.Effect readonly claim: (aggregateID: string, ownerID: string) => Effect.Effect } @@ -212,9 +333,11 @@ export class Service extends Context.Service()("@deepagent-c export interface LayerOptions { readonly beforeAggregateRead?: (aggregateID: string) => Effect.Effect readonly afterAggregateReadMetadata?: (aggregateID: string, eventIDs: readonly ID[]) => Effect.Effect + readonly afterAggregateRead?: (aggregateID: string) => Effect.Effect + readonly afterReplayAllCommit?: (aggregateID: string) => Effect.Effect } -export const layerWith = (options?: LayerOptions) => +export const layerWith = (layerOptions?: LayerOptions) => Layer.effect( Service, Effect.gen(function* () { @@ -258,6 +381,7 @@ export const layerWith = (options?: LayerOptions) => }, commit?: (seq: number) => Effect.Effect, idempotent = false, + deferDurableWake = false, ) { return Effect.gen(function* () { const definition = registry.get(event.type) @@ -323,12 +447,44 @@ export const layerWith = (options?: LayerOptions) => .where(and(eq(EventTable.aggregate_id, aggregateID), eq(EventTable.seq, input.seq))) .get() .pipe(Effect.orDie) + const artifact = stored + ? yield* db + .select({ + originalDataHash: EventArtifactTable.original_data_hash, + canonicalData: EventArtifactTable.canonical_data, + }) + .from(EventArtifactTable) + .where(eq(EventArtifactTable.event_id, stored.id)) + .get() + .pipe(Effect.orDie) + : undefined + const dedupe = !stored + ? yield* db + .select() + .from(EventDedupeTable) + .where( + and( + eq(EventDedupeTable.aggregate_id, aggregateID), + eq(EventDedupeTable.seq, input.seq), + ), + ) + .get() + .pipe(Effect.orDie) + : undefined if ( - stored?.id === event.id && - stored.type === versionedType(definition.type, sync.version) && - isDeepStrictEqual(stored.data, encoded) + ((stored?.id === event.id && + stored.type === versionedType(definition.type, sync.version) && + (isDeepStrictEqual(stored.data, encoded) || + isDeepStrictEqual(artifact?.canonicalData, encoded) || + artifact?.originalDataHash === hashJson(encoded))) || + (dedupe?.event_id === event.id && + dedupe.type === versionedType(definition.type, sync.version) && + dedupe.data_hash === hashJson(encoded))) ) { if (input.ownerID && row?.ownerID == null) { + for (const guard of commitGuards) { + yield* guard({ ...event, replayExact: true }) + } yield* db .update(EventSequenceTable) .set({ owner_id: input.ownerID }) @@ -404,6 +560,17 @@ export const layerWith = (options?: LayerOptions) => }) .run() .pipe(Effect.orDie) + const syncSequence = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) + if (!syncSequence) + return yield* Effect.die( + new InvalidSyncEventError({ type: event.type, message: "Sync sequence authority missing" }), + ) yield* db .insert(EventTable) .values([ @@ -413,6 +580,7 @@ export const layerWith = (options?: LayerOptions) => seq, type: versionedType(definition.type, sync.version), data: encoded, + sync_seq: syncSequence.seq, }, ]) .run() @@ -422,7 +590,7 @@ export const layerWith = (options?: LayerOptions) => { behavior: "immediate" }, ) .pipe(Effect.orDie) - if (committed?.inserted) { + if (committed?.inserted && !deferDurableWake) { yield* Effect.forEach( synchronized.get(committed.aggregateID) ?? [], (pubsub) => PubSub.publish(pubsub, undefined), @@ -526,7 +694,13 @@ export const layerWith = (options?: LayerOptions) => function replay( event: SerializedEvent, - options?: { readonly publish?: boolean; readonly ownerID?: string; readonly strictOwner?: boolean }, + options?: { + readonly publish?: boolean + readonly ownerID?: string + readonly strictOwner?: boolean + readonly onCommitted?: (event: Payload) => void + readonly deferDurableWake?: boolean + }, ) { return Effect.gen(function* () { const definition = syncRegistry.get(event.type) @@ -539,15 +713,24 @@ export const layerWith = (options?: LayerOptions) => id: event.id, type: definition.type, version: definition.sync.version, + seq: event.seq, data: definition.decode(event.data), replay: true, + ...(options?.ownerID ? { replayOwnerID: options.ownerID } : {}), } as Payload - const committed = yield* commitSyncEvent(payload, { - seq: event.seq, - aggregateID: event.aggregateID, - ownerID: options?.ownerID, - strictOwner: options?.strictOwner, - }) + const committed = yield* commitSyncEvent( + payload, + { + seq: event.seq, + aggregateID: event.aggregateID, + ownerID: options?.ownerID, + strictOwner: options?.strictOwner, + }, + undefined, + false, + options?.deferDurableWake, + ) + if (committed) options?.onCommitted?.({ ...payload, seq: committed.seq }) if (committed && options?.publish) { yield* notify({ ...payload, seq: committed.seq }, true) } @@ -593,13 +776,451 @@ export const layerWith = (options?: LayerOptions) => const admission = admitEncodedPayload(definition, definition.decode(event.data)) if ("error" in admission) return yield* Effect.die(admission.error) } - for (const event of events) { - yield* replay(event, options) - } + const committed: Payload[] = [] + yield* Effect.uninterruptible( + Effect.gen(function* () { + yield* db + .transaction( + () => + Effect.forEach( + events, + (event) => + replay(event, { + ...options, + publish: false, + deferDurableWake: true, + onCommitted: (event) => committed.push(event), + }), + { discard: true }, + ), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie) + yield* (layerOptions?.afterReplayAllCommit?.(source) ?? Effect.void).pipe( + Effect.catchCause(() => Effect.void), + ) + if (committed.length > 0) + yield* Effect.forEach(synchronized.get(source) ?? [], (pubsub) => PubSub.publish(pubsub, undefined), { + discard: true, + }) + }), + ) + if (options?.publish) yield* Effect.forEach(committed, (event) => notify(event, true), { discard: true }) return source }) } + function snapshot(aggregateID: string) { + return db + .select({ + snapshotID: EventSnapshotTable.snapshot_id, + aggregateID: EventSnapshotTable.aggregate_id, + throughSeq: EventSnapshotTable.through_seq, + syncSeq: EventSnapshotTable.sync_seq, + codec: EventSnapshotTable.codec, + schemaVersion: EventSnapshotTable.schema_version, + snapshotHash: EventSnapshotTable.snapshot_hash, + body: EventSnapshotTable.body, + ownerID: EventSnapshotTable.owner_id, + createdAt: EventSnapshotTable.created_at, + }) + .from(EventSnapshotTable) + .innerJoin(EventSequenceTable, eq(EventSequenceTable.snapshot_id, EventSnapshotTable.snapshot_id)) + .where(eq(EventSnapshotTable.aggregate_id, aggregateID)) + .get() + .pipe( + Effect.orDie, + Effect.map((row) => + row + ? { + snapshotID: row.snapshotID, + aggregateID: row.aggregateID, + throughSeq: row.throughSeq, + syncSeq: row.syncSeq, + codec: row.codec, + schemaVersion: row.schemaVersion, + snapshotHash: row.snapshotHash, + body: row.body, + ...(row.ownerID ? { ownerID: row.ownerID } : {}), + createdAt: row.createdAt, + } + : undefined, + ), + ) + } + + function checkpoint(input: { + readonly aggregateID: string + readonly throughSeq: Cursor + readonly codec: string + readonly schemaVersion: number + readonly body: Record + readonly expectedLatest: Cursor + readonly ownerID?: string + readonly now?: number + }) { + return Effect.gen(function* () { + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: "Snapshot checkpointing is disabled until a registered canonical projection codec is available", + }), + ) + /* c8 ignore start - retained design is unreachable until a projection codec is registered */ + if (input.schemaVersion < 1 || input.codec.length === 0) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: "Snapshot codec and schema version are required" }), + ) + const encodedBytes = Buffer.byteLength(JSON.stringify(input.body)) + if (encodedBytes > SNAPSHOT_MAX_ENCODED_BYTES) + return yield* Effect.die( + new EncodedPayloadTooLargeError({ + type: "event.snapshot", + encodedBytes, + limitBytes: SNAPSHOT_MAX_ENCODED_BYTES, + message: `Snapshot body is ${encodedBytes} bytes; limit is ${SNAPSHOT_MAX_ENCODED_BYTES} bytes`, + }), + ) + const snapshotHash = hashJson(input.body) + const snapshotID = `evtsnap_${Hash.sha256( + `${input.codec}:${input.schemaVersion}:${input.aggregateID}:${input.throughSeq}:${snapshotHash}`, + )}` + const result = { + snapshotID, + aggregateID: input.aggregateID, + throughSeq: input.throughSeq, + syncSeq: 0, + codec: input.codec, + schemaVersion: input.schemaVersion, + snapshotHash, + body: input.body, + ...(input.ownerID ? { ownerID: input.ownerID } : {}), + createdAt: input.now ?? Date.now(), + } + const syncSeq = yield* db + .transaction( + () => + Effect.gen(function* () { + const current = yield* db + .select({ + seq: EventSequenceTable.seq, + ownerID: EventSequenceTable.owner_id, + floor: EventSequenceTable.retention_floor_seq, + }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, input.aggregateID)) + .get() + .pipe(Effect.orDie) + if ( + !current || + current.seq !== input.expectedLatest || + input.throughSeq > current.seq || + input.throughSeq < (current.floor ?? -1) || + (input.ownerID !== undefined && current.ownerID !== input.ownerID) + ) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Snapshot authority changed for aggregate ${input.aggregateID}`, + }), + ) + const syncSequence = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) + if (!syncSequence) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: "Sync sequence authority missing" }), + ) + yield* db + .insert(EventSnapshotTable) + .values({ + snapshot_id: snapshotID, + aggregate_id: input.aggregateID, + through_seq: input.throughSeq, + sync_seq: syncSequence.seq, + codec: input.codec, + schema_version: input.schemaVersion, + snapshot_hash: snapshotHash, + body: input.body, + owner_id: input.ownerID, + created_at: result.createdAt, + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ retention_floor_seq: input.throughSeq, snapshot_id: snapshotID }) + .where( + and( + eq(EventSequenceTable.aggregate_id, input.aggregateID), + eq(EventSequenceTable.seq, input.expectedLatest), + ), + ) + .run() + .pipe(Effect.orDie) + return syncSequence.seq + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie) + return { ...result, syncSeq } + /* c8 ignore stop */ + }) + } + + function importSnapshot(input: SerializedSnapshot) { + return Effect.gen(function* () { + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Unsupported snapshot codec ${input.codec}@${input.schemaVersion}; canonical projection import is disabled`, + }), + ) + /* c8 ignore start - retained design is unreachable until a projection codec is registered */ + if (hashJson(input.body) !== input.snapshotHash) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} hash mismatch` }), + ) + if (Buffer.byteLength(JSON.stringify(input.body)) > SNAPSHOT_MAX_ENCODED_BYTES) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} exceeds limit` }), + ) + yield* db + .transaction( + () => + Effect.gen(function* () { + const current = yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, input.aggregateID)) + .get() + .pipe(Effect.orDie) + if (current && current.seq !== input.throughSeq) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Snapshot import requires an empty or exact aggregate ${input.aggregateID}`, + }), + ) + yield* db + .insert(EventSequenceTable) + .values({ + aggregate_id: input.aggregateID, + seq: input.throughSeq, + owner_id: input.ownerID, + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`max(${EventSyncSequenceTable.seq}, ${input.syncSeq})` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .run() + .pipe(Effect.orDie) + yield* db + .insert(EventSnapshotTable) + .values({ + snapshot_id: input.snapshotID, + aggregate_id: input.aggregateID, + through_seq: input.throughSeq, + sync_seq: input.syncSeq, + codec: input.codec, + schema_version: input.schemaVersion, + snapshot_hash: input.snapshotHash, + body: input.body, + owner_id: input.ownerID, + created_at: input.createdAt, + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ retention_floor_seq: input.throughSeq, snapshot_id: input.snapshotID }) + .where(eq(EventSequenceTable.aggregate_id, input.aggregateID)) + .run() + .pipe(Effect.orDie) + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie) + /* c8 ignore stop */ + }) + } + + function compact(input: { + readonly aggregateID: string + readonly throughSeq: Cursor + readonly limit?: number + readonly now?: number + }) { + return Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Event compaction is disabled until canonical projection snapshots can be imported for ${input.aggregateID}`, + }), + ) + /* c8 ignore start - retained design is unreachable until a projection codec is registered */ + return db + .transaction( + () => + Effect.gen(function* () { + const sequence = yield* db + .select({ floor: EventSequenceTable.retention_floor_seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, input.aggregateID)) + .get() + .pipe(Effect.orDie) + if (sequence?.floor === undefined || sequence.floor === null || input.throughSeq > sequence.floor) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Compaction exceeds the durable retention floor for ${input.aggregateID}`, + }), + ) + const rows = yield* db + .select() + .from(EventTable) + .where(and(eq(EventTable.aggregate_id, input.aggregateID), lte(EventTable.seq, input.throughSeq))) + .orderBy(asc(EventTable.seq)) + .limit(Math.min(Math.max(input.limit ?? 100, 1), 100)) + .all() + .pipe(Effect.orDie) + yield* Effect.forEach( + rows, + (row) => + db + .insert(EventDedupeTable) + .values({ + aggregate_id: row.aggregate_id, + seq: row.seq, + event_id: row.id, + type: row.type, + data_hash: hashJson(row.data), + compacted_at: input.now ?? Date.now(), + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie), + { discard: true }, + ) + if (rows.length > 0) + yield* db + .delete(EventTable) + .where(inArray(EventTable.id, rows.map((row) => row.id))) + .run() + .pipe(Effect.orDie) + const remaining = yield* db + .select({ id: EventTable.id }) + .from(EventTable) + .where(and(eq(EventTable.aggregate_id, input.aggregateID), lte(EventTable.seq, input.throughSeq))) + .limit(1) + .get() + .pipe(Effect.orDie) + return { deleted: rows.length, complete: !remaining } + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie) + /* c8 ignore stop */ + } + + function canonicalizeLegacyArtifacts(input?: { readonly afterID?: ID; readonly limit?: number; readonly now?: number }) { + return db + .transaction( + () => + Effect.gen(function* () { + const rows = yield* db + .select() + .from(EventTable) + .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) + .where( + and( + eq(EventTable.type, versionedType("message.updated", 1)), + sql`length(CAST(${EventTable.data} AS BLOB)) > ${MAX_ENCODED_PAYLOAD_BYTES}`, + input?.afterID ? gt(EventTable.id, input.afterID) : undefined, + sql`${EventArtifactTable.event_id} is null`, + ), + ) + .orderBy(asc(EventTable.id)) + .limit(Math.min(Math.max(input?.limit ?? LEGACY_ARTIFACT_BATCH_EVENTS, 1), LEGACY_ARTIFACT_BATCH_EVENTS)) + .all() + .pipe(Effect.orDie) + const candidates = rows.flatMap((row) => { + const artifact = canonicalizeLegacyMessageDiff(row.event.id, row.event.type, row.event.data) + return artifact + ? [{ + eventID: row.event.id, + aggregateID: row.event.aggregate_id, + seq: row.event.seq, + artifact, + }] + : [] + }) + yield* Effect.forEach( + candidates, + ({ eventID, aggregateID, seq, artifact }) => { + const bytes = Buffer.from(JSON.stringify(artifact.body)) + const chunks = Array.from( + { length: Math.ceil(bytes.length / ARTIFACT_CHUNK_BYTES) || 1 }, + (_, index) => bytes.subarray(index * ARTIFACT_CHUNK_BYTES, (index + 1) * ARTIFACT_CHUNK_BYTES), + ) + return db + .insert(EventArtifactTable) + .values({ + artifact_id: artifact.artifactID, + event_id: eventID, + aggregate_id: aggregateID, + seq, + kind: artifact.kind, + original_data_hash: artifact.originalDataHash, + canonical_data_hash: artifact.canonicalDataHash, + canonical_data: artifact.canonicalData, + body_hash: hashJson(artifact.body), + body_bytes: bytes.length, + chunk_count: chunks.length, + codec_version: 1, + created_at: input?.now ?? Date.now(), + }) + .onConflictDoNothing() + .run() + .pipe( + Effect.orDie, + Effect.andThen( + Effect.forEach( + chunks, + (chunk, index) => + db + .insert(EventArtifactChunkTable) + .values({ + artifact_id: artifact.artifactID, + chunk_index: index, + data: chunk, + chunk_hash: Hash.sha256(chunk), + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie), + { discard: true }, + ), + ), + ) + }, + { discard: true }, + ) + return { processed: candidates.length, next: rows.at(-1)?.event.id } + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie) + } + function remove(aggregateID: string) { return db .transaction(() => @@ -648,7 +1269,37 @@ export const layerWith = (options?: LayerOptions) => const readAfter = (aggregateID: string, after: number) => Effect.gen(function* () { - yield* options?.beforeAggregateRead?.(aggregateID) ?? Effect.void + yield* layerOptions?.beforeAggregateRead?.(aggregateID) ?? Effect.void + const authority = yield* db + .select({ + floor: EventSequenceTable.retention_floor_seq, + snapshotID: EventSequenceTable.snapshot_id, + snapshotHash: EventSnapshotTable.snapshot_hash, + }) + .from(EventSequenceTable) + .leftJoin(EventSnapshotTable, eq(EventSnapshotTable.snapshot_id, EventSequenceTable.snapshot_id)) + .where(eq(EventSequenceTable.aggregate_id, aggregateID)) + .get() + .pipe(Effect.orDie) + if (authority?.floor !== null && authority?.floor !== undefined && after < authority.floor) { + if (!authority.snapshotID || !authority.snapshotHash) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Retention floor for ${aggregateID} has no readable snapshot`, + }), + ) + return yield* Effect.die( + new ResyncRequiredError({ + aggregateID, + requestedAfter: after, + snapshotID: authority.snapshotID, + baseSeq: authority.floor, + snapshotHash: authority.snapshotHash, + message: `Aggregate ${aggregateID} requires snapshot resynchronization`, + }), + ) + } const candidates = yield* db .all<{ id: ID @@ -662,8 +1313,9 @@ export const layerWith = (options?: LayerOptions) => ${EventTable.id} AS id, ${EventTable.seq} AS seq, ${EventTable.type} AS type, - length(CAST(${EventTable.data} AS BLOB)) AS bytes + length(CAST(COALESCE(${EventArtifactTable.canonical_data}, ${EventTable.data}) AS BLOB)) AS bytes FROM ${EventTable} + LEFT JOIN ${EventArtifactTable} ON ${EventArtifactTable.event_id} = ${EventTable.id} WHERE ${EventTable.aggregate_id} = ${aggregateID} AND ${EventTable.seq} > ${after} ORDER BY ${EventTable.seq} ASC @@ -692,14 +1344,21 @@ export const layerWith = (options?: LayerOptions) => const metadata = candidates .slice(0, AGGREGATE_READ_BATCH_EVENTS) .filter((event) => event.cumulativeBytes <= AGGREGATE_READ_BATCH_BYTES) - if (metadata.length === 0) return { events: [], more: false } - yield* options?.afterAggregateReadMetadata?.( + if (metadata.length === 0) { + yield* layerOptions?.afterAggregateRead?.(aggregateID) ?? Effect.void + return { events: [], more: false } + } + yield* layerOptions?.afterAggregateReadMetadata?.( aggregateID, metadata.map((event) => event.id), ) ?? Effect.void const rows = yield* db - .select({ id: EventTable.id, data: sql`${EventTable.data}` }) + .select({ + id: EventTable.id, + data: sql`COALESCE(${EventArtifactTable.canonical_data}, ${EventTable.data})`, + }) .from(EventTable) + .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) .where(inArray(EventTable.id, metadata.map((event) => event.id))) .all() .pipe(Effect.orDie) @@ -721,6 +1380,7 @@ export const layerWith = (options?: LayerOptions) => data: decodeStoredEventData(dataByID.get(event.id)!) as Record, }), ) + yield* layerOptions?.afterAggregateRead?.(aggregateID) ?? Effect.void return { events, more: candidates.length > metadata.length } }) @@ -813,6 +1473,11 @@ export const layerWith = (options?: LayerOptions) => project, replay, replayAll, + snapshot, + checkpoint, + importSnapshot, + compact, + canonicalizeLegacyArtifacts, remove, claim, }) diff --git a/packages/core/src/event/sql.ts b/packages/core/src/event/sql.ts index 38fe34f1e..35ed94722 100644 --- a/packages/core/src/event/sql.ts +++ b/packages/core/src/event/sql.ts @@ -1,10 +1,12 @@ -import { sqliteTable, text, integer, index, uniqueIndex } from "drizzle-orm/sqlite-core" +import { sqliteTable, text, integer, index, uniqueIndex, primaryKey, blob } from "drizzle-orm/sqlite-core" import type { EventV2 } from "../event" export const EventSequenceTable = sqliteTable("event_sequence", { aggregate_id: text().notNull().primaryKey(), seq: integer().notNull(), owner_id: text(), + retention_floor_seq: integer(), + snapshot_id: text(), }) export const EventTable = sqliteTable( @@ -17,9 +19,108 @@ export const EventTable = sqliteTable( seq: integer().notNull(), type: text().notNull(), data: text({ mode: "json" }).$type>().notNull(), + sync_seq: integer(), }, (table) => [ uniqueIndex("event_aggregate_seq_idx").on(table.aggregate_id, table.seq), index("event_aggregate_type_seq_idx").on(table.aggregate_id, table.type, table.seq), + uniqueIndex("event_sync_seq_idx").on(table.sync_seq), ], ) + +export const EventSnapshotTable = sqliteTable( + "event_snapshot", + { + snapshot_id: text().primaryKey(), + aggregate_id: text() + .notNull() + .references(() => EventSequenceTable.aggregate_id, { onDelete: "cascade" }), + through_seq: integer().notNull(), + sync_seq: integer().notNull(), + codec: text().notNull(), + schema_version: integer().notNull(), + snapshot_hash: text().notNull(), + body: text({ mode: "json" }).$type>().notNull(), + owner_id: text(), + created_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("event_snapshot_aggregate_seq_idx").on(table.aggregate_id, table.through_seq), + index("event_snapshot_aggregate_created_idx").on(table.aggregate_id, table.created_at), + uniqueIndex("event_snapshot_sync_seq_idx").on(table.sync_seq), + ], +) + +export const EventSyncSequenceTable = sqliteTable("event_sync_sequence", { + id: integer().primaryKey(), + seq: integer().notNull(), + generation: text().notNull(), + cursor_secret: text().notNull(), +}) + +export const WorkspaceSyncCursorTable = sqliteTable( + "workspace_sync_cursor", + { + workspace_id: text().notNull(), + remote_fingerprint: text().notNull(), + cursor: text().notNull(), + updated_at: integer().notNull(), + }, + (table) => [primaryKey({ columns: [table.workspace_id, table.remote_fingerprint] })], +) + +export const EventDedupeTable = sqliteTable( + "event_dedupe", + { + aggregate_id: text() + .notNull() + .references(() => EventSequenceTable.aggregate_id, { onDelete: "cascade" }), + seq: integer().notNull(), + event_id: text().$type().notNull(), + type: text().notNull(), + data_hash: text().notNull(), + compacted_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("event_dedupe_aggregate_seq_idx").on(table.aggregate_id, table.seq), + uniqueIndex("event_dedupe_event_idx").on(table.event_id), + ], +) + +export const EventArtifactTable = sqliteTable( + "event_artifact", + { + artifact_id: text().primaryKey(), + event_id: text().$type().notNull(), + aggregate_id: text() + .notNull() + .references(() => EventSequenceTable.aggregate_id, { onDelete: "cascade" }), + seq: integer().notNull(), + kind: text().$type<"legacy_message_diff">().notNull(), + original_data_hash: text().notNull(), + canonical_data_hash: text().notNull(), + canonical_data: text({ mode: "json" }).$type>().notNull(), + body_hash: text().notNull(), + body_bytes: integer().notNull(), + chunk_count: integer().notNull(), + codec_version: integer().notNull(), + created_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("event_artifact_event_idx").on(table.event_id), + uniqueIndex("event_artifact_aggregate_seq_idx").on(table.aggregate_id, table.seq), + ], +) + +export const EventArtifactChunkTable = sqliteTable( + "event_artifact_chunk", + { + artifact_id: text() + .notNull() + .references(() => EventArtifactTable.artifact_id, { onDelete: "cascade" }), + chunk_index: integer().notNull(), + data: blob({ mode: "buffer" }).notNull(), + chunk_hash: text().notNull(), + }, + (table) => [primaryKey({ columns: [table.artifact_id, table.chunk_index] })], +) diff --git a/packages/core/src/npm.ts b/packages/core/src/npm.ts index 31f8ec96a..d77756d61 100644 --- a/packages/core/src/npm.ts +++ b/packages/core/src/npm.ts @@ -83,17 +83,17 @@ export const layer = Layer.effect( const { Arborist } = yield* Effect.promise(() => import("@npmcli/arborist")) const add = input.add ?? [] const npmOptions = yield* NpmConfig.load(input.dir) - const arborist = new Arborist({ - ...npmOptions, - path: input.dir, - binLinks: true, - progress: false, - savePrefix: "", - ignoreScripts: true, - }) return yield* Effect.tryPromise({ - try: () => - arborist.reify({ + try: (signal) => + new Arborist({ + ...npmOptions, + path: input.dir, + binLinks: true, + progress: false, + savePrefix: "", + ignoreScripts: true, + signal, + }).reify({ ...npmOptions, add, save: true, diff --git a/packages/core/src/session/input.ts b/packages/core/src/session/input.ts index d19dfffa1..e697c8408 100644 --- a/packages/core/src/session/input.ts +++ b/packages/core/src/session/input.ts @@ -217,6 +217,7 @@ export const guardReservedID = Effect.fn("SessionInput.guardReservedID")(functio db: DatabaseService, event: EventV2.Payload, ) { + if (event.replayExact) return if ( Schema.is(SessionEvent.PromptLifecycle.Admitted)(event) || Schema.is(SessionEvent.PromptLifecycle.Promoted)(event) diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index 78f22271e..5b85e7e48 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -15,6 +15,7 @@ import { WorkspaceV2 } from "../workspace" import { SessionContextEpoch } from "./context-epoch" import { MessageTable, PartTable, SessionMessageTable, SessionTable } from "./sql" import type { DeepMutable } from "../schema" +import { SessionSchema } from "./schema" type DatabaseService = Database.Interface["db"] @@ -229,6 +230,87 @@ export const layer = Layer.effectDiscard( const events = yield* EventV2.Service const { db } = yield* Database.Service yield* events.beforeCommit((event) => SessionInput.guardReservedID(db, event)) + yield* events.beforeCommit((event) => + Effect.gen(function* () { + if (!event.replay) return + if ( + (Schema.is(SessionV1.Event.Created)(event) || Schema.is(SessionV1.Event.Updated)(event)) && + event.data.info.id !== event.data.sessionID + ) + return yield* Effect.die( + new EventV2.InvalidSyncEventError({ + type: event.type, + message: "Session replay identity does not match its aggregate", + }), + ) + const replaySessionID = + event.replayOwnerID && + typeof event.data === "object" && + event.data !== null && + "sessionID" in event.data && + Schema.is(SessionSchema.ID)(event.data.sessionID) + ? event.data.sessionID + : undefined + if (replaySessionID) { + const authority = yield* db + .select({ workspaceID: SessionTable.workspace_id }) + .from(SessionTable) + .where(eq(SessionTable.id, replaySessionID)) + .get() + .pipe(Effect.orDie) + const freshCreated = + !authority && + Schema.is(SessionV1.Event.Created)(event) && + event.seq === 0 && + event.data.info.id === replaySessionID && + event.data.info.workspaceID === event.replayOwnerID + if ((!authority && !freshCreated) || (authority && authority.workspaceID !== event.replayOwnerID)) + return yield* Effect.die( + new EventV2.InvalidSyncEventError({ + type: event.type, + message: "Session replay owner does not match the current workspace authority", + }), + ) + } + if (Schema.is(SessionEvent.Moved)(event) && !event.replayExact) + return yield* Effect.die( + new EventV2.InvalidSyncEventError({ + type: event.type, + message: "Session placement replay requires a durable transfer operation receipt", + }), + ) + if (!Schema.is(SessionV1.Event.Updated)(event)) return + const current = yield* db + .select({ + projectID: SessionTable.project_id, + directory: SessionTable.directory, + workspaceID: SessionTable.workspace_id, + }) + .from(SessionTable) + .where(eq(SessionTable.id, event.data.sessionID)) + .get() + .pipe(Effect.orDie) + if (!current) + return yield* Effect.die( + new EventV2.InvalidSyncEventError({ + type: event.type, + message: "Session update replay requires an existing projected Session", + }), + ) + if ( + current.projectID === event.data.info.projectID && + current.directory === event.data.info.directory && + (current.workspaceID ?? undefined) === (event.data.info.workspaceID ?? undefined) + ) + return + return yield* Effect.die( + new EventV2.InvalidSyncEventError({ + type: event.type, + message: "Session update replay cannot change project, directory, or workspace placement", + }), + ) + }), + ) yield* events.project(SessionV1.Event.Created, (event) => Effect.gen(function* () { const stored = yield* db diff --git a/packages/core/src/v1/session.ts b/packages/core/src/v1/session.ts index 6c1bb2772..6fbb375a4 100644 --- a/packages/core/src/v1/session.ts +++ b/packages/core/src/v1/session.ts @@ -379,6 +379,13 @@ const DiffManifestDescriptor = Schema.Struct({ truncatedFiles: NonNegativeInt, }).annotate({ identifier: "SnapshotDiffManifestDescriptor" }) +const DiffArtifactDescriptor = Schema.Struct({ + id: Schema.String, + hash: Schema.String, + codec: Schema.Literal("legacy-message-diff.v1"), + fileCount: NonNegativeInt, +}).annotate({ identifier: "SnapshotDiffArtifactDescriptor" }) + export const User = Schema.Struct({ ...messageBase, role: Schema.Literal("user"), @@ -392,6 +399,7 @@ export const User = Schema.Struct({ body: Schema.optional(Schema.String), diffs: Schema.Array(FileDiff), diffManifest: optionalOmitUndefined(DiffManifestDescriptor), + diffArtifact: optionalOmitUndefined(DiffArtifactDescriptor), }), ), agent: Schema.String, diff --git a/packages/core/test/database-migration.test.ts b/packages/core/test/database-migration.test.ts index 171bfbe99..4f6012d51 100644 --- a/packages/core/test/database-migration.test.ts +++ b/packages/core/test/database-migration.test.ts @@ -28,6 +28,7 @@ import compactionContinuationAdmissionMigration from "@deepagent-code/core/datab import partIntegrityBackfillMigration from "@deepagent-code/core/database/migration/20260810170000_part_integrity_backfill" import legacyActivityProgressMigration from "@deepagent-code/core/database/migration/20260811090000_legacy_activity_progress" import legacyActivityOwnerMigration from "@deepagent-code/core/database/migration/20260811100000_legacy_activity_owner" +import eventSnapshotAuthorityMigration from "@deepagent-code/core/database/migration/20260813100000_event_snapshot_authority" import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" import { Database } from "@deepagent-code/core/database/database" import { tmpdir } from "./fixture/tmpdir" @@ -40,6 +41,51 @@ const run = (effect: Effect.Effect) => const makeDb = EffectDrizzleSqlite.makeWithDefaults() describe("DatabaseMigration", () => { + test("rejects duplicate migration IDs before changing the database", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const duplicate = { + id: "duplicate-migration-id", + up: () => Effect.die("duplicate migration must not run"), + } + const result = yield* DatabaseMigration.applyOnly(db, [duplicate, duplicate]).pipe(Effect.exit) + + expect(result).toMatchObject({ _tag: "Failure" }) + expect( + yield* db.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'migration'`), + ).toBeUndefined() + }), + ) + }) + + test("upgrades EventV2 history and allocates sync_seq for old and new writers", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`CREATE TABLE event_sequence (aggregate_id TEXT PRIMARY KEY, seq INTEGER NOT NULL, owner_id TEXT)`) + yield* db.run(sql`CREATE TABLE event (id TEXT PRIMARY KEY, aggregate_id TEXT NOT NULL, seq INTEGER NOT NULL, type TEXT NOT NULL, data TEXT NOT NULL)`) + yield* db.run(sql`INSERT INTO event_sequence VALUES ('session-a', 1, NULL)`) + yield* db.run(sql`INSERT INTO event VALUES ('event-a', 'session-a', 0, 'test.1', '{}')`) + yield* db.run(sql`INSERT INTO event VALUES ('event-b', 'session-a', 1, 'test.1', '{}')`) + + yield* DatabaseMigration.applyOnly(db, [eventSnapshotAuthorityMigration]) + yield* db.run(sql`INSERT INTO event(id, aggregate_id, seq, type, data) VALUES ('event-old-writer', 'session-a', 2, 'test.1', '{}')`) + yield* db.run(sql`UPDATE event_sync_sequence SET seq = seq + 1 WHERE id = 1`) + yield* db.run(sql`INSERT INTO event(id, aggregate_id, seq, type, data, sync_seq) VALUES ('event-new-writer', 'session-a', 3, 'test.1', '{}', (SELECT seq FROM event_sync_sequence WHERE id = 1))`) + yield* db.run(sql`INSERT INTO event(id, aggregate_id, seq, type, data) VALUES ('event-old-writer-again', 'session-a', 4, 'test.1', '{}')`) + + expect(yield* db.all(sql`SELECT id, sync_seq FROM event ORDER BY sync_seq`)).toEqual([ + { id: "event-a", sync_seq: 1 }, + { id: "event-b", sync_seq: 2 }, + { id: "event-old-writer", sync_seq: 3 }, + { id: "event-new-writer", sync_seq: 4 }, + { id: "event-old-writer-again", sync_seq: 5 }, + ]) + expect(yield* db.get(sql`SELECT seq, length(generation) AS generation_length, length(cursor_secret) AS secret_length FROM event_sync_sequence WHERE id = 1`)).toEqual({ seq: 5, generation_length: 32, secret_length: 64 }) + }), + ) + }) test("serializes concurrent embedded initialization for one database path", async () => { await using tmp = await tmpdir() const filename = path.join(tmp.path, "embedded.sqlite") diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index d005ce4e6..5c807ea49 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -2,7 +2,14 @@ import { describe, expect } from "bun:test" import { Cause, DateTime, Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect" import { EventV2 } from "@deepagent-code/core/event" import { Database } from "@deepagent-code/core/database/database" -import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { + EventArtifactChunkTable, + EventArtifactTable, + EventDedupeTable, + EventSequenceTable, + EventSnapshotTable, + EventTable, +} from "@deepagent-code/core/event/sql" import { Location } from "@deepagent-code/core/location" import { AbsolutePath } from "@deepagent-code/core/schema" import { WorkspaceV2 } from "@deepagent-code/core/workspace" @@ -872,6 +879,67 @@ describe("EventV2", () => { }), ) + it.effect("refuses checkpoint and compaction while no canonical projection codec is registered", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const aggregateID = EventV2.ID.create() + yield* events.publish(SyncMessage, { id: aggregateID, text: "zero" }) + const checkpoint = yield* events.checkpoint({ + aggregateID, + throughSeq: EventV2.Cursor.make(0), + expectedLatest: EventV2.Cursor.make(0), + codec: "test.v1", + schemaVersion: 1, + body: { value: "zero" }, + }).pipe(Effect.catchDefect(Effect.succeed)) + expect(checkpoint).toBeInstanceOf(EventV2.InvalidSyncEventError) + const compact = yield* events + .compact({ aggregateID, throughSeq: EventV2.Cursor.make(0), limit: 1 }) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(compact).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect(yield* events.snapshot(aggregateID)).toBeUndefined() + expect(yield* db.select().from(EventDedupeTable).all().pipe(Effect.orDie)).toEqual([]) + expect(yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, aggregateID)).all().pipe(Effect.orDie)).toHaveLength(1) + }), + ) + + it.effect("canonicalizes oversized legacy message diffs into deterministic BLOB chunks", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const aggregateID = EventV2.ID.create() + yield* events.publish(SyncMessage, { id: aggregateID, text: "zero" }) + const id = EventV2.ID.make("evt_legacy_message_diff_artifact") + const patch = "x".repeat(EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1) + yield* db.insert(EventTable).values({ + id, + aggregate_id: aggregateID, + seq: 1, + type: EventV2.versionedType("message.updated", 1), + data: { + sessionID: aggregateID, + info: { summary: { diffs: [{ file: "large.patch", patch, additions: 1, deletions: 0 }] } }, + }, + }).run().pipe(Effect.orDie) + yield* db.update(EventSequenceTable).set({ seq: 1 }).where(eq(EventSequenceTable.aggregate_id, aggregateID)).run().pipe(Effect.orDie) + + const first = yield* events.canonicalizeLegacyArtifacts({ limit: 1, now: 10 }) + expect(first).toEqual({ processed: 1, next: id }) + expect(yield* events.canonicalizeLegacyArtifacts({ afterID: id, limit: 1, now: 11 })).toEqual({ processed: 0 }) + const artifact = yield* db.select().from(EventArtifactTable).where(eq(EventArtifactTable.event_id, id)).get().pipe(Effect.orDie) + expect(artifact?.kind).toBe("legacy_message_diff") + expect(artifact?.body_bytes).toBeGreaterThan(EventV2.MAX_ENCODED_PAYLOAD_BYTES) + expect(artifact?.chunk_count).toBeGreaterThan(1) + const summary = (artifact?.canonical_data.info as { summary: { diffs: Array<{ patch?: string }>; diffArtifact: { hash: string } } }).summary + expect(summary.diffs[0]?.patch).toBeUndefined() + expect(summary.diffArtifact.hash).toBe(artifact!.body_hash) + const chunks = yield* db.select().from(EventArtifactChunkTable).where(eq(EventArtifactChunkTable.artifact_id, artifact!.artifact_id)).all().pipe(Effect.orDie) + expect(chunks).toHaveLength(artifact?.chunk_count ?? 0) + expect(Buffer.concat(chunks.map((chunk) => chunk.data)).length).toBe(artifact!.body_bytes) + }), + ) + it.effect("uses custom sync aggregate field", () => Effect.gen(function* () { const events = yield* EventV2.Service @@ -1127,6 +1195,65 @@ describe("EventV2", () => { }), ) + it.effect("wakes aggregate subscribers when replayAll is interrupted after commit", () => { + const initialRead = Deferred.make() + const committed = Deferred.make() + const releaseCommit = Deferred.make() + return Effect.all({ initialRead, committed, releaseCommit }).pipe( + Effect.flatMap((signals) => { + const database = Database.layerFromPath(":memory:") + const eventLayer = EventV2.layerWith({ + afterAggregateRead: () => Deferred.succeed(signals.initialRead, undefined), + afterReplayAllCommit: () => + Deferred.succeed(signals.committed, undefined).pipe(Effect.andThen(Deferred.await(signals.releaseCommit))), + }).pipe(Layer.provide(database)) + return Effect.gen(function* () { + const events = yield* EventV2.Service + const aggregateID = EventV2.ID.create() + const event = { + id: EventV2.ID.create(), + type: EventV2.versionedType(SyncMessage.type, 1), + seq: 0, + aggregateID, + data: { id: aggregateID, text: "committed before interruption" }, + } + const subscriber = yield* events + .aggregateEvents({ aggregateID }) + .pipe(Stream.take(1), Stream.runCollect, Effect.forkScoped) + yield* Deferred.await(signals.initialRead) + const replay = yield* events.replayAll([event]).pipe(Effect.forkChild) + yield* Deferred.await(signals.committed) + const interruption = yield* Fiber.interrupt(replay).pipe(Effect.forkChild) + yield* Deferred.succeed(signals.releaseCommit, undefined) + yield* Fiber.join(interruption) + + expect(Array.from(yield* Fiber.join(subscriber)).map((item) => item.event.data)).toEqual([event.data]) + }).pipe(Effect.provide(Layer.mergeAll(database, eventLayer))) + }), + ) + }) + + it.effect("publishes replayAll observers once across an exact retry", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const aggregateID = EventV2.ID.create() + const received: EventV2.Payload[] = [] + const event = { + id: EventV2.ID.create(), + type: EventV2.versionedType(SyncMessage.type, 1), + seq: 0, + aggregateID, + data: { id: aggregateID, text: "exact replay" }, + } + yield* events.listen((payload) => Effect.sync(() => received.push(payload))) + + yield* events.replayAll([event], { publish: true }) + yield* events.replayAll([event], { publish: true }) + + expect(received.map((payload) => payload.id)).toEqual([event.id]) + }), + ) + it.effect("claim fences replay owners", () => Effect.gen(function* () { const events = yield* EventV2.Service diff --git a/packages/core/test/legacy-provider-receipt-supersession-migration.test.ts b/packages/core/test/legacy-provider-receipt-supersession-migration.test.ts new file mode 100644 index 000000000..7c655ac69 --- /dev/null +++ b/packages/core/test/legacy-provider-receipt-supersession-migration.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from "bun:test" +import { DatabaseMigration } from "@deepagent-code/core/database/migration" +import { EffectDrizzleSqlite } from "@deepagent-code/effect-drizzle-sqlite" +import { SqliteClient } from "@effect/sql-sqlite-bun" +import { Effect } from "effect" +import migration from "@deepagent-code/core/database/migration/20260813120000_legacy_provider_receipt_supersession" + +describe("legacy provider receipt supersession migration", () => { + test("terminalizes only legacy unknown receipts superseded by a later request", async () => { + await Effect.runPromise( + Effect.gen(function* () { + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + yield* db.run(` + CREATE TABLE session_tool_request_receipt ( + receipt_id TEXT PRIMARY KEY, + request_ordinal INTEGER NOT NULL, + session_id TEXT NOT NULL, + provider_attempt_id TEXT, + provider_state TEXT NOT NULL, + request_error_code TEXT, + created_at INTEGER NOT NULL, + terminal_at INTEGER + ) + `) + yield* db.run(` + CREATE TRIGGER session_tool_request_receipt_provider_transition + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_state != OLD.provider_state + BEGIN + SELECT RAISE(ABORT, 'original transition guard'); + END + `) + yield* db.run(` + INSERT INTO session_tool_request_receipt VALUES + ('superseded', 1, 'session-a', NULL, 'indeterminate_after_crash', 'legacy_dispatch_outcome_unknown', 10, 10), + ('tail', 2, 'session-a', NULL, 'indeterminate_after_crash', 'legacy_dispatch_outcome_unknown', 20, 20), + ('attempt', 1, 'session-b', 'attempt-1', 'indeterminate_after_crash', 'legacy_dispatch_outcome_unknown', 30, 30), + ('attempt-tail', 2, 'session-b', NULL, 'indeterminate_after_crash', 'legacy_dispatch_outcome_unknown', 40, 40), + ('non-legacy', 1, 'session-c', NULL, 'indeterminate_after_crash', 'process_restart_outcome_unknown', 50, 50), + ('non-legacy-tail', 2, 'session-c', NULL, 'indeterminate_after_crash', 'process_restart_outcome_unknown', 60, 60) + `) + + yield* DatabaseMigration.applyOnly(db, [migration]) + + expect( + yield* db.all<{ + receipt_id: string + provider_state: string + request_error_code: string | null + }>(`SELECT receipt_id, provider_state, request_error_code FROM session_tool_request_receipt ORDER BY receipt_id`), + ).toEqual([ + { + receipt_id: "attempt", + provider_state: "indeterminate_after_crash", + request_error_code: "legacy_dispatch_outcome_unknown", + }, + { + receipt_id: "attempt-tail", + provider_state: "indeterminate_after_crash", + request_error_code: "legacy_dispatch_outcome_unknown", + }, + { + receipt_id: "non-legacy", + provider_state: "indeterminate_after_crash", + request_error_code: "process_restart_outcome_unknown", + }, + { + receipt_id: "non-legacy-tail", + provider_state: "indeterminate_after_crash", + request_error_code: "process_restart_outcome_unknown", + }, + { + receipt_id: "superseded", + provider_state: "failed", + request_error_code: "legacy_request_superseded_by_later_request", + }, + { + receipt_id: "tail", + provider_state: "indeterminate_after_crash", + request_error_code: "legacy_dispatch_outcome_unknown", + }, + ]) + + expect( + yield* db + .run("UPDATE session_tool_request_receipt SET provider_state = 'failed' WHERE receipt_id = 'tail'") + .pipe(Effect.exit), + ).toMatchObject({ _tag: "Failure" }) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), Effect.scoped), + ) + }) +}) diff --git a/packages/core/test/npm.test.ts b/packages/core/test/npm.test.ts index 8b2b81d7f..806c8f8ca 100644 --- a/packages/core/test/npm.test.ts +++ b/packages/core/test/npm.test.ts @@ -2,7 +2,7 @@ import fs from "fs/promises" import path from "path" import { describe, expect, test } from "bun:test" import { NodeFileSystem } from "@effect/platform-node" -import { Effect, Layer, Option } from "effect" +import { Deferred, Effect, Fiber, Layer, Option } from "effect" import { FSUtil } from "@deepagent-code/core/fs-util" import { Global } from "@deepagent-code/core/global" import { Npm } from "@deepagent-code/core/npm" @@ -88,4 +88,43 @@ describe("Npm.install", () => { await expect(fs.stat(path.join(tmp.path, "node_modules", "prod-pkg"))).resolves.toBeDefined() await expect(fs.stat(path.join(tmp.path, "node_modules", "dev-pkg"))).rejects.toThrow() }) + + test("aborts registry requests when the install fiber is interrupted", async () => { + await using tmp = await tmpdir() + const started = Deferred.makeUnsafe() + const aborted = Deferred.makeUnsafe() + const server = Bun.serve({ + port: 0, + fetch: (request) => { + Deferred.doneUnsafe(started, Effect.void) + return new Promise((resolve) => { + request.signal.addEventListener( + "abort", + () => { + Deferred.doneUnsafe(aborted, Effect.void) + resolve(new Response(null, { status: 499 })) + }, + { once: true }, + ) + }) + }, + }) + + try { + await writePackage(tmp.path, { name: "abort-fixture" }) + await Bun.write(path.join(tmp.path, ".npmrc"), `registry=${server.url.origin}/\n`) + + await Effect.gen(function* () { + const npm = yield* Npm.Service + const fiber = yield* npm + .install(tmp.path, { add: [{ name: "abort-fixture-package", version: "1.0.0" }] }) + .pipe(Effect.forkScoped) + yield* Deferred.await(started).pipe(Effect.timeout("2 seconds")) + yield* Fiber.interrupt(fiber).pipe(Effect.timeout("2 seconds")) + yield* Deferred.await(aborted).pipe(Effect.timeout("2 seconds")) + }).pipe(Effect.scoped, Effect.provide(npmLayer(path.join(tmp.path, "cache"))), Effect.runPromise) + } finally { + server.stop(true) + } + }) }) diff --git a/packages/core/test/session-create.test.ts b/packages/core/test/session-create.test.ts index 2f55b7a24..3ab4c00f4 100644 --- a/packages/core/test/session-create.test.ts +++ b/packages/core/test/session-create.test.ts @@ -5,7 +5,7 @@ import { AgentV2 } from "@deepagent-code/core/agent" import { asc, eq } from "drizzle-orm" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" -import { EventTable } from "@deepagent-code/core/event/sql" +import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" import { Location } from "@deepagent-code/core/location" import { ModelV2 } from "@deepagent-code/core/model" import { ProjectV2 } from "@deepagent-code/core/project" @@ -311,6 +311,280 @@ describe("SessionV2.create", () => { }), ) + it.effect("rejects replayed Session identity and placement changes without committing partial state", () => + Effect.gen(function* () { + const session = yield* SessionV2.Service + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const created = yield* session.create({ location }) + const event = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, created.id)) + .get() + .pipe(Effect.orDie) + const sequence = yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, created.id)) + .get() + .pipe(Effect.orDie) + expect(event).toBeDefined() + expect(sequence).toBeDefined() + const data = event!.data as { sessionID: string; info: Record } + const beforeSession = yield* db + .select() + .from(SessionTable) + .where(eq(SessionTable.id, created.id)) + .get() + .pipe(Effect.orDie) + const beforeEvents = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, created.id)) + .all() + .pipe(Effect.orDie) + + const exactOwnerDefect = yield* events + .replay( + { + id: event!.id, + aggregateID: created.id, + seq: event!.seq, + type: event!.type, + data: event!.data, + }, + { ownerID: "wrk_other", strictOwner: true }, + ) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(exactOwnerDefect).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect((exactOwnerDefect as EventV2.InvalidSyncEventError).message).toContain("current workspace authority") + expect( + yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, created.id)) + .get() + .pipe(Effect.orDie), + ).toEqual(sequence) + + const childOwnerDefect = yield* events + .replay( + { + id: EventV2.ID.make("evt_replay_session_child_owner"), + aggregateID: created.id, + seq: sequence!.seq + 1, + type: EventV2.versionedType(SessionEvent.AgentSwitched.type, 1), + data: { + sessionID: created.id, + timestamp: 1, + messageID: "msg_replay_session_child_owner", + agent: "build", + }, + }, + { ownerID: "wrk_other", strictOwner: true }, + ) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(childOwnerDefect).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect((childOwnerDefect as EventV2.InvalidSyncEventError).message).toContain("current workspace authority") + expect( + yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, created.id)) + .get() + .pipe(Effect.orDie), + ).toEqual(sequence) + expect( + yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, created.id)) + .all() + .pipe(Effect.orDie), + ).toEqual(beforeEvents) + + yield* db + .insert(ProjectTable) + .values({ + id: ProjectV2.ID.make("prj_replay_other"), + worktree: AbsolutePath.make("/other"), + sandboxes: [], + }) + .run() + .pipe(Effect.orDie) + + const batchAggregateID = SessionV2.ID.make("ses_replay_atomic_batch") + const batchDefect = yield* events + .replayAll([ + { + id: EventV2.ID.make("evt_replay_atomic_created"), + aggregateID: batchAggregateID, + seq: 0, + type: EventV2.versionedType(SessionV1.Event.Created.type, 1), + data: { + ...data, + sessionID: batchAggregateID, + info: { ...data.info, id: batchAggregateID }, + }, + }, + { + id: EventV2.ID.make("evt_replay_atomic_invalid_update"), + aggregateID: batchAggregateID, + seq: 1, + type: EventV2.versionedType(SessionV1.Event.Updated.type, 1), + data: { + ...data, + sessionID: batchAggregateID, + info: { + ...data.info, + id: batchAggregateID, + projectID: ProjectV2.ID.make("prj_replay_other"), + }, + }, + }, + ]) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(batchDefect).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect((batchDefect as EventV2.InvalidSyncEventError).message).toContain("cannot change project") + expect(yield* db.select().from(SessionTable).where(eq(SessionTable.id, batchAggregateID)).get()).toBeUndefined() + expect( + yield* db.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, batchAggregateID)).get(), + ).toBeUndefined() + expect(yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, batchAggregateID)).all()).toEqual([]) + + const invalid = [ + { + id: EventV2.ID.make("evt_replay_session_identity"), + type: EventV2.versionedType(SessionV1.Event.Updated.type, 1), + data: { ...data, info: { ...data.info, id: SessionV2.ID.make("ses_replay_other") } }, + message: "identity does not match", + }, + { + id: EventV2.ID.make("evt_replay_session_project"), + type: EventV2.versionedType(SessionV1.Event.Updated.type, 1), + data: { ...data, info: { ...data.info, projectID: ProjectV2.ID.make("prj_replay_other") } }, + message: "cannot change project", + }, + { + id: EventV2.ID.make("evt_replay_session_moved"), + type: EventV2.versionedType(SessionEvent.Moved.type, 1), + data: { + sessionID: created.id, + timestamp: 1, + location: { directory: "/other" }, + }, + message: "requires a durable transfer operation receipt", + }, + ] + + for (const item of invalid) { + const defect = yield* events + .replay({ + id: item.id, + aggregateID: created.id, + seq: sequence!.seq + 1, + type: item.type, + data: item.data, + }) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(defect).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect((defect as EventV2.InvalidSyncEventError).message).toContain(item.message) + expect( + yield* db.select().from(SessionTable).where(eq(SessionTable.id, created.id)).get().pipe(Effect.orDie), + ).toEqual(beforeSession) + expect( + yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, created.id)) + .get() + .pipe(Effect.orDie), + ).toEqual(sequence) + expect( + yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, created.id)) + .all() + .pipe(Effect.orDie), + ).toEqual(beforeEvents) + } + + const aggregateID = SessionV2.ID.make("ses_replay_created_aggregate") + const projectedID = SessionV2.ID.make("ses_replay_created_projection") + const defect = yield* events + .replay({ + id: EventV2.ID.make("evt_replay_created_identity"), + aggregateID, + seq: 0, + type: EventV2.versionedType(SessionV1.Event.Created.type, 1), + data: { ...data, sessionID: aggregateID, info: { ...data.info, id: projectedID } }, + }) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(defect).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect((defect as EventV2.InvalidSyncEventError).message).toContain("identity does not match") + expect(yield* db.select().from(SessionTable).where(eq(SessionTable.id, projectedID)).get()).toBeUndefined() + expect( + yield* db.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, aggregateID)).get(), + ).toBeUndefined() + expect(yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, aggregateID)).all()).toEqual([]) + + const ownedAggregateID = SessionV2.ID.make("ses_replay_owned_created") + const ownedDefect = yield* events + .replay( + { + id: EventV2.ID.make("evt_replay_owned_created"), + aggregateID: ownedAggregateID, + seq: 0, + type: EventV2.versionedType(SessionV1.Event.Created.type, 1), + data: { + ...data, + sessionID: ownedAggregateID, + info: { ...data.info, id: ownedAggregateID, workspaceID: "wrk_other" }, + }, + }, + { ownerID: "wrk_owner", strictOwner: true }, + ) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(ownedDefect).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect((ownedDefect as EventV2.InvalidSyncEventError).message).toContain("current workspace authority") + expect(yield* db.select().from(SessionTable).where(eq(SessionTable.id, ownedAggregateID)).get()).toBeUndefined() + expect( + yield* db.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, ownedAggregateID)).get(), + ).toBeUndefined() + expect(yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, ownedAggregateID)).all()).toEqual([]) + + const owned = yield* session.create({ + location: Location.Ref.make({ directory: location.directory, workspaceID: WorkspaceV2.ID.make("wrk_owner") }), + }) + const ownedEvent = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, owned.id)) + .get() + .pipe(Effect.orDie) + yield* events.replay( + { + id: ownedEvent!.id, + aggregateID: owned.id, + seq: ownedEvent!.seq, + type: ownedEvent!.type, + data: ownedEvent!.data, + }, + { ownerID: "wrk_owner", strictOwner: true }, + ) + expect( + yield* db + .select({ ownerID: EventSequenceTable.owner_id }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, owned.id)) + .get() + .pipe(Effect.orDie), + ).toEqual({ ownerID: "wrk_owner" }) + }), + ) + it.effect("does not mask unrelated created projector defects", () => Effect.gen(function* () { const session = yield* SessionV2.Service diff --git a/packages/core/test/session-projector.test.ts b/packages/core/test/session-projector.test.ts index 14adf2b2a..1bf2742c4 100644 --- a/packages/core/test/session-projector.test.ts +++ b/packages/core/test/session-projector.test.ts @@ -3,7 +3,7 @@ import { DateTime, Effect, Layer, Schema } from "effect" import { asc, eq } from "drizzle-orm" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" -import { EventTable } from "@deepagent-code/core/event/sql" +import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" import { ModelV2 } from "@deepagent-code/core/model" import { Project } from "@deepagent-code/core/project" import { ProjectTable } from "@deepagent-code/core/project/sql" @@ -21,6 +21,8 @@ import { SessionInput } from "@deepagent-code/core/session/input" import { SessionStore } from "@deepagent-code/core/session/store" import { MessageTable, SessionInputTable, SessionMessageTable, SessionTable } from "@deepagent-code/core/session/sql" import { testEffect } from "./lib/effect" +import { WorkspaceTable } from "@deepagent-code/core/control-plane/workspace.sql" +import { WorkspaceV2 } from "@deepagent-code/core/workspace" const database = Database.layerFromPath(":memory:") const events = EventV2.layer.pipe(Layer.provide(database)) @@ -461,6 +463,117 @@ describe("SessionProjector", () => { }), ) + it.effect("keeps legacy Prompted exact replay idempotent while enforcing workspace owner authority", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + const workspaceID = WorkspaceV2.ID.make("wrk_prompt_exact") + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(WorkspaceTable) + .values({ id: workspaceID, type: "test", project_id: Project.ID.global }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + workspace_id: workspaceID, + slug: "test", + directory: "/project", + title: "test", + version: "test", + }) + .run() + .pipe(Effect.orDie) + const events = yield* EventV2.Service + const eventID = EventV2.ID.make("evt_prompted_exact_replay") + const messageID = SessionMessage.ID.make("msg_prompted_exact_replay") + yield* events.publish( + SessionEvent.Prompted, + { + sessionID, + messageID, + timestamp: created, + prompt: new Prompt({ text: "exact" }), + delivery: "steer", + }, + { id: eventID }, + ) + const stored = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.id, eventID)) + .get() + .pipe(Effect.orDie) + expect(stored).toBeDefined() + const serialized = { + id: stored!.id, + aggregateID: stored!.aggregate_id, + seq: stored!.seq, + type: stored!.type, + data: stored!.data, + } + const beforeEvents = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, sessionID)) + .all() + .pipe(Effect.orDie) + const beforeInputs = yield* db + .select() + .from(SessionInputTable) + .where(eq(SessionInputTable.session_id, sessionID)) + .all() + .pipe(Effect.orDie) + const beforeMessages = yield* db + .select() + .from(SessionMessageTable) + .where(eq(SessionMessageTable.session_id, sessionID)) + .all() + .pipe(Effect.orDie) + + yield* events.replay(serialized) + yield* events.replay(serialized, { ownerID: workspaceID, strictOwner: true }) + + expect( + yield* db + .select({ ownerID: EventSequenceTable.owner_id }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, sessionID)) + .get() + .pipe(Effect.orDie), + ).toEqual({ ownerID: workspaceID }) + expect(yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, sessionID)).all()).toEqual( + beforeEvents, + ) + expect(yield* db.select().from(SessionInputTable).where(eq(SessionInputTable.session_id, sessionID)).all()).toEqual( + beforeInputs, + ) + expect( + yield* db.select().from(SessionMessageTable).where(eq(SessionMessageTable.session_id, sessionID)).all(), + ).toEqual(beforeMessages) + + const wrongOwner = yield* events + .replay(serialized, { ownerID: "wrk_wrong", strictOwner: true }) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(wrongOwner).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect((wrongOwner as EventV2.InvalidSyncEventError).message).toContain("Replay owner mismatch") + expect( + yield* db + .select({ ownerID: EventSequenceTable.owner_id }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, sessionID)) + .get() + .pipe(Effect.orDie), + ).toEqual({ ownerID: workspaceID }) + }), + ) + it.effect("rejects an assistant message ID that conflicts with an admitted inbox row", () => Effect.gen(function* () { const { db } = yield* Database.Service diff --git a/packages/core/test/session-runner-tool-events.test.ts b/packages/core/test/session-runner-tool-events.test.ts index a6f4076df..c69c5d2ad 100644 --- a/packages/core/test/session-runner-tool-events.test.ts +++ b/packages/core/test/session-runner-tool-events.test.ts @@ -33,6 +33,11 @@ const capture = () => { project: () => Effect.void, replay: () => Effect.void, replayAll: () => Effect.succeed(undefined), + snapshot: () => Effect.succeed(undefined), + checkpoint: () => Effect.die("unused"), + importSnapshot: () => Effect.die("unused"), + compact: () => Effect.die("unused"), + canonicalizeLegacyArtifacts: () => Effect.succeed({ processed: 0 }), remove: () => Effect.void, claim: () => Effect.void, }) diff --git a/packages/deepagent-code/src/config/config.ts b/packages/deepagent-code/src/config/config.ts index 711694212..40b4f5071 100644 --- a/packages/deepagent-code/src/config/config.ts +++ b/packages/deepagent-code/src/config/config.ts @@ -776,7 +776,7 @@ export const layer = Layer.effect( : Effect.void, ), Effect.asVoid, - Effect.forkDetach, + Effect.forkScoped, ) deps.push(dep) diff --git a/packages/deepagent-code/src/context-federation/diagnostics.ts b/packages/deepagent-code/src/context-federation/diagnostics.ts index f8ba95e93..a32881a3f 100644 --- a/packages/deepagent-code/src/context-federation/diagnostics.ts +++ b/packages/deepagent-code/src/context-federation/diagnostics.ts @@ -1,6 +1,12 @@ export * as ContextFederationDiagnostics from "./diagnostics" -import { ContextArtifactTable, SessionActivityTable, SessionContextSelectionTable, SessionProviderAttemptResolutionTable, SessionProviderAttemptTable } from "@deepagent-code/core/context-federation/session-sql" +import { + ContextArtifactTable, + SessionActivityTable, + SessionContextSelectionTable, + SessionProviderAttemptResolutionTable, + SessionProviderAttemptTable, +} from "@deepagent-code/core/context-federation/session-sql" import { GraphKind } from "@deepagent-code/core/context-federation/contract" import { GraphQueryStatus } from "@deepagent-code/core/context-federation/federation" import { Sensitivity } from "@deepagent-code/core/context-federation/authorization" @@ -8,7 +14,9 @@ import { SessionProviderAttempt } from "@deepagent-code/core/context-federation/ import { Database } from "@deepagent-code/core/database/database" import { SessionSchema } from "@deepagent-code/core/session/schema" import { SessionV1 } from "@deepagent-code/core/v1/session" -import { and, desc, eq, inArray } from "drizzle-orm" +import { SessionToolRequestResolutionTable } from "@deepagent-code/core/session/sql" +import { SessionToolRequestReceiptTable } from "../session/tool-request-receipt.sql" +import { and, desc, eq, inArray, isNull } from "drizzle-orm" import { Context, Effect, Layer, Schema } from "effect" import { Session } from "../session/session" import { SessionFederatedContext } from "./session-context-runtime" @@ -39,7 +47,10 @@ export class DiagnosticsError extends Schema.TaggedErrorClass( ) {} export interface Interface { - readonly get: (sessionId: SessionSchema.ID, now?: number) => Effect.Effect, DiagnosticsError> + readonly get: ( + sessionId: SessionSchema.ID, + now?: number, + ) => Effect.Effect, DiagnosticsError> readonly resolveAttempt: (input: { readonly session: Session.Info readonly attemptId: string @@ -71,22 +82,31 @@ export const layer = Layer.effect( .limit(50) .all() .pipe(Effect.orDie) - const activityRows = selectionRows.length === 0 - ? [] - : yield* database.db - .select() - .from(SessionActivityTable) - .where(inArray(SessionActivityTable.activity_id, [...new Set(selectionRows.map((row) => row.activity_id))])) - .all() - .pipe(Effect.orDie) - const artifactRows = selectionRows.length === 0 - ? [] - : yield* database.db - .select() - .from(ContextArtifactTable) - .where(inArray(ContextArtifactTable.selection_id, selectionRows.map((row) => row.selection_id))) - .all() - .pipe(Effect.orDie) + const activityRows = + selectionRows.length === 0 + ? [] + : yield* database.db + .select() + .from(SessionActivityTable) + .where( + inArray(SessionActivityTable.activity_id, [...new Set(selectionRows.map((row) => row.activity_id))]), + ) + .all() + .pipe(Effect.orDie) + const artifactRows = + selectionRows.length === 0 + ? [] + : yield* database.db + .select() + .from(ContextArtifactTable) + .where( + inArray( + ContextArtifactTable.selection_id, + selectionRows.map((row) => row.selection_id), + ), + ) + .all() + .pipe(Effect.orDie) const attemptRows = yield* database.db .select() .from(SessionProviderAttemptTable) @@ -95,31 +115,41 @@ export const layer = Layer.effect( .limit(50) .all() .pipe(Effect.orDie) - const resolutionRows = attemptRows.length === 0 - ? [] - : yield* database.db - .select() - .from(SessionProviderAttemptResolutionTable) - .where(inArray(SessionProviderAttemptResolutionTable.attempt_id, attemptRows.map((row) => row.attempt_id))) - .all() - .pipe(Effect.orDie) + const resolutionRows = + attemptRows.length === 0 + ? [] + : yield* database.db + .select() + .from(SessionProviderAttemptResolutionTable) + .where( + inArray( + SessionProviderAttemptResolutionTable.attempt_id, + attemptRows.map((row) => row.attempt_id), + ), + ) + .all() + .pipe(Effect.orDie) const messages = attemptRows.some((row) => row.state === "indeterminate_after_crash") ? yield* sessions.messages({ sessionID: SessionSchema.ID.make(sessionId) }).pipe(Effect.orDie) : [] return dashboard({ sessionId, - selections: selectionRows.map((row) => selectionView({ - row, - activity: activityRows.find((activity) => activity.activity_id === row.activity_id), - artifact: artifactRows.find((artifact) => artifact.selection_id === row.selection_id), - now, - })), - attempts: attemptRows.map((row) => attemptView( - row, - resolutionRows.find((resolution) => resolution.attempt_id === row.attempt_id), - hasTerminalMessage(messages, row.attempt_id), - now, - )), + selections: selectionRows.map((row) => + selectionView({ + row, + activity: activityRows.find((activity) => activity.activity_id === row.activity_id), + artifact: artifactRows.find((artifact) => artifact.selection_id === row.selection_id), + now, + }), + ), + attempts: attemptRows.map((row) => + attemptView( + row, + resolutionRows.find((resolution) => resolution.attempt_id === row.attempt_id), + hasTerminalMessage(messages, row.attempt_id), + now, + ), + ), }) }).pipe(Effect.mapError(diagnosticsError)) @@ -129,6 +159,27 @@ export const layer = Layer.effect( if (!attempt || attempt.sessionId !== input.session.id) { return yield* new DiagnosticsError({ reason: "provider_attempt_not_found" }) } + const legacyReceipt = yield* database.db + .select({ receiptID: SessionToolRequestReceiptTable.receipt_id }) + .from(SessionToolRequestReceiptTable) + .leftJoin( + SessionToolRequestResolutionTable, + eq(SessionToolRequestResolutionTable.receipt_id, SessionToolRequestReceiptTable.receipt_id), + ) + .where( + and( + eq(SessionToolRequestReceiptTable.session_id, input.session.id), + eq(SessionToolRequestReceiptTable.provider_attempt_id, input.attemptId), + eq(SessionToolRequestReceiptTable.provider_state, "indeterminate_after_crash"), + isNull(SessionToolRequestResolutionTable.resolution_id), + ), + ) + .get() + .pipe(Effect.orDie) + if (legacyReceipt) + return yield* new DiagnosticsError({ + reason: "legacy_provider_recovery_required", + }) if (input.decision === "replayed") { const resolved = yield* federation.replayIndeterminate({ session: input.session, @@ -140,11 +191,12 @@ export const layer = Layer.effect( }) return attemptView(resolved.replay, undefined, false, input.now ?? Date.now()) } - const terminal = input.decision === "settled" - ? (yield* sessions.messages({ sessionID: input.session.id }).pipe(Effect.orDie)).find( - (message) => hasTerminalMessage([message], input.attemptId), - ) - : undefined + const terminal = + input.decision === "settled" + ? (yield* sessions.messages({ sessionID: input.session.id }).pipe(Effect.orDie)).find((message) => + hasTerminalMessage([message], input.attemptId), + ) + : undefined if (input.decision === "settled" && !terminal) { return yield* new DiagnosticsError({ reason: "persisted_terminal_event_required" }) } @@ -240,10 +292,10 @@ function selectionView(input: { activityState: input.activity?.state ?? "failed", revision: input.row.revision, summary: statuses.some((status) => status.kind !== "complete") - ? "partial" as const + ? ("partial" as const) : statuses.some((status) => status.outcome === "matched") - ? "complete" as const - : "empty" as const, + ? ("complete" as const) + : ("empty" as const), statuses, evidence: selected.map((item) => ({ token: item.token, @@ -286,19 +338,22 @@ function attemptView( terminalEvidence = false, now = Date.now(), ) { - const value = "attemptId" in attempt ? attempt : { - attemptId: attempt.attempt_id, - activityId: attempt.activity_id, - providerTurnSeq: attempt.provider_turn_seq, - selectionId: attempt.selection_id, - providerId: attempt.provider_id, - parentAttemptId: attempt.parent_attempt_id ?? undefined, - state: attempt.state, - createdAt: attempt.created_at, - firstEventAt: attempt.first_event_at ?? undefined, - settledAt: attempt.settled_at ?? undefined, - errorCode: attempt.error_code ?? undefined, - } + const value = + "attemptId" in attempt + ? attempt + : { + attemptId: attempt.attempt_id, + activityId: attempt.activity_id, + providerTurnSeq: attempt.provider_turn_seq, + selectionId: attempt.selection_id, + providerId: attempt.provider_id, + parentAttemptId: attempt.parent_attempt_id ?? undefined, + state: attempt.state, + createdAt: attempt.created_at, + firstEventAt: attempt.first_event_at ?? undefined, + settledAt: attempt.settled_at ?? undefined, + errorCode: attempt.error_code ?? undefined, + } return { attemptId: value.attemptId, activityId: value.activityId, @@ -333,17 +388,17 @@ function hasTerminalMessage(messages: readonly SessionV1.WithParts[], attemptId: const info = message.info return Boolean( info && - typeof info === "object" && - "role" in info && - info.role === "assistant" && - "providerAttemptID" in info && - info.providerAttemptID === attemptId && - "time" in info && - info.time && - typeof info.time === "object" && - "completed" in info.time && - typeof info.time.completed === "number" && - !("error" in info && info.error), + typeof info === "object" && + "role" in info && + info.role === "assistant" && + "providerAttemptID" in info && + info.providerAttemptID === attemptId && + "time" in info && + info.time && + typeof info.time === "object" && + "completed" in info.time && + typeof info.time.completed === "number" && + !("error" in info && info.error), ) }) } diff --git a/packages/deepagent-code/src/control-plane/workspace.ts b/packages/deepagent-code/src/control-plane/workspace.ts index 3e044f07c..152d6df68 100644 --- a/packages/deepagent-code/src/control-plane/workspace.ts +++ b/packages/deepagent-code/src/control-plane/workspace.ts @@ -1,4 +1,4 @@ -import { Context, Effect, FiberMap, Layer, Schema, Stream } from "effect" +import { Context, Effect, FiberMap, Layer, Option, Schema, Stream } from "effect" import { serviceUse } from "@deepagent-code/core/effect/service-use" import { FetchHttpClient, HttpBody, HttpClient, HttpClientError, HttpClientRequest } from "effect/unstable/http" import { Database } from "@deepagent-code/core/database/database" @@ -11,7 +11,7 @@ import { GlobalBus } from "@/bus/global" import { Auth } from "@/auth" import { EventV2 } from "@deepagent-code/core/event" import { EventV2Bridge } from "@/event-v2-bridge" -import { EventSequenceTable } from "@deepagent-code/core/event/sql" +import { EventSequenceTable, WorkspaceSyncCursorTable } from "@deepagent-code/core/event/sql" import { FSUtil } from "@deepagent-code/core/fs-util" import * as Log from "@deepagent-code/core/util/log" import { RuntimeFlags } from "@/effect/runtime-flags" @@ -31,6 +31,7 @@ import { WorkspaceRef } from "@/effect/instance-ref" import { Vcs } from "@/project/vcs" import { WorkspaceAdapterRuntime } from "./workspace-adapter-runtime" import { SessionRecoveryTransferGuard } from "@/session/recovery-transfer-guard" +import { Hash } from "@deepagent-code/core/util/hash" export const Info = Schema.Struct({ ...WorkspaceInfoSchema.fields, @@ -74,6 +75,7 @@ function fromRow(row: typeof WorkspaceTable.$inferSelect): Info { } const log = Log.create({ service: "workspace-sync" }) +const decodeHttpError = Schema.decodeUnknownOption(Schema.UnknownFromJsonString) export const CreateInput = Schema.Struct({ id: Schema.optional(WorkspaceV2.ID), @@ -301,38 +303,63 @@ export const layer = Layer.effect( url: URL | string, headers: HeadersInit | undefined, ) { - const sessionIDs = (yield* db - .select({ id: SessionTable.id }) - .from(SessionTable) - .where(eq(SessionTable.workspace_id, space.id)) - .all() - .pipe(Effect.orDie)).map((row) => row.id) - const state = sessionIDs.length - ? Object.fromEntries( - (yield* db - .select() - .from(EventSequenceTable) - .where(inArray(EventSequenceTable.aggregate_id, sessionIDs)) - .all() - .pipe(Effect.orDie)).map((row) => [row.aggregate_id, row.seq]), - ) - : {} - + const normalized = new URL(url) + normalized.hash = "" + normalized.search = "" + normalized.pathname = normalized.pathname.replace(/\/$/, "") + const remoteFingerprint = Hash.sha256(`${normalized.toString()}:workspace:${space.id}`) + const receipt = yield* db + .select({ cursor: WorkspaceSyncCursorTable.cursor }) + .from(WorkspaceSyncCursorTable) + .where( + and( + eq(WorkspaceSyncCursorTable.workspace_id, space.id), + eq(WorkspaceSyncCursorTable.remote_fingerprint, remoteFingerprint), + ), + ) + .get() + .pipe(Effect.orDie) log.info("syncing workspace history", { workspaceID: space.id, - sessions: sessionIDs.length, - known: Object.keys(state).length, + resume: Boolean(receipt), }) let total = 0 + let cursor: string | undefined = receipt?.cursor + let reset = Boolean(cursor) while (true) { + const expectedCursor = cursor const response = yield* http.execute( HttpClientRequest.post(route(url, "/sync/history"), { headers: new Headers(headers), - body: HttpBody.jsonUnsafe(state), + body: HttpBody.jsonUnsafe({ version: 1, ...(cursor ? { cursor } : {}) }), }), ) + if (response.status === 409) { + const body = yield* response.text + const error = Option.getOrUndefined(decodeHttpError(body)) + const stale = + error && + typeof error === "object" && + "resource" in error && + error.resource === `sync-cursor-reset:${space.id}` + if (stale && reset && expectedCursor) { + yield* db + .delete(WorkspaceSyncCursorTable) + .where(and(eq(WorkspaceSyncCursorTable.workspace_id, space.id), eq(WorkspaceSyncCursorTable.remote_fingerprint, remoteFingerprint), eq(WorkspaceSyncCursorTable.cursor, expectedCursor))) + .run() + .pipe(Effect.orDie) + cursor = undefined + reset = false + continue + } + return yield* new SyncHttpError({ + message: `Workspace history HTTP failure: ${response.status} ${body}`, + status: response.status, + body, + }) + } if (response.status < 200 || response.status >= 300) { const body = yield* response.text return yield* new SyncHttpError({ @@ -342,29 +369,41 @@ export const layer = Layer.effect( }) } - const history = (yield* response.json) as HistoryEvent[] - if (history.length === 0) break + const decoded = (yield* response.json) as HistoryEnvelope | HistoryEvent[] + const history = Array.isArray(decoded) ? decoded : decoded.items + const nextCursor = Array.isArray(decoded) ? cursor : decoded.nextCursor yield* Effect.forEach( history, - (event) => - events + (item) => { + return events .replay( { - id: EventV2.ID.make(event.id), - aggregateID: event.aggregate_id, - seq: event.seq, - type: event.type, - data: event.data, + id: EventV2.ID.make(item.id), + aggregateID: item.aggregate_id, + seq: item.seq, + type: item.type, + data: item.data, }, { publish: true, ownerID: space.id }, ) .pipe( Effect.provideService(WorkspaceRef, space.id), - Effect.tap(() => Effect.sync(() => (state[event.aggregate_id] = event.seq))), - ), + ) + }, { discard: true }, ) + if (!Array.isArray(decoded)) { + const advanced = expectedCursor + ? yield* db.update(WorkspaceSyncCursorTable).set({ cursor: decoded.nextCursor, updated_at: Date.now() }).where(and(eq(WorkspaceSyncCursorTable.workspace_id, space.id), eq(WorkspaceSyncCursorTable.remote_fingerprint, remoteFingerprint), eq(WorkspaceSyncCursorTable.cursor, expectedCursor))).returning({ cursor: WorkspaceSyncCursorTable.cursor }).get().pipe(Effect.orDie) + : yield* db.insert(WorkspaceSyncCursorTable).values({ workspace_id: space.id, remote_fingerprint: remoteFingerprint, cursor: decoded.nextCursor, updated_at: Date.now() }).onConflictDoNothing().returning({ cursor: WorkspaceSyncCursorTable.cursor }).get().pipe(Effect.orDie) + if (!advanced) + return yield* new SyncHttpError({ message: "Workspace sync cursor changed concurrently", status: 409 }) + } + cursor = nextCursor + reset = false total += history.length + if (history.length === 0) break + if (!Array.isArray(decoded) && decoded.complete) break } log.info("workspace history synced", { @@ -825,6 +864,7 @@ export const defaultLayer = layer.pipe( const TIMEOUT = 5000 type HistoryEvent = { + kind: "event" id: string aggregate_id: string seq: number @@ -832,6 +872,13 @@ type HistoryEvent = { data: Record } +type HistoryEnvelope = { + version: 1 + items: HistoryEvent[] + nextCursor: string + complete: boolean +} + function waitUntilSynced(input: { db: Database.Interface["db"] workspaceID: WorkspaceV2.ID diff --git a/packages/deepagent-code/src/effect/runner.ts b/packages/deepagent-code/src/effect/runner.ts index 40f946192..747e6372d 100644 --- a/packages/deepagent-code/src/effect/runner.ts +++ b/packages/deepagent-code/src/effect/runner.ts @@ -203,7 +203,6 @@ export const make = ( case "Idle": return [Effect.void, st] as const case "Running": - case "Finalizing": return [ Effect.gen(function* () { yield* Fiber.interrupt(st.run.fiber) @@ -212,6 +211,16 @@ export const make = ( }), { _tag: "Idle" } as const, ] as const + case "Finalizing": + // Finalization owns durable terminal settlement. Keep this runner busy until the + // interrupted fiber's onExit has completed that teardown and published Idle. + return [ + Effect.gen(function* () { + yield* Fiber.interrupt(st.run.fiber) + yield* Deferred.fail(st.run.done, new Cancelled()).pipe(Effect.asVoid) + }), + st, + ] as const case "Shell": return [ Effect.gen(function* () { diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts index 50a00a734..1ab71a58e 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts @@ -32,20 +32,44 @@ export const ReplayResponse = Schema.Struct({ export const SessionPayload = Schema.Struct({ sessionID: SessionID, }) -export const HistoryPayload = Schema.Record(Schema.String, NonNegativeInt) +export const HistoryKnown = Schema.Record(Schema.String, NonNegativeInt) +export const HistoryPayload = Schema.Union([ + HistoryKnown, + Schema.Struct({ + version: Schema.Literal(1), + cursor: Schema.optional(Schema.String.check(Schema.isMaxLength(512))), + known: Schema.optional(HistoryKnown), + }), +]) export const HistoryEvent = Schema.Struct({ + kind: Schema.Literal("event"), id: EventV2.ID, aggregate_id: Schema.String, seq: NonNegativeInt, type: Schema.String, data: Schema.Record(Schema.String, Schema.Unknown), }) +export const HistoryEnvelope = Schema.Struct({ + version: Schema.Literal(1), + items: Schema.Array(HistoryEvent), + nextCursor: Schema.String, + complete: Schema.Boolean, +}) +export const ArtifactMaintenancePayload = Schema.Struct({ + cursor: Schema.optional(EventV2.ID), + limit: Schema.optional(NonNegativeInt), +}) +export const ArtifactMaintenanceResponse = Schema.Struct({ + processed: NonNegativeInt, + nextCursor: Schema.optional(EventV2.ID), +}) export const SyncPaths = { start: `${root}/start`, replay: `${root}/replay`, steal: `${root}/steal`, history: `${root}/history`, + artifacts: `${root}/maintenance/artifacts`, } as const export const SyncApi = HttpApi.make("sync") @@ -66,7 +90,7 @@ export const SyncApi = HttpApi.make("sync") query: WorkspaceRoutingQuery, payload: ReplayPayload, success: described(ReplayResponse, "Replayed sync events"), - error: HttpApiError.BadRequest, + error: [HttpApiError.BadRequest, ConflictError], }).annotateMerge( OpenApi.annotations({ identifier: "sync.replay", @@ -77,26 +101,42 @@ export const SyncApi = HttpApi.make("sync") HttpApiEndpoint.post("steal", SyncPaths.steal, { query: WorkspaceRoutingQuery, payload: SessionPayload, - success: described(SessionPayload, "Session stolen into workspace"), + success: described(Schema.Never, "Disabled until durable ownership transfer is available"), error: [HttpApiError.BadRequest, ConflictError], }).annotateMerge( OpenApi.annotations({ identifier: "sync.steal", - summary: "Steal session into workspace", - description: "Update a session to belong to the current workspace through the sync event system.", + summary: "Reject unsupported session ownership transfer", + description: + "This endpoint is disabled until durable transfer admission, source fencing, and canonical owner handoff are available.", + exclude: true, }), ), HttpApiEndpoint.post("history", SyncPaths.history, { query: WorkspaceRoutingQuery, payload: HistoryPayload, - success: described(Schema.Array(HistoryEvent), "Sync events"), - error: [HttpApiError.BadRequest, ServiceUnavailableError], + success: described(HistoryEnvelope, "Bounded sync history page"), + error: [HttpApiError.BadRequest, ConflictError, ServiceUnavailableError], }).annotateMerge( OpenApi.annotations({ identifier: "sync.history.list", summary: "List sync events", description: - "List a bounded page of sync events. Keys are aggregate IDs the client already knows about, values are the last known sequence ID. Oversized legacy events fail closed until maintenance migration externalizes their inline payload.", + "List a bounded page with an opaque global cursor. A bounded legacy aggregate map is accepted only for the first upgrade request. Retention-floor crossings return resync_required with a canonical snapshot.", + }), + ), + HttpApiEndpoint.post("artifacts", SyncPaths.artifacts, { + query: WorkspaceRoutingQuery, + payload: ArtifactMaintenancePayload, + success: described(ArtifactMaintenanceResponse, "Canonicalized legacy EventV2 artifacts"), + error: HttpApiError.BadRequest, + }).annotateMerge( + OpenApi.annotations({ + identifier: "sync.maintenance.artifacts", + summary: "Canonicalize a bounded legacy EventV2 artifact batch", + description: + "Run one authenticated, idempotent maintenance batch and return the durable scan cursor for the next call.", + exclude: true, }), ), ) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts index d8399cd85..6fea763b0 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts @@ -5,37 +5,75 @@ import { SessionV1 } from "@deepagent-code/core/v1/session" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" import { EventV2Bridge } from "@/event-v2-bridge" -import { EventTable } from "@deepagent-code/core/event/sql" +import { + EventArtifactTable, + EventSequenceTable, + EventSyncSequenceTable, + EventTable, +} from "@deepagent-code/core/event/sql" import { asc } from "drizzle-orm" import { and } from "drizzle-orm" import { eq } from "drizzle-orm" import { inArray } from "drizzle-orm" import { lte } from "drizzle-orm" +import { gt } from "drizzle-orm" import { not } from "drizzle-orm" import { or } from "drizzle-orm" import { sql } from "drizzle-orm" import { Effect, Option, Schema, Scope } from "effect" import { HttpApiBuilder, HttpApiError } from "effect/unstable/httpapi" import { InstanceHttpApi } from "../api" -import { HistoryPayload, ReplayPayload, SessionPayload } from "../groups/sync" +import { ArtifactMaintenancePayload, HistoryPayload, ReplayPayload, SessionPayload } from "../groups/sync" import { encodeReplayRequestPrefix } from "@/sync/replay-protocol" import * as Log from "@deepagent-code/core/util/log" import { ServiceUnavailableError } from "../errors" import { ConflictError } from "../errors" +import { SessionTable } from "@deepagent-code/core/session/sql" +import { ProjectTable } from "@deepagent-code/core/project/sql" +import { ProjectV2 } from "@deepagent-code/core/project" +import { SessionID } from "@/session/schema" +import { Flag } from "@deepagent-code/core/flag/flag" +import { WorkspaceAdapterRuntime } from "@/control-plane/workspace-adapter-runtime" +import { createHmac, timingSafeEqual } from "node:crypto" const log = Log.create({ service: "server.sync" }) export const SyncHistoryLimits = { events: 100, - bytes: 4 * 1024 * 1024, + dataBytes: 4 * 1024 * 1024, + wireBytes: 5 * 1024 * 1024 + 64 * 1024, + legacyKnownAggregates: 256, } as const +const HistoryCursor = Schema.Struct({ + version: Schema.Literal(1), + scope: Schema.String, + generation: Schema.String, + syncSeq: Schema.Number, +}) +const decodeHistoryCursor = Schema.decodeUnknownOption(Schema.fromJsonString(HistoryCursor)) +const encodeHistoryCursor = (scope: string, generation: string, secret: string, syncSeq: number) => { + const payload = Buffer.from(JSON.stringify({ version: 1, scope, generation, syncSeq })).toString("base64url") + return `${payload}.${createHmac("sha256", secret).update(payload).digest("base64url")}` +} +const historyCursor = (scope: string, generation: string, secret: string, highWater: number, value: string | undefined) => { + if (!value) return -1 + const [payload, signature, extra] = value.split(".") + if (!payload || !signature || extra) return + const expected = Buffer.from(createHmac("sha256", secret).update(payload).digest("base64url")) + const supplied = Buffer.from(signature) + if (expected.length !== supplied.length || !timingSafeEqual(expected, supplied)) return + const decoded = Option.getOrUndefined(decodeHistoryCursor(Buffer.from(payload, "base64url").toString())) + if (!decoded || decoded.scope !== scope || decoded.generation !== generation) return + if (!Number.isSafeInteger(decoded.syncSeq) || decoded.syncSeq < -1 || decoded.syncSeq > highWater) return + return decoded.syncSeq +} const decodeJson = Schema.decodeUnknownOption(Schema.UnknownFromJsonString) const record = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value) -const projectedEventData = sql`CASE +const projectedEventData = sql`COALESCE(${EventArtifactTable.canonical_data}, CASE WHEN ${EventTable.type} = ${EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1)} AND json_valid(${EventTable.data}) AND json_type(${EventTable.data}, '$.info.summary.diffs') = 'array' - AND length(${EventTable.data}) > ${SyncHistoryLimits.bytes} + AND length(${EventTable.data}) > ${SyncHistoryLimits.dataBytes} THEN json_set(${EventTable.data}, '$.info.summary.diffs', json('[]')) WHEN ${EventTable.type} = ${EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1)} AND json_valid(${EventTable.data}) @@ -53,7 +91,7 @@ const projectedEventData = sql`CASE ), '[]')) ) ELSE ${EventTable.data} -END` +END)` export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handlers) => Effect.gen(function* () { const workspace = yield* Workspace.Service @@ -72,7 +110,115 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl const payload: EventV2.SerializedEvent[] = [...ctx.payload.events] const encoded = encodeReplayRequestPrefix(ctx.payload.directory, payload) if (!encoded.complete || encoded.events.length !== payload.length) return yield* new HttpApiError.BadRequest({}) + const instance = yield* InstanceState.context const source = payload[0].aggregateID + const ownerID = yield* InstanceState.workspaceID + if (!ownerID) return yield* new HttpApiError.BadRequest({}) + const selected = ownerID ? yield* workspace.get(ownerID) : undefined + if (selected && Flag.DEEPAGENT_CODE_WORKSPACE_ID !== ownerID) { + const target = yield* WorkspaceAdapterRuntime.target(selected) + if (target.type === "remote") + return yield* new ConflictError({ + message: "Remote replay requires a durable transfer operation receipt and a compatible target", + resource: `workspace:${ownerID}`, + }) + } + const current = yield* db + .select({ + projectID: SessionTable.project_id, + directory: SessionTable.directory, + workspaceID: SessionTable.workspace_id, + }) + .from(SessionTable) + .where(eq(SessionTable.id, SessionID.make(source))) + .get() + .pipe(Effect.orDie) + if (!current && ownerID) { + const first = payload[0] + const info = record(first.data.info) ? first.data.info : undefined + const project = info && typeof info.projectID === "string" + ? yield* db.select({ id: ProjectTable.id }).from(ProjectTable).where(eq(ProjectTable.id, ProjectV2.ID.make(info.projectID))).get().pipe(Effect.orDie) + : undefined + if ( + first.seq !== 0 || + first.type !== EventV2.versionedType(SessionV1.Event.Created.type, 1) || + first.aggregateID !== source || + first.data.sessionID !== source || + info?.id !== source || + info.workspaceID !== ownerID || + ctx.payload.directory !== instance.directory || + info.directory !== instance.directory || + info.projectID !== instance.project.id || + !project + ) + return yield* new ConflictError({ + message: "Owned Session creation replay does not match workspace, project, or routed directory authority", + resource: `session:${source}`, + }) + } + if (current && ownerID && current.workspaceID !== ownerID) + return yield* new ConflictError({ + message: "Session replay owner does not match the current workspace authority", + resource: `session:${source}`, + }) + if ( + current && + (ctx.payload.directory !== instance.directory || + current.directory !== instance.directory || + current.projectID !== instance.project.id) + ) + return yield* new ConflictError({ + message: "Session replay projection does not match the routed project and directory authority", + resource: `session:${source}`, + }) + if (!current && payload[0].type !== EventV2.versionedType(SessionV1.Event.Created.type, 1)) + return yield* new ConflictError({ + message: "Session replay requires an existing projection or its canonical creation event", + resource: `session:${source}`, + }) + const placementConflict = payload.reduce< + { projectID: string; directory: string; workspaceID?: string } | "conflict" | undefined + >((placement, event) => { + if (placement === "conflict") return placement + if (event.type === EventV2.versionedType("session.next.moved", 1)) return "conflict" + if ( + event.type !== EventV2.versionedType(SessionV1.Event.Created.type, 1) && + event.type !== EventV2.versionedType(SessionV1.Event.Updated.type, 1) + ) + return placement + const info = record(event.data.info) ? event.data.info : undefined + if ( + !info || + info.id !== source || + typeof info.projectID !== "string" || + typeof info.directory !== "string" + ) + return "conflict" + const next = { + projectID: info.projectID, + directory: info.directory, + ...(typeof info.workspaceID === "string" ? { workspaceID: info.workspaceID } : {}), + } + if ( + placement && + (placement.projectID !== next.projectID || + placement.directory !== next.directory || + placement.workspaceID !== next.workspaceID) + ) + return "conflict" + return placement ?? next + }, current + ? { + projectID: current.projectID, + directory: current.directory, + ...(current.workspaceID ? { workspaceID: current.workspaceID } : {}), + } + : undefined) + if (placementConflict === "conflict") + return yield* new ConflictError({ + message: "Session placement replay requires a durable transfer operation receipt", + resource: `session:${source}`, + }) log.info("sync replay requested", { sessionID: source, events: payload.length, @@ -80,7 +226,6 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl last: payload.at(-1)?.seq, directory: ctx.payload.directory, }) - const ownerID = yield* InstanceState.workspaceID yield* events.replayAll(payload, { ownerID, strictOwner: true }) log.info("sync replay complete", { sessionID: source, @@ -101,52 +246,90 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl }) const history = Effect.fn("SyncHttpApi.history")(function* (ctx: { payload: typeof HistoryPayload.Type }) { - const exclude = Object.entries(ctx.payload) - const where = - exclude.length > 0 - ? not(or(...exclude.map(([id, seq]) => and(eq(EventTable.aggregate_id, id), lte(EventTable.seq, seq))))!) - : undefined - const candidates = yield* db - .select({ - id: EventTable.id, - bytes: sql`length(CAST(${EventTable.data} AS BLOB))`, + const workspaceID = yield* InstanceState.workspaceID + if (!workspaceID) return yield* new HttpApiError.BadRequest({}) + const authority = yield* db + .select({ generation: EventSyncSequenceTable.generation, secret: EventSyncSequenceTable.cursor_secret, seq: EventSyncSequenceTable.seq }) + .from(EventSyncSequenceTable) + .where(eq(EventSyncSequenceTable.id, 1)) + .get() + .pipe(Effect.orDie) + if (!authority) + return yield* new ServiceUnavailableError({ + service: "sync.history", + message: "Sync cursor authority is unavailable", }) - .from(EventTable) - .where(where) - .orderBy(asc(EventTable.seq), asc(EventTable.aggregate_id)) - .limit(SyncHistoryLimits.events) - .all() + const envelope = "version" in ctx.payload ? ctx.payload : { version: 1 as const, known: ctx.payload } + const known = envelope.known ?? {} + if (Object.keys(known).length > SyncHistoryLimits.legacyKnownAggregates) + return yield* new HttpApiError.BadRequest({}) + const after = historyCursor(workspaceID, authority.generation, authority.secret, authority.seq, typeof envelope.cursor === "string" ? envelope.cursor : undefined) + if (after === undefined) + return yield* new ConflictError({ + message: "Sync history cursor is invalid, stale, or belongs to another workspace", + resource: `sync-cursor-reset:${workspaceID}`, + }) + const knownJson = JSON.stringify(known) + const candidates = yield* db + .all<{ kind: "event"; id: string; syncSeq: number; bytes: number }>(sql` + SELECT + 'event' AS kind, + ${EventTable.id} AS id, + ${EventTable.sync_seq} AS syncSeq, + length(CAST(COALESCE(${EventArtifactTable.canonical_data}, ${EventTable.data}) AS BLOB)) AS bytes + FROM ${EventTable} + INNER JOIN ${EventSequenceTable} + ON ${EventSequenceTable.aggregate_id} = ${EventTable.aggregate_id} + INNER JOIN ${SessionTable} + ON ${SessionTable.id} = ${EventTable.aggregate_id} + LEFT JOIN ${EventArtifactTable} + ON ${EventArtifactTable.event_id} = ${EventTable.id} + WHERE ${EventTable.sync_seq} > ${after} + AND ${EventTable.sync_seq} <= ${authority.seq} + AND ${SessionTable.workspace_id} = ${workspaceID} + AND ${EventTable.seq} > COALESCE(${EventSequenceTable.retention_floor_seq}, -1) + AND ${EventTable.seq} > COALESCE(( + SELECT CAST(value AS INTEGER) + FROM json_each(${knownJson}) + WHERE key = ${EventTable.aggregate_id} + ), -1) + ORDER BY ${EventTable.sync_seq} ASC + LIMIT ${SyncHistoryLimits.events + 1} + `) .pipe(Effect.orDie) const page = candidates.reduce<{ - ids: EventV2.ID[] + rows: typeof candidates bytes: number oversized: string | undefined closed: boolean }>( (result, row) => { if (result.closed) return result - if (row.bytes > SyncHistoryLimits.bytes) + if (row.bytes > SyncHistoryLimits.dataBytes) return { ...result, - oversized: result.ids.length === 0 ? row.id : undefined, + oversized: result.rows.length === 0 ? row.id : undefined, closed: true, } - if (result.bytes > 0 && result.bytes + row.bytes > SyncHistoryLimits.bytes) return { ...result, closed: true } + if (result.bytes > 0 && result.bytes + row.bytes > SyncHistoryLimits.dataBytes) return { ...result, closed: true } return { - ids: [...result.ids, row.id], + rows: [...result.rows, row], bytes: result.bytes + row.bytes, oversized: undefined, closed: false, } }, - { ids: [], bytes: 0, oversized: undefined, closed: false }, + { rows: [], bytes: 0, oversized: undefined, closed: false }, ) if (page.oversized) return yield* new ServiceUnavailableError({ service: "sync.history", message: `Legacy sync event ${page.oversized} exceeds the bounded history page and requires artifact migration`, }) - if (page.ids.length === 0) return [] + if (page.rows.length === 0) + return { version: 1 as const, items: [], nextCursor: encodeHistoryCursor(workspaceID, authority.generation, authority.secret, authority.seq), complete: true } + const selectedRows = page.rows.slice(0, SyncHistoryLimits.events) + const eventIDs = selectedRows.filter((row) => row.kind === "event").map((row) => EventV2.ID.make(row.id)) const rows = yield* db .select({ id: EventTable.id, @@ -156,19 +339,28 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl data: projectedEventData, }) .from(EventTable) - .where(inArray(EventTable.id, page.ids)) - .orderBy(asc(EventTable.seq), asc(EventTable.aggregate_id)) + .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) + .where(inArray(EventTable.id, eventIDs)) .all() .pipe(Effect.orDie) - const result = rows.map((row) => { + const eventByID = new Map(rows.map((row) => { const data = Option.getOrUndefined(decodeJson(row.data)) - return { ...row, data: record(data) ? data : {} } + return [row.id, { kind: "event" as const, ...row, data: record(data) ? data : {} }] + })) + const result = selectedRows.flatMap((row) => { + const item = eventByID.get(EventV2.ID.make(row.id)) + return item ? [item] : [] }) + if (result.length !== selectedRows.length) + return yield* new ServiceUnavailableError({ + service: "sync.history", + message: "Sync history changed while hydrating the selected page", + }) const bounded = result.reduce<{ items: typeof result; bytes: number; closed: boolean }>( (page, item) => { if (page.closed) return page const bytes = Buffer.byteLength(JSON.stringify(item)) + (page.items.length > 0 ? 1 : 0) - if (page.bytes + bytes + 2 > SyncHistoryLimits.bytes) return { ...page, closed: true } + if (page.bytes + bytes + 2 > SyncHistoryLimits.wireBytes) return { ...page, closed: true } return { items: [...page.items, item], bytes: page.bytes + bytes, closed: false } }, { items: [], bytes: 0, closed: false }, @@ -176,11 +368,35 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl if (bounded.items.length === 0 && result.length > 0) return yield* new ServiceUnavailableError({ service: "sync.history", - message: `Projected sync event ${result[0]!.id} exceeds the bounded history page`, + message: `Projected sync item ${page.rows[0]!.id} exceeds the bounded history page`, }) - return bounded.items + const complete = candidates.length <= selectedRows.length && bounded.items.length === selectedRows.length + const next = complete ? authority.seq : selectedRows[bounded.items.length - 1]?.syncSeq ?? after + return { + version: 1 as const, + items: bounded.items, + nextCursor: encodeHistoryCursor(workspaceID, authority.generation, authority.secret, next), + complete, + } + }) + + const artifacts = Effect.fn("SyncHttpApi.artifacts")(function* (ctx: { + payload: typeof ArtifactMaintenancePayload.Type + }) { + if (ctx.payload.limit !== undefined && ctx.payload.limit > EventV2.LEGACY_ARTIFACT_BATCH_EVENTS) + return yield* new HttpApiError.BadRequest({}) + const result = yield* events.canonicalizeLegacyArtifacts({ + ...(ctx.payload.cursor ? { afterID: ctx.payload.cursor } : {}), + ...(ctx.payload.limit ? { limit: ctx.payload.limit } : {}), + }) + return { processed: result.processed, ...(result.next ? { nextCursor: result.next } : {}) } }) - return handlers.handle("start", start).handle("replay", replay).handle("steal", steal).handle("history", history) + return handlers + .handle("start", start) + .handle("replay", replay) + .handle("steal", steal) + .handle("history", history) + .handle("artifacts", artifacts) }), ) diff --git a/packages/deepagent-code/src/server/shared/workspace-routing.ts b/packages/deepagent-code/src/server/shared/workspace-routing.ts index 0edfb2b30..8118f6362 100644 --- a/packages/deepagent-code/src/server/shared/workspace-routing.ts +++ b/packages/deepagent-code/src/server/shared/workspace-routing.ts @@ -4,6 +4,8 @@ type Rule = { method?: string; path: string; exact?: boolean; action: "local" | const RULES: Array = [ { path: "/experimental/workspace", action: "local" }, + { method: "POST", path: "/sync/replay", exact: true, action: "local" }, + { method: "POST", path: "/sync/steal", exact: true, action: "local" }, { path: "/session/status", action: "forward" }, { method: "GET", path: "/session", action: "local" }, ] diff --git a/packages/deepagent-code/src/session/legacy-provider-resolution.ts b/packages/deepagent-code/src/session/legacy-provider-resolution.ts index a92a862ee..8d9e551ca 100644 --- a/packages/deepagent-code/src/session/legacy-provider-resolution.ts +++ b/packages/deepagent-code/src/session/legacy-provider-resolution.ts @@ -25,6 +25,12 @@ import { MessageID, SessionID } from "./schema" import { SessionToolRequestReceiptTable } from "./tool-request-receipt.sql" import { EventV2Bridge } from "../event-v2-bridge" import { WorldStateSlot } from "@deepagent-code/core/deepagent/context/world-state" +import { + SessionActivityTable, + SessionProviderAttemptRecoveryBridgeTable, + SessionProviderAttemptResolutionTable, + SessionProviderAttemptTable, +} from "@deepagent-code/core/context-federation/session-sql" import { renderSessionWorldStateBaseline, sessionWorldStateBaselineHash, @@ -53,24 +59,63 @@ export const ResolveInput = Schema.Struct({ riskAcknowledged: Schema.optional(Schema.Boolean), }) -export const Descriptor = Schema.Struct({ +export const UnsupportedReason = Schema.Literals([ + "legacy_receipt_authority_incomplete", + "source_prompt_epoch_missing", + "source_prompt_epoch_not_recovery_required", + "source_prompt_epoch_binding_mismatch", + "provider_attempt_authority_incomplete", + "provider_attempt_resolution_requires_reconciliation", + "compaction_continuation_requires_maintenance", + "workspace_recovery_requires_coordination", + "source_world_state_baseline_missing", + "source_world_state_baseline_invalid", +]) + +const DescriptorBase = { receiptID: Schema.String, sessionID: SessionID, - assistantMessageID: MessageID, providerID: Schema.String, modelID: Schema.String, providerState: Schema.Literal("indeterminate_after_crash"), + sessionMutationEpoch: NonNegativeInt, +} as const + +const ResolvableDescriptor = Schema.Struct({ + ...DescriptorBase, + resolutionSupported: Schema.Literal(true), + unsupportedReasons: Schema.Tuple([]), + assistantMessageID: MessageID, promptEpoch: NonNegativeInt, promptWindowID: Schema.String, historyHash: Schema.String, requestHash: Schema.String, - sessionMutationEpoch: NonNegativeInt, + continuationRecoverySupported: Schema.Boolean, + workspaceRecoverySupported: Schema.Boolean, + sourceWorldStateBaselineStatus: Schema.Literal("available"), + worldStateBaselineHash: Schema.String, +}) + +const MaintenanceDescriptor = Schema.Struct({ + ...DescriptorBase, + resolutionSupported: Schema.Literal(false), + unsupportedReasons: Schema.NonEmptyArray(UnsupportedReason), + assistantMessageID: Schema.optional(MessageID), + promptEpoch: Schema.optional(NonNegativeInt), + promptWindowID: Schema.optional(Schema.String), + historyHash: Schema.optional(Schema.String), + requestHash: Schema.optional(Schema.String), continuationRecoverySupported: Schema.Boolean, workspaceRecoverySupported: Schema.Boolean, sourceWorldStateBaselineStatus: Schema.Literals(["available", "missing", "invalid"]), worldStateBaselineHash: Schema.optional(Schema.String), }) +export const Descriptor = Schema.Union([ResolvableDescriptor, MaintenanceDescriptor]).annotate({ + discriminator: "resolutionSupported", + identifier: "SessionLegacyProviderResolution.Descriptor", +}) + export const Resolution = Schema.Struct({ resolutionID: Schema.String, commandID: Schema.String, @@ -184,17 +229,31 @@ const service = Effect.gen(function* () { .pipe(Effect.orDie) if (!session) return yield* new NotFound({ reason: "session was not found" }) const rows = yield* db - .select({ receipt: SessionToolRequestReceiptTable }) + .select({ + receipt: SessionToolRequestReceiptTable, + providerAttemptID: SessionProviderAttemptTable.attempt_id, + providerAttemptSessionID: SessionProviderAttemptTable.session_id, + providerAttemptProviderID: SessionProviderAttemptTable.provider_id, + providerAttemptState: SessionProviderAttemptTable.state, + providerAttemptResolutionID: SessionProviderAttemptResolutionTable.resolution_id, + }) .from(SessionToolRequestReceiptTable) .leftJoin( SessionToolRequestResolutionTable, eq(SessionToolRequestResolutionTable.receipt_id, SessionToolRequestReceiptTable.receipt_id), ) + .leftJoin( + SessionProviderAttemptTable, + eq(SessionProviderAttemptTable.attempt_id, SessionToolRequestReceiptTable.provider_attempt_id), + ) + .leftJoin( + SessionProviderAttemptResolutionTable, + eq(SessionProviderAttemptResolutionTable.attempt_id, SessionToolRequestReceiptTable.provider_attempt_id), + ) .where( and( eq(SessionToolRequestReceiptTable.session_id, sessionID), eq(SessionToolRequestReceiptTable.provider_state, "indeterminate_after_crash"), - isNull(SessionToolRequestReceiptTable.provider_attempt_id), isNull(SessionToolRequestResolutionTable.resolution_id), ), ) @@ -209,14 +268,6 @@ const service = Effect.gen(function* () { ) const promptEpoch = receipt.prompt_epoch !== null && receipt.prompt_epoch >= 0 ? receipt.prompt_epoch : undefined - if ( - !assistantMessageID || - promptEpoch === undefined || - !receipt.prompt_window_id || - !receipt.effective_history_hash || - !request - ) - return const continuation = yield* db .select({ runID: CompactionRunTable.run_id }) .from(CompactionRunTable) @@ -228,28 +279,39 @@ const service = Effect.gen(function* () { ) .get() .pipe(Effect.orDie) - const epoch = yield* db - .select({ baselineHash: SessionPromptEpochTable.world_state_baseline_hash }) - .from(SessionPromptEpochTable) - .where( - and( - eq(SessionPromptEpochTable.session_id, sessionID), - eq(SessionPromptEpochTable.epoch, promptEpoch), - ), - ) - .get() - .pipe(Effect.orDie) - const baseline = yield* db - .select() - .from(SessionWorldStateBaselineTable) - .where( - and( - eq(SessionWorldStateBaselineTable.session_id, sessionID), - eq(SessionWorldStateBaselineTable.prompt_epoch, promptEpoch), - ), - ) - .all() - .pipe(Effect.orDie) + const epoch = + promptEpoch === undefined + ? undefined + : yield* db + .select({ + baselineHash: SessionPromptEpochTable.world_state_baseline_hash, + state: SessionPromptEpochTable.state, + authorityState: SessionPromptEpochTable.authority_state, + windowID: SessionPromptEpochTable.window_id, + }) + .from(SessionPromptEpochTable) + .where( + and( + eq(SessionPromptEpochTable.session_id, sessionID), + eq(SessionPromptEpochTable.epoch, promptEpoch), + ), + ) + .get() + .pipe(Effect.orDie) + const baseline = + promptEpoch === undefined + ? [] + : yield* db + .select() + .from(SessionWorldStateBaselineTable) + .where( + and( + eq(SessionWorldStateBaselineTable.session_id, sessionID), + eq(SessionWorldStateBaselineTable.prompt_epoch, promptEpoch), + ), + ) + .all() + .pipe(Effect.orDie) const sections = baseline.flatMap((row) => { const snapshot = Option.getOrUndefined(Schema.decodeUnknownOption(WorldStateSlot)(row.snapshot)) if (!snapshot) return [] @@ -265,27 +327,84 @@ const service = Effect.gen(function* () { : computed !== epoch.baselineHash ? ("invalid" as const) : ("available" as const) - return { + const incomplete = + !assistantMessageID || + promptEpoch === undefined || + !receipt.prompt_window_id || + !receipt.effective_history_hash || + !request + const providerAttemptIncomplete = + receipt.provider_attempt_id !== null && + (item.providerAttemptID !== receipt.provider_attempt_id || + item.providerAttemptSessionID !== sessionID || + item.providerAttemptProviderID !== receipt.provider_id || + item.providerAttemptState !== "indeterminate_after_crash") + const reasons: (typeof UnsupportedReason.Type)[] = [ + ...(incomplete ? (["legacy_receipt_authority_incomplete"] as const) : []), + ...(!incomplete && !epoch ? (["source_prompt_epoch_missing"] as const) : []), + ...(!incomplete && epoch && (epoch.state !== "active" || epoch.authorityState !== "recovery_required") + ? (["source_prompt_epoch_not_recovery_required"] as const) + : []), + ...(!incomplete && epoch && epoch.windowID !== receipt.prompt_window_id + ? (["source_prompt_epoch_binding_mismatch"] as const) + : []), + ...(providerAttemptIncomplete ? (["provider_attempt_authority_incomplete"] as const) : []), + ...(item.providerAttemptResolutionID + ? (["provider_attempt_resolution_requires_reconciliation"] as const) + : []), + ...(continuation ? (["compaction_continuation_requires_maintenance"] as const) : []), + ...(session.workspaceID !== null ? (["workspace_recovery_requires_coordination"] as const) : []), + ...(baselineStatus === "missing" ? (["source_world_state_baseline_missing"] as const) : []), + ...(baselineStatus === "invalid" ? (["source_world_state_baseline_invalid"] as const) : []), + ] + const common = { receiptID: receipt.receipt_id, sessionID, - assistantMessageID, providerID: receipt.provider_id, modelID: receipt.model_id, providerState: "indeterminate_after_crash" as const, - promptEpoch, - promptWindowID: receipt.prompt_window_id, - historyHash: receipt.effective_history_hash, - requestHash: request, sessionMutationEpoch: session.mutationEpoch, continuationRecoverySupported: !continuation, workspaceRecoverySupported: session.workspaceID === null, sourceWorldStateBaselineStatus: baselineStatus, + ...(assistantMessageID ? { assistantMessageID } : {}), + ...(promptEpoch !== undefined ? { promptEpoch } : {}), + ...(receipt.prompt_window_id ? { promptWindowID: receipt.prompt_window_id } : {}), + ...(receipt.effective_history_hash ? { historyHash: receipt.effective_history_hash } : {}), + ...(request ? { requestHash: request } : {}), ...(baselineStatus === "available" && epoch?.baselineHash ? { worldStateBaselineHash: epoch.baselineHash } : {}), } + if (reasons.length > 0) + return { + ...common, + resolutionSupported: false as const, + unsupportedReasons: reasons as [typeof UnsupportedReason.Type, ...(typeof UnsupportedReason.Type)[]], + } + if ( + !assistantMessageID || + promptEpoch === undefined || + !receipt.prompt_window_id || + !receipt.effective_history_hash || + !request || + !epoch?.baselineHash + ) + return yield* Effect.die(new Error(`provider recovery descriptor invariant failed: ${receipt.receipt_id}`)) + return { + ...common, + resolutionSupported: true as const, + unsupportedReasons: [] as const, + assistantMessageID, + promptEpoch, + promptWindowID: receipt.prompt_window_id, + historyHash: receipt.effective_history_hash, + requestHash: request, + sourceWorldStateBaselineStatus: "available" as const, + worldStateBaselineHash: epoch.baselineHash, + } }), - ).pipe(Effect.map((items) => items.filter((item): item is NonNullable => item !== undefined))) + ) }) const resolve: Interface["resolve"] = Effect.fn("SessionLegacyProviderResolution.resolve")(function* (input) { @@ -295,11 +414,77 @@ const service = Effect.gen(function* () { reason: "the Incident Hotfix only supports abandoned", }) const hash = requestHash(input) + const admission = yield* Effect.exit( + db.transaction( + (tx) => + Effect.gen(function* () { + const receipt = yield* tx + .select({ receiptID: SessionToolRequestReceiptTable.receipt_id }) + .from(SessionToolRequestReceiptTable) + .where( + and( + eq(SessionToolRequestReceiptTable.receipt_id, input.receiptID), + eq(SessionToolRequestReceiptTable.session_id, input.sessionID), + ), + ) + .get() + if (!receipt) return yield* new NotFound({ reason: "provider receipt was not found" }) + const inserted = yield* tx + .insert(SessionToolRequestResolutionCommandTable) + .values({ + command_id: input.commandID, + request_hash: hash, + session_id: input.sessionID, + receipt_id: input.receiptID, + result_resolution_id: null, + created_at: Date.now(), + }) + .onConflictDoNothing() + .returning() + .get() + const command = + inserted ?? + (yield* tx + .select() + .from(SessionToolRequestResolutionCommandTable) + .where(eq(SessionToolRequestResolutionCommandTable.command_id, input.commandID)) + .get()) + if ( + !command || + command.request_hash !== hash || + command.session_id !== input.sessionID || + command.receipt_id !== input.receiptID + ) + return yield* new Conflict({ + code: "command_id_conflict", + reason: "command ID was reused with different recovery input", + }) + return command + }), + { behavior: "immediate" }, + ), + ) + if (Exit.isFailure(admission)) { + const failure = Cause.squash(admission.cause) + if (failure instanceof Conflict || failure instanceof NotFound) return yield* failure + return yield* Effect.die(failure) + } + if (admission.value.result_resolution_id) { + const result = yield* db + .select() + .from(SessionToolRequestResolutionTable) + .where(eq(SessionToolRequestResolutionTable.resolution_id, admission.value.result_resolution_id)) + .get() + .pipe(Effect.orDie) + if (!result) + return yield* Effect.die(new Error(`resolution result disappeared: ${admission.value.result_resolution_id}`)) + return fromResolution(input.commandID, result) + } const proposed = { resolutionID: resolutionID(input), successorPromptEpoch: input.expected.promptEpoch + 1, successorMutationEpoch: input.expected.sessionMutationEpoch + 1, - createdAt: Date.now(), + createdAt: admission.value.created_at, } const transaction = yield* Effect.exit( db.transaction( @@ -441,7 +626,6 @@ const service = Effect.gen(function* () { if ( !source || source.authority_state !== "recovery_required" || - receipt.provider_attempt_id !== null || receipt.provider_state !== input.expected.providerState || receipt.prompt_epoch !== input.expected.promptEpoch || receipt.prompt_window_id !== source.window_id || @@ -536,6 +720,36 @@ const service = Effect.gen(function* () { const safeHistoryHash = HistoryAuthority.hash(safeMessages) const successorWindowID = HistoryAuthority.windowID() const highWater = MessageID.make(chronological.at(-1)?.info.id ?? receipt.assistant_message_id) + const providerAttempt = receipt.provider_attempt_id + ? yield* tx + .select() + .from(SessionProviderAttemptTable) + .where(eq(SessionProviderAttemptTable.attempt_id, receipt.provider_attempt_id)) + .get() + : undefined + if ( + receipt.provider_attempt_id && + (!providerAttempt || + providerAttempt.session_id !== input.sessionID || + providerAttempt.provider_id !== receipt.provider_id || + providerAttempt.state !== "indeterminate_after_crash") + ) + return yield* new Conflict({ + code: "stale_recovery_state", + reason: "provider attempt recovery authority changed", + }) + const providerAttemptResolution = providerAttempt + ? yield* tx + .select() + .from(SessionProviderAttemptResolutionTable) + .where(eq(SessionProviderAttemptResolutionTable.attempt_id, providerAttempt.attempt_id)) + .get() + : undefined + if (providerAttemptResolution) + return yield* new Conflict({ + code: "stale_recovery_state", + reason: "provider attempt was independently resolved and requires Maintenance reconciliation", + }) const activity = yield* tx .select({ activityID: SessionActivityProgressTable.activity_id }) .from(SessionActivityProgressTable) @@ -566,6 +780,18 @@ const service = Effect.gen(function* () { .returning({ epoch: SessionPromptEpochTable.epoch }) .get() if (!retired) return yield* Effect.die(new Error("provider recovery epoch retirement CAS failed")) + if (providerAttempt) { + yield* tx + .insert(SessionProviderAttemptRecoveryBridgeTable) + .values({ + resolution_id: proposed.resolutionID, + attempt_id: providerAttempt.attempt_id, + receipt_id: input.receiptID, + command_id: input.commandID, + created_at: proposed.createdAt, + }) + .run() + } const resolution = yield* tx .insert(SessionToolRequestResolutionTable) .values({ @@ -599,6 +825,47 @@ const service = Effect.gen(function* () { .returning() .get() if (!resolution) return yield* Effect.die(new Error("provider recovery resolution insert failed")) + if (providerAttempt) { + yield* tx + .insert(SessionProviderAttemptResolutionTable) + .values({ + resolution_id: proposed.resolutionID, + attempt_id: providerAttempt.attempt_id, + actor_type: "user", + actor_id: input.actorID, + decision: "abandoned", + provider_evidence: null, + risk_acknowledged: false, + reason: input.reason ?? DEFAULT_REASON, + created_at: proposed.createdAt, + }) + .run() + const terminalAttempt = yield* tx + .update(SessionProviderAttemptTable) + .set({ state: "resolved_abandoned", settled_at: proposed.createdAt }) + .where( + and( + eq(SessionProviderAttemptTable.attempt_id, providerAttempt.attempt_id), + eq(SessionProviderAttemptTable.state, "indeterminate_after_crash"), + ), + ) + .returning({ attemptID: SessionProviderAttemptTable.attempt_id }) + .get() + if (!terminalAttempt) return yield* Effect.die(new Error("provider attempt recovery CAS failed")) + const terminalActivity = yield* tx + .update(SessionActivityTable) + .set({ state: "interrupted", settled_at: proposed.createdAt }) + .where( + and( + eq(SessionActivityTable.activity_id, providerAttempt.activity_id), + eq(SessionActivityTable.session_id, input.sessionID), + eq(SessionActivityTable.state, "active"), + ), + ) + .returning({ activityID: SessionActivityTable.activity_id }) + .get() + if (!terminalActivity) return yield* Effect.die(new Error("provider activity recovery CAS failed")) + } yield* tx .insert(SessionPromptEpochTable) .values({ @@ -738,28 +1005,25 @@ const service = Effect.gen(function* () { if (failure instanceof Conflict || failure instanceof NotFound) return yield* failure return yield* Effect.die(failure) } - const committed = transaction.value - const result = - committed ?? - (yield* db - .select() - .from(SessionToolRequestResolutionTable) - .where(eq(SessionToolRequestResolutionTable.resolution_id, proposed.resolutionID)) - .get() - .pipe(Effect.orDie)) + const result = yield* db + .select() + .from(SessionToolRequestResolutionTable) + .where(eq(SessionToolRequestResolutionTable.resolution_id, proposed.resolutionID)) + .get() + .pipe(Effect.orDie) if (!result) return yield* Effect.die(new Error(`resolution result disappeared: ${proposed.resolutionID}`)) yield* events .publish(Event.Completed, { sessionID: input.sessionID, - resolutionID: result.resolution_id, + resolutionID: proposed.resolutionID, commandID: input.commandID, - receiptID: result.receipt_id, + receiptID: input.receiptID, decision: "abandoned", - sourcePromptEpoch: result.source_prompt_epoch, - successorPromptEpoch: result.successor_prompt_epoch, - sourceMutationEpoch: result.source_mutation_epoch, - successorMutationEpoch: result.successor_mutation_epoch, - createdAt: result.created_at, + sourcePromptEpoch: input.expected.promptEpoch, + successorPromptEpoch: proposed.successorPromptEpoch, + sourceMutationEpoch: input.expected.sessionMutationEpoch, + successorMutationEpoch: proposed.successorMutationEpoch, + createdAt: proposed.createdAt, }) .pipe(Effect.catchCause(() => Effect.void)) return fromResolution(input.commandID, result) diff --git a/packages/deepagent-code/src/session/message-v2.ts b/packages/deepagent-code/src/session/message-v2.ts index f943bb1de..9385c3827 100644 --- a/packages/deepagent-code/src/session/message-v2.ts +++ b/packages/deepagent-code/src/session/message-v2.ts @@ -1349,13 +1349,6 @@ export function promptHistoryCutoffAuthorityInTransaction( const unresolved = yield* tx .select({ userMessageID: SessionToolRequestReceiptTable.user_message_id, - assistantMessageID: SessionToolRequestReceiptTable.assistant_message_id, - promptEpoch: SessionToolRequestReceiptTable.prompt_epoch, - promptWindowID: SessionToolRequestReceiptTable.prompt_window_id, - effectiveHistoryHash: SessionToolRequestReceiptTable.effective_history_hash, - requestInputHash: SessionToolRequestReceiptTable.request_input_hash, - providerRequestHash: SessionToolRequestReceiptTable.provider_request_hash, - finalRequestHash: SessionToolRequestReceiptTable.final_request_hash, }) .from(SessionToolRequestReceiptTable) .leftJoin( @@ -1366,7 +1359,6 @@ export function promptHistoryCutoffAuthorityInTransaction( and( eq(SessionToolRequestReceiptTable.session_id, input.sessionID), eq(SessionToolRequestReceiptTable.provider_state, "indeterminate_after_crash"), - isNull(SessionToolRequestReceiptTable.provider_attempt_id), isNull(SessionToolRequestResolutionTable.resolution_id), ), ) @@ -1374,16 +1366,7 @@ export function promptHistoryCutoffAuthorityInTransaction( const cutoffIndex = selected.messages.findIndex((message) => message.info.id === input.cutoffMessageID) const unsafe = unresolved.some((receipt) => { const userIndex = selected.messages.findIndex((message) => message.info.id === receipt.userMessageID) - const assistantIndex = selected.messages.findIndex((message) => message.info.id === receipt.assistantMessageID) - return ( - receipt.promptEpoch === epoch.epoch && - receipt.promptWindowID === epoch.window_id && - !!receipt.effectiveHistoryHash && - !!(receipt.finalRequestHash ?? receipt.providerRequestHash ?? receipt.requestInputHash) && - userIndex >= 0 && - assistantIndex > userIndex && - cutoffIndex > userIndex - ) + return userIndex < 0 || cutoffIndex > userIndex }) if (unsafe) return return { projection, physical: snapshotExit.value.physical } @@ -1405,12 +1388,7 @@ export const promptHistoryCutoffProjectionEffect = Effect.fn("MessageV2.promptHi const authority = yield* db .select({ state: SessionPromptEpochTable.authority_state }) .from(SessionPromptEpochTable) - .where( - and( - eq(SessionPromptEpochTable.session_id, input.sessionID), - eq(SessionPromptEpochTable.state, "active"), - ), - ) + .where(and(eq(SessionPromptEpochTable.session_id, input.sessionID), eq(SessionPromptEpochTable.state, "active"))) .get() .pipe(Effect.orDie) const history = yield* db @@ -1419,10 +1397,7 @@ export const promptHistoryCutoffProjectionEffect = Effect.fn("MessageV2.promptHi .where(eq(SessionHistoryStateTable.session_id, input.sessionID)) .get() .pipe(Effect.orDie) - if ( - (!authority || authority.state === "legacy_pending") && - history?.state !== "recovery_required" - ) + if ((!authority || authority.state === "legacy_pending") && history?.state !== "recovery_required") yield* promptHistoryProjectionEffect(input.sessionID) const projection = yield* db .transaction((tx) => promptHistoryCutoffProjectionInTransaction(tx as unknown as Database.Interface["db"], input)) diff --git a/packages/deepagent-code/src/session/session.ts b/packages/deepagent-code/src/session/session.ts index feaed6d38..0ee097ed4 100644 --- a/packages/deepagent-code/src/session/session.ts +++ b/packages/deepagent-code/src/session/session.ts @@ -694,6 +694,15 @@ export class UnavailableError extends Schema.TaggedErrorClass( reason: Schema.String, }) {} +export class PlacementChangeUnsupportedError extends Schema.TaggedErrorClass()( + "SessionPlacementChangeUnsupportedError", + { + sessionID: SessionID, + operation: Schema.Literals(["workspace", "directory"]), + message: Schema.String, + }, +) {} + export type NotFound = NotFoundError export interface Interface { @@ -757,9 +766,15 @@ export interface Interface { readonly clearRevert: (sessionID: SessionID) => Effect.Effect readonly setSummary: (input: { sessionID: SessionID; summary: Info["summary"] }) => Effect.Effect readonly setShare: (input: { sessionID: SessionID; share: Info["share"] }) => Effect.Effect - readonly setWorkspace: (input: { sessionID: SessionID; workspaceID: Info["workspaceID"] }) => Effect.Effect + readonly setWorkspace: (input: { + sessionID: SessionID + workspaceID: Info["workspaceID"] + }) => Effect.Effect /** Relocates a durable Session after a managed worktree has replaced its original checkout. */ - readonly setDirectory: (input: { sessionID: SessionID; directory: string }) => Effect.Effect + readonly setDirectory: (input: { + sessionID: SessionID + directory: string + }) => Effect.Effect readonly diff: (sessionID: SessionID) => Effect.Effect readonly messages: (input: { sessionID: SessionID; limit?: number }) => Effect.Effect readonly messagesPage: (input: { @@ -3068,6 +3083,12 @@ export const layer: Layer.Layer< }) { const current = yield* get(input.sessionID).pipe(Effect.orDie) if (current.directory === input.directory) return + if (current.workspaceID) + return yield* new PlacementChangeUnsupportedError({ + sessionID: input.sessionID, + operation: "directory", + message: "Workspace Session relocation requires durable transfer admission and a target receipt", + }) yield* events.publish(SessionEvent.Moved, { sessionID: input.sessionID, location: Location.Ref.make({ @@ -3210,9 +3231,13 @@ export const layer: Layer.Layer< sessionID: SessionID workspaceID: Info["workspaceID"] }) { - yield* patch(input.sessionID, { workspaceID: input.workspaceID, time: { updated: Date.now() } }).pipe( - Effect.orDie, - ) + const current = yield* get(input.sessionID).pipe(Effect.orDie) + if (current.workspaceID === input.workspaceID) return + return yield* new PlacementChangeUnsupportedError({ + sessionID: input.sessionID, + operation: "workspace", + message: "Session workspace changes require durable transfer admission and a target receipt", + }) }) const diff = Effect.fn("Session.diff")(function* (sessionID: SessionID) { diff --git a/packages/deepagent-code/test/config/config.test.ts b/packages/deepagent-code/test/config/config.test.ts index 6be234127..c6728b518 100644 --- a/packages/deepagent-code/test/config/config.test.ts +++ b/packages/deepagent-code/test/config/config.test.ts @@ -1,6 +1,6 @@ import { test, expect, describe, afterEach, beforeEach, spyOn } from "bun:test" import { ConfigV1 } from "@deepagent-code/core/v1/config/config" -import { Effect, Exit, Layer, Option } from "effect" +import { Deferred, Effect, Exit, Layer, Option } from "effect" import { FetchHttpClient, HttpClient, HttpClientResponse } from "effect/unstable/http" import { NodeFileSystem, NodePath } from "@effect/platform-node" import { Config } from "@/config/config" @@ -14,6 +14,7 @@ import { Auth } from "../../src/auth" import { Account } from "../../src/account/account" import { AccessToken, AccountID, OrgID } from "../../src/account/schema" import { FSUtil } from "@deepagent-code/core/fs-util" +import { Npm } from "@deepagent-code/core/npm" import { Env } from "../../src/env" import { provideTmpdirInstance, @@ -25,6 +26,7 @@ import { testInstanceStoreLayer, } from "../fixture/fixture" import { InstanceRuntime } from "@/project/instance-runtime" +import { InstanceStore } from "@/project/instance-store" import { CrossSpawnSpawner } from "@deepagent-code/core/cross-spawn-spawner" import { testEffect } from "../lib/effect" import path from "path" @@ -93,6 +95,7 @@ const configLayer = ( auth?: Layer.Layer account?: Layer.Layer client?: HttpClient.HttpClient + npm?: Layer.Layer } = {}, ) => Config.layer.pipe( @@ -101,7 +104,7 @@ const configLayer = ( Layer.provide(options.auth ?? AuthTest.empty), Layer.provide(options.account ?? AccountTest.empty), Layer.provideMerge(infra), - Layer.provide(NpmTest.noop), + Layer.provide(options.npm ?? NpmTest.noop), Layer.provide(Layer.succeed(HttpClient.HttpClient, options.client ?? unexpectedHttp)), Layer.provideMerge(FSUtil.defaultLayer), ) @@ -1073,6 +1076,34 @@ it.effect("does not install dependencies in a writable DEEPAGENT_CODE_CONFIG_DIR }).pipe(Effect.provide(testInstanceStoreLayer), Effect.provide(CrossSpawnSpawner.defaultLayer)), ) +test("interrupts background dependency installs when the instance is disposed", async () => { + const started = Deferred.makeUnsafe() + const interrupted = Deferred.makeUnsafe() + const npm = Layer.mock(Npm.Service)({ + install: () => + Deferred.succeed(started, undefined).pipe( + Effect.andThen(Effect.never), + Effect.onInterrupt(() => Deferred.succeed(interrupted, undefined)), + ), + }) + + await Effect.runPromise( + Effect.gen(function* () { + const dir = yield* tmpdirScoped() + const store = yield* InstanceStore.Service + yield* store.provide({ directory: dir }, Config.use.get()) + yield* Deferred.await(started).pipe(Effect.timeout("1 second")) + yield* store.disposeDirectory(dir).pipe(Effect.timeout("1 second")) + yield* Deferred.await(interrupted).pipe(Effect.timeout("1 second")) + }).pipe( + Effect.scoped, + Effect.provide(configLayer({ npm })), + Effect.provide(testInstanceStoreLayer), + Effect.provide(CrossSpawnSpawner.defaultLayer), + ), + ) +}) + // Note: deduplication and serialization of npm installs is now handled by the // core Npm.Service (via EffectFlock). Those behaviors are tested in the core // package's npm tests, not here. diff --git a/packages/deepagent-code/test/context-federation/diagnostics.test.ts b/packages/deepagent-code/test/context-federation/diagnostics.test.ts index 700ccda37..de943d859 100644 --- a/packages/deepagent-code/test/context-federation/diagnostics.test.ts +++ b/packages/deepagent-code/test/context-federation/diagnostics.test.ts @@ -4,6 +4,7 @@ import { SessionProviderAttempt } from "@deepagent-code/core/context-federation/ import { SessionActivityTable, SessionContextSelectionTable, + SessionProviderAttemptResolutionTable, SessionProviderAttemptTable, } from "@deepagent-code/core/context-federation/session-sql" import { Database } from "@deepagent-code/core/database/database" @@ -13,6 +14,7 @@ import { SessionMessage } from "@deepagent-code/core/session/message" import { Prompt } from "@deepagent-code/core/session/prompt" import { SessionSchema } from "@deepagent-code/core/session/schema" import { SessionInputTable, SessionTable } from "@deepagent-code/core/session/sql" +import { SessionToolRequestReceiptTable } from "../../src/session/tool-request-receipt.sql" import { SessionV1 } from "@deepagent-code/core/v1/session" import { AbsolutePath } from "@deepagent-code/core/schema" import { Effect, Layer } from "effect" @@ -51,60 +53,126 @@ describe("ContextFederationDiagnostics", () => { Layer.provide(federationLayer), ) - await Effect.runPromise(Effect.gen(function* () { - const db = (yield* Database.Service).db - yield* seed(db) - const service = yield* ContextFederationDiagnostics.Service - const first = yield* service.get(sessionId, 50) - expect(first.selections[0]).toMatchObject({ - summary: "partial", - artifact: { status: "degraded_unavailable", reasonCode: "QuotaExceededError" }, - evidence: [{ token: "ctx_opaque", provenance: ["ctx_provenance"] }], - }) - expect(first.attempts[0]).toMatchObject({ - state: "indeterminate_after_crash", - canAbandon: true, - canSettle: false, - canReplay: true, - }) - const serialized = JSON.stringify(first) - expect(serialized).not.toContain("SECRET SOURCE BODY") - expect(serialized).not.toContain("src/private.ts") - expect(serialized).not.toContain("entity-private") - expect(first.metrics.alerts).toContainEqual({ graph: "documents", state: "degraded", reasonCode: "source_error" }) - expect(first.metrics.shadow).toMatchObject({ - comparisons: 1, - legacyKnowledgeRefs: 2, - legacyMemoryRefs: 1, - federated: { code: 1, knowledge: 1, memory: 0, documents: 1 }, - knowledgeMemoryDelta: -2, - }) + await Effect.runPromise( + Effect.gen(function* () { + const db = (yield* Database.Service).db + yield* seed(db) + const service = yield* ContextFederationDiagnostics.Service + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: "receipt_context_legacy", + request_ordinal: 1, + session_id: sessionId, + user_message_id: inputId, + assistant_message_id: null, + provider_attempt_id: attemptId, + provider_id: "provider-test", + model_id: "model-test", + protocol: "chat", + registry_tool_ids: [], + permission_filtered_tool_ids: [], + final_offered_tool_ids: [], + call_ids: [], + provider_state: "indeterminate_after_crash", + request_state: "dispatched", + request_error_code: "provider_started_outcome_unknown_after_process_restart", + owner_token: "dead-process", + created_at: 35, + }) + .run() + const bypass = yield* db + .insert(SessionProviderAttemptResolutionTable) + .values({ + resolution_id: "resolution_context_legacy_bypass", + attempt_id: attemptId, + actor_type: "user", + actor_id: "local-user", + decision: "abandoned", + provider_evidence: null, + risk_acknowledged: false, + reason: "attempt to bypass unified recovery", + created_at: 40, + }) + .run() + .pipe(Effect.orDie, Effect.exit) + expect(bypass._tag).toBe("Failure") + expect(String(bypass)).toContain("provider_attempt_requires_unified_legacy_recovery") + const blocked = yield* service + .resolveAttempt({ + session: { id: sessionId } as Session.Info, + attemptId, + decision: "abandoned", + reason: "do not split recovery authorities", + riskAcknowledged: false, + actorId: "local-user", + now: 50, + }) + .pipe(Effect.flip) + expect(blocked).toMatchObject({ reason: "legacy_provider_recovery_required" }) + expect((yield* db.select().from(SessionProviderAttemptTable).get())?.state).toBe("indeterminate_after_crash") + expect(yield* db.select().from(SessionProviderAttemptResolutionTable).all()).toEqual([]) + expect((yield* db.select().from(SessionToolRequestReceiptTable).get())?.provider_state).toBe( + "indeterminate_after_crash", + ) + yield* db.delete(SessionToolRequestReceiptTable).run() + const first = yield* service.get(sessionId, 50) + expect(first.selections[0]).toMatchObject({ + summary: "partial", + artifact: { status: "degraded_unavailable", reasonCode: "QuotaExceededError" }, + evidence: [{ token: "ctx_opaque", provenance: ["ctx_provenance"] }], + }) + expect(first.attempts[0]).toMatchObject({ + state: "indeterminate_after_crash", + canAbandon: true, + canSettle: false, + canReplay: true, + }) + const serialized = JSON.stringify(first) + expect(serialized).not.toContain("SECRET SOURCE BODY") + expect(serialized).not.toContain("src/private.ts") + expect(serialized).not.toContain("entity-private") + expect(first.metrics.alerts).toContainEqual({ + graph: "documents", + state: "degraded", + reasonCode: "source_error", + }) + expect(first.metrics.shadow).toMatchObject({ + comparisons: 1, + legacyKnowledgeRefs: 2, + legacyMemoryRefs: 1, + federated: { code: 1, knowledge: 1, memory: 0, documents: 1 }, + knowledgeMemoryDelta: -2, + }) - const missingEvidence = yield* service.resolveAttempt({ - session: { id: sessionId } as Session.Info, - attemptId, - decision: "settled", - reason: "provider completed", - riskAcknowledged: false, - actorId: "local-user", - now: 60, - }).pipe(Effect.flip) - expect(missingEvidence.reason).toBe("persisted_terminal_event_required") + const missingEvidence = yield* service + .resolveAttempt({ + session: { id: sessionId } as Session.Info, + attemptId, + decision: "settled", + reason: "provider completed", + riskAcknowledged: false, + actorId: "local-user", + now: 60, + }) + .pipe(Effect.flip) + expect(missingEvidence.reason).toBe("persisted_terminal_event_required") - messages.value = [terminalMessage()] - expect((yield* service.get(sessionId, 61)).attempts[0]?.canSettle).toBe(true) - const settled = yield* service.resolveAttempt({ - session: { id: sessionId } as Session.Info, - attemptId, - decision: "settled", - reason: "durable assistant message completed", - riskAcknowledged: false, - actorId: "local-user", - now: 62, - }) - expect(settled.state).toBe("resolved_settled") - expect((yield* db.select().from(SessionActivityTable).get())?.state).toBe("settled") - }).pipe(Effect.provide(Layer.merge(database, diagnostics)), Effect.scoped)) + messages.value = [terminalMessage()] + expect((yield* service.get(sessionId, 61)).attempts[0]?.canSettle).toBe(true) + const settled = yield* service.resolveAttempt({ + session: { id: sessionId } as Session.Info, + attemptId, + decision: "settled", + reason: "durable assistant message completed", + riskAcknowledged: false, + actorId: "local-user", + now: 62, + }) + expect(settled.state).toBe("resolved_settled") + expect((yield* db.select().from(SessionActivityTable).get())?.state).toBe("settled") + }).pipe(Effect.provide(Layer.merge(database, diagnostics)), Effect.scoped), + ) }) }) @@ -134,91 +202,111 @@ function seed(db: Database.Interface["db"]) { federated: { code: 1, knowledge: 1, memory: 0, documents: 1 }, }) return Effect.gen(function* () { - yield* db.insert(ProjectTable).values({ - id: projectId, - worktree: AbsolutePath.make("/tmp/context-diagnostics"), - sandboxes: [], - }).run() - yield* db.insert(SessionTable).values({ - id: sessionId, - project_id: projectId, - slug: "diagnostics", - directory: "/tmp/context-diagnostics", - title: "Diagnostics", - version: "test", - }).run() - yield* db.insert(SessionInputTable).values({ - id: inputId, - session_id: sessionId, - prompt: new Prompt({ text: "inspect context" }), - delivery: "steer", - admitted_seq: 0, - promoted_seq: 0, - }).run() - yield* db.insert(SessionActivityTable).values({ - activity_id: activityId, - session_id: sessionId, - ordinal: 0, - trigger_input_id: inputId, - delivery: "steer", - state: "active", - created_at: 10, - }).run() - yield* db.insert(SessionContextSelectionTable).values({ - selection_id: selectionId, - session_id: sessionId, - activity_id: activityId, - revision: 0, - trigger_input_id: inputId, - location_key: "loc_context_diagnostics", - query_fingerprint: "query", - authorization_fingerprint: "authorization", - authorization_epoch: 1, - execution_fingerprint: "execution", - selected_source_fingerprint: "sources", - observed_location_mutation_epoch: 1, - next_revalidation_at: 1_000, - graph_revisions: JSON.stringify({ code: "1", knowledge: "1", memory: "1", documents: "1" }), - graph_statuses: JSON.stringify(statuses), - selected_refs: JSON.stringify([{ - ref: { - graph: "code", - entityId: "entity-private", - binding: { scope: "builtin" }, - locator: { path: "src/private.ts" }, - revision: "code:1", - }, - token: "ctx_opaque", - provenanceTokens: ["ctx_provenance"], - relations: [{ relation: "implements", token: "ctx_relation", freshness: "broken" }], - freshness: "current", - sensitivity: "source_code", - score: 0.9, - reason: "federated_rank", - excerpt: "SECRET SOURCE BODY", - projectionStart: 0, - projectionEnd: 10, - }]), - projection: "project-context-json-v1 bytes=2\n{}", - projection_hash: "projection", - token_count: 8, - artifact_write_status: "degraded_unavailable", - inline_audit: JSON.stringify({ reasonCode: "QuotaExceededError" }), - created_at: 20, - }).run() - yield* db.insert(SessionProviderAttemptTable).values({ - attempt_id: attemptId, - session_id: sessionId, - activity_id: activityId, - provider_turn_seq: 0, - selection_id: selectionId, - projection_hash: "projection", - request_hash: "request", - provider_id: "provider-test", - state: "indeterminate_after_crash", - created_at: 30, - error_code: "process_recovery", - }).run() + yield* db + .insert(ProjectTable) + .values({ + id: projectId, + worktree: AbsolutePath.make("/tmp/context-diagnostics"), + sandboxes: [], + }) + .run() + yield* db + .insert(SessionTable) + .values({ + id: sessionId, + project_id: projectId, + slug: "diagnostics", + directory: "/tmp/context-diagnostics", + title: "Diagnostics", + version: "test", + }) + .run() + yield* db + .insert(SessionInputTable) + .values({ + id: inputId, + session_id: sessionId, + prompt: new Prompt({ text: "inspect context" }), + delivery: "steer", + admitted_seq: 0, + promoted_seq: 0, + }) + .run() + yield* db + .insert(SessionActivityTable) + .values({ + activity_id: activityId, + session_id: sessionId, + ordinal: 0, + trigger_input_id: inputId, + delivery: "steer", + state: "active", + created_at: 10, + }) + .run() + yield* db + .insert(SessionContextSelectionTable) + .values({ + selection_id: selectionId, + session_id: sessionId, + activity_id: activityId, + revision: 0, + trigger_input_id: inputId, + location_key: "loc_context_diagnostics", + query_fingerprint: "query", + authorization_fingerprint: "authorization", + authorization_epoch: 1, + execution_fingerprint: "execution", + selected_source_fingerprint: "sources", + observed_location_mutation_epoch: 1, + next_revalidation_at: 1_000, + graph_revisions: JSON.stringify({ code: "1", knowledge: "1", memory: "1", documents: "1" }), + graph_statuses: JSON.stringify(statuses), + selected_refs: JSON.stringify([ + { + ref: { + graph: "code", + entityId: "entity-private", + binding: { scope: "builtin" }, + locator: { path: "src/private.ts" }, + revision: "code:1", + }, + token: "ctx_opaque", + provenanceTokens: ["ctx_provenance"], + relations: [{ relation: "implements", token: "ctx_relation", freshness: "broken" }], + freshness: "current", + sensitivity: "source_code", + score: 0.9, + reason: "federated_rank", + excerpt: "SECRET SOURCE BODY", + projectionStart: 0, + projectionEnd: 10, + }, + ]), + projection: "project-context-json-v1 bytes=2\n{}", + projection_hash: "projection", + token_count: 8, + artifact_write_status: "degraded_unavailable", + inline_audit: JSON.stringify({ reasonCode: "QuotaExceededError" }), + created_at: 20, + }) + .run() + yield* db + .insert(SessionProviderAttemptTable) + .values({ + attempt_id: attemptId, + session_id: sessionId, + activity_id: activityId, + provider_turn_seq: 0, + selection_id: selectionId, + projection_hash: "projection", + request_hash: "request", + provider_id: "provider-test", + state: "indeterminate_after_crash", + created_at: 30, + error_code: "process_recovery", + }) + .run() }) } diff --git a/packages/deepagent-code/test/control-plane/workspace.test.ts b/packages/deepagent-code/test/control-plane/workspace.test.ts index 9709d934f..a5421cb4b 100644 --- a/packages/deepagent-code/test/control-plane/workspace.test.ts +++ b/packages/deepagent-code/test/control-plane/workspace.test.ts @@ -6,7 +6,7 @@ import path from "node:path" import { NodeHttpServer } from "@effect/platform-node" import { Cause, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect" import { FetchHttpClient, HttpServer, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" -import { eq } from "drizzle-orm" +import { and, eq } from "drizzle-orm" import { FSUtil } from "@deepagent-code/core/fs-util" import * as Log from "@deepagent-code/core/util/log" import { GlobalBus, type GlobalEvent } from "@/bus/global" @@ -17,7 +17,7 @@ import { AbsolutePath } from "@deepagent-code/core/schema" import { Session as SessionNs } from "@/session/session" import { SessionID } from "@/session/schema" import { SessionTable } from "@deepagent-code/core/session/sql" -import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { EventSequenceTable, EventTable, WorkspaceSyncCursorTable } from "@deepagent-code/core/event/sql" import { SessionToolRequestReceiptTable } from "@/session/tool-request-receipt.sql" import { resetDatabase } from "../fixture/db" import { disposeAllInstances, provideTmpdirInstance, requireInstance, TestInstance } from "../fixture/fixture" @@ -38,6 +38,7 @@ import { EventV2Bridge } from "@/event-v2-bridge" import { EventV2 } from "@deepagent-code/core/event" import { SessionV1 } from "@deepagent-code/core/v1/session" import { SyncReplayLimits } from "@/sync/replay-protocol" +import { Hash } from "@deepagent-code/core/util/hash" void Log.init({ print: false }) @@ -728,7 +729,8 @@ describe("workspace CRUD", () => { calls.push(call) if (call.url.pathname === "/base/global/event") return HttpServerResponse.fromWeb(eventStreamResponse([], false)) - if (call.url.pathname === "/base/sync/history") return yield* HttpServerResponse.json([]) + if (call.url.pathname === "/base/sync/history") + return yield* HttpServerResponse.json({ version: 1, items: [], nextCursor: "cursor-empty", complete: true }) return HttpServerResponse.text("unexpected", { status: 500 }) }), ) @@ -747,7 +749,7 @@ describe("workspace CRUD", () => { expect( calls.map((call) => `${call.method} ${call.url.pathname}${call.url.search}${call.url.hash}`), ).toEqual(["GET /base/global/event", "POST /base/sync/history"]) - expect(calls[1].json).toEqual({}) + expect(calls[1].json).toEqual({ version: 1 }) expect((yield* workspace.status()).find((item) => item.workspaceID === info.id)?.status).toBe("connected") expect(yield* workspace.isSyncing(info.id)).toBe(true) @@ -1661,7 +1663,7 @@ describe("workspace sync state", () => { }), ) - it.live("sync history sends the local sequence fence and replays returned events in workspace context", () => { + it.live("sync history advances its durable cursor and replays returned events in workspace context", () => { const historyBodies: unknown[] = [] let historySessionID: SessionID | undefined let historySession: SessionNs.Info | undefined @@ -1676,17 +1678,26 @@ describe("workspace sync state", () => { if (url.pathname === "/history/sync/history") { const body = bodyText ? JSON.parse(bodyText) : undefined historyBodies.push(body) - if (body?.[historySessionID!] === historyNextSeq) return HttpServerResponse.fromWeb(Response.json([])) + if (body?.cursor === "cursor-history-1") + return HttpServerResponse.fromWeb( + Response.json({ version: 1, items: [], nextCursor: "cursor-history-1", complete: true }), + ) return HttpServerResponse.fromWeb( - Response.json([ - { - id: `evt_${unique("history")}`, - aggregate_id: historySessionID!, - seq: historyNextSeq, - type: "session.updated.1", - data: { sessionID: historySessionID!, info: historySession! }, - }, - ]), + Response.json({ + version: 1, + items: [ + { + kind: "event", + id: `evt_${unique("history")}`, + aggregate_id: historySessionID!, + seq: historyNextSeq, + type: "session.updated.1", + data: { sessionID: historySessionID!, info: historySession! }, + }, + ], + nextCursor: "cursor-history-1", + complete: false, + }), ) } return HttpServerResponse.text("unexpected", { status: 500 }) @@ -1718,7 +1729,19 @@ describe("workspace sync state", () => { expect((yield* sessionSvc.get(session.id).pipe(Effect.orDie)).title).toBe("from history") }), ) - expect(historyBodies).toEqual([{ [session.id]: historyNextSeq - 1 }, { [session.id]: historyNextSeq }]) + expect(historyBodies).toEqual([ + { version: 1 }, + { version: 1, cursor: "cursor-history-1" }, + ]) + expect( + yield* Database.Service.use(({ db }) => + db + .select({ workspaceID: WorkspaceSyncCursorTable.workspace_id, cursor: WorkspaceSyncCursorTable.cursor }) + .from(WorkspaceSyncCursorTable) + .where(eq(WorkspaceSyncCursorTable.workspace_id, info.id)) + .get(), + ), + ).toEqual({ workspaceID: info.id, cursor: "cursor-history-1" }) expect( captured.events.some( (event) => @@ -1736,7 +1759,269 @@ describe("workspace sync state", () => { { git: true }, ) }) - }) + }, 30_000) + + it.live("does not advance its durable cursor when a history page fails replay", () => { + return Effect.gen(function* () { + yield* HttpServer.serveEffect()( + Effect.gen(function* () { + const req = yield* HttpServerRequest.HttpServerRequest + const url = new URL(req.url, "http://localhost") + if (url.pathname === "/history-replay-fail/global/event") + return HttpServerResponse.fromWeb(eventStreamResponse([], false)) + if (url.pathname === "/history-replay-fail/sync/history") + return HttpServerResponse.fromWeb( + Response.json({ + version: 1, + items: [{ kind: "event", id: `evt_${unique("bad-history")}`, aggregate_id: "ses_missing", seq: 1, type: "unknown.sync.1", data: {} }], + nextCursor: "must-not-commit", + complete: true, + }), + ) + return HttpServerResponse.text("unexpected", { status: 500 }) + }), + ) + const url = yield* serverUrl() + yield* provideTmpdirInstance( + () => + Effect.gen(function* () { + const workspace = yield* Workspace.Service + const instance = yield* requireInstance + const type = unique("history-replay-fail") + const info = workspaceInfo(instance.project.id, type) + yield* insertWorkspace(info) + registerAdapter(instance.project.id, type, remoteAdapter(`${url}/history-replay-fail`).adapter) + yield* workspace.startWorkspaceSyncing(instance.project.id) + yield* Effect.sleep("100 millis") + expect( + yield* Database.Service.use(({ db }) => + db.select().from(WorkspaceSyncCursorTable).where(eq(WorkspaceSyncCursorTable.workspace_id, info.id)).get(), + ), + ).toBeUndefined() + yield* workspace.remove(info.id) + }), + { git: true }, + ) + }) + }, 30_000) + + it.live("resumes from its durable cursor after reconnecting", () => { + const historyBodies = new Array<{ version: 1; cursor?: string }>() + return Effect.gen(function* () { + yield* HttpServer.serveEffect()( + Effect.gen(function* () { + const req = yield* HttpServerRequest.HttpServerRequest + const url = new URL(req.url, "http://localhost") + if (url.pathname === "/history-reconnect/global/event") + return HttpServerResponse.fromWeb(eventStreamResponse([], false)) + if (url.pathname === "/history-reconnect/sync/history") { + const body = JSON.parse(yield* req.text) as { version: 1; cursor?: string } + historyBodies.push(body) + return HttpServerResponse.fromWeb( + Response.json({ + version: 1, + items: [], + nextCursor: body.cursor ? "cursor-reconnect-2" : "cursor-reconnect-1", + complete: true, + }), + ) + } + return HttpServerResponse.text("unexpected", { status: 500 }) + }), + ) + const url = yield* serverUrl() + yield* provideTmpdirInstance( + () => + Effect.gen(function* () { + const workspace = yield* Workspace.Service + const instance = yield* requireInstance + const type = unique("history-reconnect") + const info = workspaceInfo(instance.project.id, type) + yield* insertWorkspace(info) + registerAdapter(instance.project.id, type, remoteAdapter(`${url}/history-reconnect`).adapter) + + yield* workspace.startWorkspaceSyncing(instance.project.id) + yield* eventuallyEffect(Effect.sync(() => expect(historyBodies.length).toBeGreaterThanOrEqual(2)), 3_000) + + expect(historyBodies.slice(0, 2)).toEqual([ + { version: 1 }, + { version: 1, cursor: "cursor-reconnect-1" }, + ]) + expect( + yield* Database.Service.use(({ db }) => + db + .select({ cursor: WorkspaceSyncCursorTable.cursor }) + .from(WorkspaceSyncCursorTable) + .where(eq(WorkspaceSyncCursorTable.workspace_id, info.id)) + .get(), + ), + ).toEqual({ cursor: "cursor-reconnect-2" }) + yield* workspace.remove(info.id) + }), + { git: true }, + ) + }) + }, 30_000) + + it.live("resets a stale cursor once and preserves it for unrelated conflicts", () => { + const historyBodies = new Array<{ version: 1; cursor?: string }>() + return Effect.gen(function* () { + yield* HttpServer.serveEffect()( + Effect.gen(function* () { + const req = yield* HttpServerRequest.HttpServerRequest + const url = new URL(req.url, "http://localhost") + if (url.pathname.endsWith("/global/event")) return HttpServerResponse.fromWeb(eventStreamResponse()) + if (url.pathname === "/history-stale/sync/history") { + const body = JSON.parse(yield* req.text) as { version: 1; cursor?: string } + historyBodies.push(body) + if (body.cursor) + return HttpServerResponse.fromWeb( + Response.json( + { + _tag: "ConflictError", + message: "stale cursor", + resource: `sync-cursor-reset:${staleWorkspaceID}`, + }, + { status: 409 }, + ), + ) + return HttpServerResponse.fromWeb( + Response.json({ version: 1, items: [], nextCursor: "cursor-stale-reset", complete: true }), + ) + } + if (url.pathname === "/history-bad-request/sync/history") { + historyBodies.push(JSON.parse(yield* req.text) as { version: 1; cursor?: string }) + return HttpServerResponse.fromWeb( + Response.json( + { _tag: "ConflictError", message: "not a cursor reset", resource: "another-conflict" }, + { status: 409 }, + ), + ) + } + return HttpServerResponse.text("unexpected", { status: 500 }) + }), + ) + const url = yield* serverUrl() + let staleWorkspaceID = "" + yield* provideTmpdirInstance( + () => + Effect.gen(function* () { + const workspace = yield* Workspace.Service + const instance = yield* requireInstance + const staleType = unique("history-stale") + const stale = workspaceInfo(instance.project.id, staleType) + staleWorkspaceID = stale.id + const staleTarget = `${url}/history-stale` + yield* insertWorkspace(stale) + registerAdapter(instance.project.id, staleType, remoteAdapter(staleTarget).adapter) + yield* Database.Service.use(({ db }) => + db.insert(WorkspaceSyncCursorTable).values({ + workspace_id: stale.id, + remote_fingerprint: Hash.sha256(`${new URL(staleTarget).toString()}:workspace:${stale.id}`), + cursor: "cursor-stale-old-generation", + updated_at: 1, + }), + ) + + yield* workspace.startWorkspaceSyncing(instance.project.id) + yield* eventuallyEffect(Effect.sync(() => expect(historyBodies.length).toBe(2)) ) + expect(historyBodies).toEqual([ + { version: 1, cursor: "cursor-stale-old-generation" }, + { version: 1 }, + ]) + expect( + yield* Database.Service.use(({ db }) => + db + .select({ cursor: WorkspaceSyncCursorTable.cursor }) + .from(WorkspaceSyncCursorTable) + .where(eq(WorkspaceSyncCursorTable.workspace_id, stale.id)) + .get(), + ), + ).toEqual({ cursor: "cursor-stale-reset" }) + yield* workspace.remove(stale.id) + + historyBodies.length = 0 + const badType = unique("history-bad-request") + const bad = workspaceInfo(instance.project.id, badType) + const badTarget = `${url}/history-bad-request` + yield* insertWorkspace(bad) + registerAdapter(instance.project.id, badType, remoteAdapter(badTarget).adapter) + yield* Database.Service.use(({ db }) => + db.insert(WorkspaceSyncCursorTable).values({ + workspace_id: bad.id, + remote_fingerprint: Hash.sha256(`${new URL(badTarget).toString()}:workspace:${bad.id}`), + cursor: "cursor-preserved-on-400", + updated_at: 1, + }), + ) + + yield* workspace.startWorkspaceSyncing(instance.project.id) + yield* eventuallyEffect( + Effect.gen(function* () { + expect((yield* workspace.status()).find((item) => item.workspaceID === bad.id)?.status).toBe("error") + }), + ) + expect(historyBodies[0]).toEqual({ version: 1, cursor: "cursor-preserved-on-400" }) + expect( + yield* Database.Service.use(({ db }) => + db + .select({ cursor: WorkspaceSyncCursorTable.cursor }) + .from(WorkspaceSyncCursorTable) + .where(eq(WorkspaceSyncCursorTable.workspace_id, bad.id)) + .get(), + ), + ).toEqual({ cursor: "cursor-preserved-on-400" }) + yield* workspace.remove(bad.id) + }), + { git: true }, + ) + }) + }, 30_000) + + it.effect("allows only one concurrent durable cursor compare-and-swap winner", () => + Effect.gen(function* () { + const workspaceID = WorkspaceV2.ID.make("wrk_cursor_cas") + const fingerprint = Hash.sha256("cursor-cas-fingerprint") + const { db } = yield* Database.Service + yield* db + .insert(WorkspaceSyncCursorTable) + .values({ workspace_id: workspaceID, remote_fingerprint: fingerprint, cursor: "cursor-base", updated_at: 1 }) + .run() + .pipe(Effect.orDie) + const winners = yield* Effect.all( + ["cursor-left", "cursor-right"].map((cursor) => + db + .update(WorkspaceSyncCursorTable) + .set({ cursor, updated_at: 2 }) + .where( + and( + eq(WorkspaceSyncCursorTable.workspace_id, workspaceID), + eq(WorkspaceSyncCursorTable.remote_fingerprint, fingerprint), + eq(WorkspaceSyncCursorTable.cursor, "cursor-base"), + ), + ) + .returning({ cursor: WorkspaceSyncCursorTable.cursor }) + .get() + .pipe(Effect.orDie), + ), + { concurrency: "unbounded" }, + ) + + expect(winners.filter(Boolean)).toHaveLength(1) + expect( + yield* db + .select({ cursor: WorkspaceSyncCursorTable.cursor }) + .from(WorkspaceSyncCursorTable) + .where( + and( + eq(WorkspaceSyncCursorTable.workspace_id, workspaceID), + eq(WorkspaceSyncCursorTable.remote_fingerprint, fingerprint), + ), + ) + .get(), + ).toEqual(winners.find(Boolean)) + }), + ) it.live("SSE forwards non-heartbeat events and ignores heartbeats", () => Effect.gen(function* () { diff --git a/packages/deepagent-code/test/effect/runner.test.ts b/packages/deepagent-code/test/effect/runner.test.ts index a5afe79f2..d8ec09f84 100644 --- a/packages/deepagent-code/test/effect/runner.test.ts +++ b/packages/deepagent-code/test/effect/runner.test.ts @@ -318,6 +318,46 @@ describe("Runner", () => { }), ) + it.live( + "keeps finalizing busy until cancellation teardown exits", + Effect.gen(function* () { + const s = yield* Scope.Scope + const interrupted = yield* Deferred.make() + const release = yield* Deferred.make() + const idle = yield* Deferred.make() + const runner = Runner.make(s, { onIdle: Deferred.succeed(idle, undefined).pipe(Effect.asVoid) }) + const run = yield* runner + .ensureRunning( + Effect.never.pipe( + Effect.onInterrupt(() => Deferred.succeed(interrupted, undefined).pipe(Effect.andThen(Deferred.await(release)))), + Effect.as("done"), + ), + ) + .pipe(Effect.forkChild) + + yield* waitForState(runner, "Running") + yield* runner.markFinalizing + const cancel = yield* runner.cancel.pipe(Effect.forkChild) + yield* Deferred.await(interrupted).pipe(Effect.timeout("250 millis")) + expect(runner.state._tag).toBe("Finalizing") + expect(runner.busy).toBe(true) + expect(yield* Deferred.isDone(idle)).toBe(false) + + const replacementWhileFinalizing = yield* runner.startRunning(Effect.succeed("replacement")).pipe(Effect.exit) + expect(Exit.isFailure(replacementWhileFinalizing)).toBe(true) + if (Exit.isFailure(replacementWhileFinalizing)) + expect(Cause.squash(replacementWhileFinalizing.cause)).toBeInstanceOf(Runner.Busy) + + yield* Deferred.succeed(release, undefined) + yield* Fiber.join(cancel).pipe(Effect.timeout("250 millis")) + yield* Fiber.await(run) + expect(yield* Deferred.isDone(idle)).toBe(true) + expect(runner.state._tag).toBe("Idle") + expect(runner.busy).toBe(false) + expect(yield* runner.startRunning(Effect.succeed("replacement"))).toBe("replacement") + }), + ) + // --- shell semantics --- it.live( diff --git a/packages/deepagent-code/test/server/httpapi-provider-recovery.test.ts b/packages/deepagent-code/test/server/httpapi-provider-recovery.test.ts index 6310c07ce..b8927001d 100644 --- a/packages/deepagent-code/test/server/httpapi-provider-recovery.test.ts +++ b/packages/deepagent-code/test/server/httpapi-provider-recovery.test.ts @@ -5,7 +5,16 @@ import { Hash } from "@deepagent-code/core/util/hash" import { ModelV2 } from "@deepagent-code/core/model" import { ProviderV2 } from "@deepagent-code/core/provider" import { WorkspaceV2 } from "@deepagent-code/core/workspace" -import { SessionHistoryStateTable, SessionTable } from "@deepagent-code/core/session/sql" +import { + SessionActivityTable, + SessionContextSelectionTable, + SessionProviderAttemptRecoveryBridgeTable, + SessionProviderAttemptResolutionTable, + SessionProviderAttemptTable, +} from "@deepagent-code/core/context-federation/session-sql" +import { Prompt } from "@deepagent-code/core/session/prompt" +import { SessionMessage } from "@deepagent-code/core/session/message" +import { SessionHistoryStateTable, SessionInputTable, SessionTable } from "@deepagent-code/core/session/sql" import { and, eq } from "drizzle-orm" import { Effect, Layer } from "effect" import { MessageV2 } from "@/session/message-v2" @@ -39,7 +48,10 @@ const pathFor = (sessionID: string) => SessionPaths.providerResolution.replace(" const parseJson = (response: { readonly json: Effect.Effect }) => response.json.pipe(Effect.map((body) => body as A)) -const seedRecovery = Effect.fn("HttpProviderRecoveryTest.seedRecovery")(function* (title: string) { +const seedRecovery = Effect.fn("HttpProviderRecoveryTest.seedRecovery")(function* ( + title: string, + providerAttempt = false, +) { const sessions = yield* Session.Service const { db } = yield* Database.Service const parent = yield* sessions.create({ title: `${title} parent` }) @@ -78,6 +90,82 @@ const seedRecovery = Effect.fn("HttpProviderRecoveryTest.seedRecovery")(function const now = Date.now() const receiptID = `receipt-${Hash.sha256(title).slice(0, 24)}` const requestHash = `request-${Hash.sha256(title).slice(0, 24)}` + const providerAttemptID = `attempt-${Hash.sha256(title).slice(0, 24)}` + const activityID = `activity-${Hash.sha256(title).slice(0, 24)}` + const selectionID = `selection-${Hash.sha256(title).slice(0, 24)}` + + if (providerAttempt) { + yield* db + .insert(SessionInputTable) + .values({ + id: SessionMessage.ID.make(user.id), + session_id: session.id, + prompt: new Prompt({ text: "dispatch this provider request" }), + delivery: "steer", + admitted_seq: 0, + promoted_seq: 0, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionActivityTable) + .values({ + activity_id: activityID, + session_id: session.id, + ordinal: 0, + trigger_input_id: user.id, + delivery: "steer", + state: "active", + created_at: now, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionContextSelectionTable) + .values({ + selection_id: selectionID, + session_id: session.id, + activity_id: activityID, + revision: 0, + trigger_input_id: user.id, + location_key: "local", + query_fingerprint: "query", + authorization_fingerprint: "authorization", + authorization_epoch: 0, + execution_fingerprint: "execution", + selected_source_fingerprint: "sources", + observed_location_mutation_epoch: 0, + next_revalidation_at: now + 60_000, + graph_revisions: "{}", + graph_statuses: "[]", + selected_refs: "[]", + projection: "", + projection_hash: "projection", + token_count: 0, + artifact_write_status: "degraded_unavailable", + inline_audit: "{}", + created_at: now, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionProviderAttemptTable) + .values({ + attempt_id: providerAttemptID, + session_id: session.id, + activity_id: activityID, + provider_turn_seq: 0, + selection_id: selectionID, + projection_hash: "projection", + request_hash: requestHash, + provider_id: model.providerID, + state: "indeterminate_after_crash", + created_at: now, + error_code: "process_recovery", + }) + .run() + .pipe(Effect.orDie) + } yield* db .update(SessionPromptEpochTable) @@ -96,7 +184,7 @@ const seedRecovery = Effect.fn("HttpProviderRecoveryTest.seedRecovery")(function session_id: session.id, user_message_id: user.id, assistant_message_id: assistant.id, - provider_attempt_id: null, + provider_attempt_id: providerAttempt ? providerAttemptID : null, provider_id: model.providerID, model_id: model.modelID, protocol: "chat", @@ -131,7 +219,7 @@ const seedRecovery = Effect.fn("HttpProviderRecoveryTest.seedRecovery")(function .run() .pipe(Effect.orDie) - return { session, receiptID } + return { session, receiptID, ...(providerAttempt ? { providerAttemptID, activityID } : {}) } }) const command = (descriptor: Descriptor, commandID: string) => ({ @@ -163,6 +251,75 @@ describe("provider recovery HttpApi", () => { const { db } = yield* Database.Service const headers = { "content-type": "application/json" } + const dual = yield* seedRecovery("http unified provider authority", true) + if (!dual.providerAttemptID || !dual.activityID) return + const contextPath = SessionPaths.contextAttemptResolve + .replace(":sessionID", dual.session.id) + .replace(":attemptID", dual.providerAttemptID) + const independentlyResolved = yield* requestInDirectory(contextPath, instance.directory, { + method: "POST", + headers, + body: JSON.stringify({ + decision: "abandoned", + reason: "must use unified provider recovery", + riskAcknowledged: false, + }), + }) + expect(independentlyResolved.status).toBe(400) + expect(yield* db.select().from(SessionProviderAttemptResolutionTable).all()).toEqual([]) + + const dualList = yield* requestInDirectory(pathFor(dual.session.id), instance.directory, { headers }) + expect(dualList.status).toBe(200) + const [dualDescriptor] = yield* parseJson(dualList) + if (!dualDescriptor) return + const dualCommand = command(dualDescriptor, "http-provider-recovery-unified") + const dualResponse = yield* requestInDirectory(pathFor(dual.session.id), instance.directory, { + method: "POST", + headers, + body: JSON.stringify(dualCommand), + }) + expect(dualResponse.status).toBe(200) + const dualResolution = yield* parseJson<{ resolutionID: string }>(dualResponse) + const dualRetry = yield* requestInDirectory(pathFor(dual.session.id), instance.directory, { + method: "POST", + headers, + body: JSON.stringify(dualCommand), + }) + expect(dualRetry.status).toBe(200) + expect(yield* parseJson<{ resolutionID: string }>(dualRetry)).toEqual(dualResolution) + expect(yield* db.select().from(SessionProviderAttemptRecoveryBridgeTable).get()).toMatchObject({ + resolution_id: dualResolution.resolutionID, + attempt_id: dual.providerAttemptID, + receipt_id: dual.receiptID, + command_id: dualCommand.commandID, + }) + expect(yield* db.select().from(SessionProviderAttemptResolutionTable).get()).toMatchObject({ + resolution_id: dualResolution.resolutionID, + attempt_id: dual.providerAttemptID, + decision: "abandoned", + }) + expect( + yield* db + .select({ state: SessionProviderAttemptTable.state }) + .from(SessionProviderAttemptTable) + .where(eq(SessionProviderAttemptTable.attempt_id, dual.providerAttemptID)) + .get(), + ).toEqual({ state: "resolved_abandoned" }) + expect( + yield* db + .select({ state: SessionActivityTable.state }) + .from(SessionActivityTable) + .where(eq(SessionActivityTable.activity_id, dual.activityID)) + .get(), + ).toEqual({ state: "interrupted" }) + expect( + yield* db + .select({ state: SessionHistoryStateTable.state }) + .from(SessionHistoryStateTable) + .where(eq(SessionHistoryStateTable.session_id, dual.session.id)) + .get(), + ).toEqual({ state: "ready" }) + const exact = yield* seedRecovery("http exact retry") const listed = yield* requestInDirectory(pathFor(exact.session.id), instance.directory, { headers }) expect(listed.status).toBe(200) diff --git a/packages/deepagent-code/test/server/httpapi-sync.test.ts b/packages/deepagent-code/test/server/httpapi-sync.test.ts index d53374111..cf8fe1e74 100644 --- a/packages/deepagent-code/test/server/httpapi-sync.test.ts +++ b/packages/deepagent-code/test/server/httpapi-sync.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, mock, spyOn } from "bun:test" +import { afterEach, beforeEach, describe, expect, mock, spyOn } from "bun:test" import { Context, Effect, Layer } from "effect" import { Flag } from "@deepagent-code/core/flag/flag" import { SyncPaths, SyncReplayLimits } from "../../src/server/routes/instance/httpapi/groups/sync" @@ -20,22 +20,105 @@ import { EventV2 } from "@deepagent-code/core/event" import { eq } from "drizzle-orm" import { HttpServer } from "effect/unstable/http" import { SessionV1 } from "@deepagent-code/core/v1/session" +import { SessionTable } from "@deepagent-code/core/session/sql" import { encodeReplayRequestPrefix } from "@/sync/replay-protocol" +import { WorkspaceV2 } from "@deepagent-code/core/workspace" void Log.init({ print: false }) const originalWorkspaces = Flag.DEEPAGENT_CODE_EXPERIMENTAL_WORKSPACES +const originalWorkspaceID = Flag.DEEPAGENT_CODE_WORKSPACE_ID +const syncWorkspaceID = WorkspaceV2.ID.make("wrk_sync_http_scope") const context = Context.empty() as Context.Context const it = testEffect(Layer.mergeAll(Session.defaultLayer, Database.defaultLayer, httpApiLayer)) +type HistoryItem = { + readonly kind: "event" | "resync_required" + readonly id?: string + readonly aggregate_id: string + readonly seq?: number + readonly type?: string + readonly data?: Record +} + +type HistoryResponse = { + readonly version: 1 + readonly items: HistoryItem[] + readonly nextCursor: string + readonly complete: boolean +} + +beforeEach(() => { + Flag.DEEPAGENT_CODE_WORKSPACE_ID = syncWorkspaceID +}) + afterEach(async () => { mock.restore() Flag.DEEPAGENT_CODE_EXPERIMENTAL_WORKSPACES = originalWorkspaces + Flag.DEEPAGENT_CODE_WORKSPACE_ID = originalWorkspaceID await disposeAllInstances() await resetDatabase() }) describe("sync HttpApi", () => { + it.instance( + "fails closed when history has no routed workspace scope", + () => + Effect.gen(function* () { + Flag.DEEPAGENT_CODE_WORKSPACE_ID = undefined + const tmp = yield* TestInstance + const response = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers: { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" }, + body: JSON.stringify({ version: 1 }), + }) + expect(response.status).toBe(400) + }), + { git: true, config: { formatter: false, lsp: false } }, + ) + + it.instance( + "rejects fresh replay without workspace scope before writing durable state", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const session = yield* Session.use.create({ title: "unscoped replay", workspaceID: syncWorkspaceID }) + const { db } = yield* Database.Service + const created = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + yield* db.delete(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, session.id)).run().pipe(Effect.orDie) + yield* db.delete(SessionTable).where(eq(SessionTable.id, session.id)).run().pipe(Effect.orDie) + Flag.DEEPAGENT_CODE_WORKSPACE_ID = undefined + const response = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + directory: tmp.directory, + events: [ + { + id: created!.id, + aggregateID: session.id, + seq: 0, + type: created!.type, + data: created!.data, + }, + ], + }), + }) + + expect(response.status).toBe(400) + expect(yield* db.select().from(SessionTable).where(eq(SessionTable.id, session.id)).get()).toBeUndefined() + expect( + yield* db.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, session.id)).get(), + ).toBeUndefined() + expect(yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, session.id)).all()).toEqual([]) + }), + { git: true, config: { formatter: false, lsp: false } }, + ) it.instance( "serves sync routes", () => @@ -44,7 +127,8 @@ describe("sync HttpApi", () => { const tmp = yield* TestInstance const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } const info = spyOn(Log.create({ service: "server.sync" }), "info") - const session = yield* Session.use.create({ title: "sync" }) + const session = yield* Session.use.create({ title: "sync", workspaceID: syncWorkspaceID }) + yield* Session.use.setTitle({ sessionID: session.id, title: "sync-updated" }) const started = yield* requestInDirectory(SyncPaths.start, tmp.directory, { method: "POST", headers }) expect(started.status).toBe(200) @@ -56,47 +140,245 @@ describe("sync HttpApi", () => { body: JSON.stringify({}), }) expect(history.status).toBe(200) - const rows = (yield* history.json) as Array<{ - id: string - aggregate_id: string - seq: number - type: string - data: Record - }> - expect(rows.map((row) => row.aggregate_id)).toContain(session.id) + const historyBody = (yield* history.json) as HistoryResponse + expect(historyBody.version).toBe(1) + expect(historyBody.nextCursor).toBeTruthy() + expect(historyBody.items.map((row) => row.aggregate_id)).toContain(session.id) + + const { db } = yield* Database.Service + yield* db.delete(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, session.id)).run().pipe(Effect.orDie) + yield* db.delete(SessionTable).where(eq(SessionTable.id, session.id)).run().pipe(Effect.orDie) const replayed = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { method: "POST", headers, body: JSON.stringify({ directory: tmp.directory, - events: rows + events: historyBody.items + .filter((row) => row.kind === "event") .filter((row) => row.aggregate_id === session.id) .map((row) => ({ - id: row.id, + id: row.id!, aggregateID: row.aggregate_id, - seq: row.seq, - type: row.type, - data: row.data, + seq: row.seq!, + type: row.type!, + data: row.data!, })), }), }) - expect(replayed.status).toBe(200) - expect(yield* replayed.json).toEqual({ sessionID: session.id }) + const replayBody = yield* replayed.text + expect(replayed.status, replayBody).toBe(200) + expect(JSON.parse(replayBody)).toEqual({ sessionID: session.id }) + expect((yield* Session.use.get(session.id)).title).toBe("sync-updated") expect(info.mock.calls.some(([message]) => message === "sync replay requested")).toBe(true) expect(info.mock.calls.some(([message]) => message === "sync replay complete")).toBe(true) + + const tampered = `${historyBody.nextCursor.slice(0, -1)}${historyBody.nextCursor.endsWith("a") ? "b" : "a"}` + const invalid = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1, cursor: tampered }), + }) + expect(invalid.status).toBe(409) + + Flag.DEEPAGENT_CODE_WORKSPACE_ID = "wrk_sync_other_scope" + const crossed = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1, cursor: historyBody.nextCursor }), + }) + expect(crossed.status).toBe(409) + Flag.DEEPAGENT_CODE_WORKSPACE_ID = syncWorkspaceID + }), + { git: true, config: { formatter: false, lsp: false } }, + 15_000, + ) + + it.instance( + "advances an empty workspace cursor to high-water and returns only later scoped events", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + yield* Session.use.create({ title: "unrelated-local" }) + const first = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1 }), + }) + const firstBody = (yield* first.json) as HistoryResponse + expect(firstBody.items).toEqual([]) + expect(firstBody.complete).toBe(true) + + const scoped = yield* Session.use.create({ title: "later-scoped", workspaceID: syncWorkspaceID }) + const second = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1, cursor: firstBody.nextCursor }), + }) + const secondBody = (yield* second.json) as HistoryResponse + expect(secondBody.items.map((item) => item.aggregate_id)).toEqual([scoped.id]) + }), + { git: true, config: { formatter: false, lsp: false } }, + ) + + it.instance( + "rejects oversized known maps in legacy and versioned history payloads", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const known = Object.fromEntries( + Array.from({ length: SyncHistoryLimits.legacyKnownAggregates + 1 }, (_, index) => [`ses_known_${index}`, 0]), + ) + const legacy = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify(known), + }) + const versioned = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1, known }), + }) + + expect(legacy.status).toBe(400) + expect(versioned.status).toBe(400) + }), + { git: true, config: { formatter: false, lsp: false } }, + ) + + it.instance( + "rejects Session identity and placement replay without committing partial state", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "sync-placement-guard", workspaceID: syncWorkspaceID }) + const { db } = yield* Database.Service + const beforeSession = yield* db + .select() + .from(SessionTable) + .where(eq(SessionTable.id, session.id)) + .get() + .pipe(Effect.orDie) + const beforeSequence = yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + const beforeEvents = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie) + const created = beforeEvents.find( + (event) => event.type === EventV2.versionedType(SessionV1.Event.Created.type, 1), + ) + expect(created).toBeDefined() + expect(beforeSession).toBeDefined() + expect(beforeSequence).toBeDefined() + const data = created!.data as { sessionID: string; info: Record } + + const response = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ + directory: tmp.directory, + events: [ + { + id: EventV2.ID.make("evt_http_replay_project_change"), + aggregateID: session.id, + seq: beforeSequence!.seq + 1, + type: EventV2.versionedType(SessionV1.Event.Updated.type, 1), + data: { ...data, info: { ...data.info, projectID: "project_replay_other" } }, + }, + ], + }), + }) + + expect(response.status, yield* response.text).toBe(409) + expect( + yield* db.select().from(SessionTable).where(eq(SessionTable.id, session.id)).get().pipe(Effect.orDie), + ).toEqual(beforeSession) + expect( + yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie), + ).toEqual(beforeSequence) + expect( + yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie), + ).toEqual(beforeEvents) }), { git: true, config: { formatter: false, lsp: false } }, 15_000, ) + it.instance( + "rejects fresh replay placement that is self-consistent but outside the routed instance", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "routed replay authority", workspaceID: syncWorkspaceID }) + const { db } = yield* Database.Service + const created = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + const otherDirectory = `${tmp.directory}-other` + const otherProjectID = session.projectID + yield* db.delete(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, session.id)).run().pipe(Effect.orDie) + yield* db.delete(SessionTable).where(eq(SessionTable.id, session.id)).run().pipe(Effect.orDie) + const data = created!.data as { sessionID: string; info: Record } + const response = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ + directory: otherDirectory, + events: [ + { + id: created!.id, + aggregateID: session.id, + seq: 0, + type: created!.type, + data: { + ...data, + info: { ...data.info, projectID: otherProjectID, directory: otherDirectory }, + }, + }, + ], + }), + }) + + expect(response.status).toBe(409) + expect(yield* db.select().from(SessionTable).where(eq(SessionTable.id, session.id)).get()).toBeUndefined() + expect( + yield* db.select().from(EventTable).where(eq(EventTable.aggregate_id, session.id)).all(), + ).toEqual([]) + }), + { git: true, config: { formatter: false, lsp: false } }, + ) + it.instance( "bounds legacy message.updated rows in sync history", () => Effect.gen(function* () { const tmp = yield* TestInstance const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } - const session = yield* Session.use.create({ title: "sync-legacy-diff" }) + const session = yield* Session.use.create({ title: "sync-legacy-diff", workspaceID: syncWorkspaceID }) yield* Session.use.updateMessage({ id: MessageID.ascending(), sessionID: session.id, @@ -120,9 +402,9 @@ describe("sync HttpApi", () => { headers, body: JSON.stringify({}), }) - const rows = (yield* response.json) as Array<{ type: string; data: Record }> - const event = rows.find((row) => row.type === "message.updated.1") - const info = event?.data.info as { summary?: { diffs?: Array<{ patch?: string }> } } | undefined + const body = (yield* response.json) as HistoryResponse + const event = body.items.find((row) => row.kind === "event" && row.type === "message.updated.1") + const info = event?.data?.info as { summary?: { diffs?: Array<{ patch?: string }> } } | undefined expect(info?.summary?.diffs).toHaveLength(MessageV2.ClientDiffLimits.files) expect(info?.summary?.diffs?.every((item) => item.patch === undefined)).toBe(true) }), @@ -136,7 +418,7 @@ describe("sync HttpApi", () => { Effect.gen(function* () { const tmp = yield* TestInstance const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } - const session = yield* Session.use.create({ title: "sync-pages" }) + const session = yield* Session.use.create({ title: "sync-pages", workspaceID: syncWorkspaceID }) const extra = SyncHistoryLimits.events + 5 yield* Effect.forEach( Array.from({ length: extra }, (_, index) => index), @@ -157,15 +439,18 @@ describe("sync HttpApi", () => { headers, body: JSON.stringify({}), }) - const firstRows = (yield* first.json) as Array<{ id: string; aggregate_id: string; seq: number }> + const firstBody = (yield* first.json) as HistoryResponse + const firstRows = firstBody.items.filter((row): row is HistoryItem & { id: string; seq: number } => row.kind === "event") expect(firstRows).toHaveLength(SyncHistoryLimits.events) + expect(firstBody.complete).toBe(false) const state = Object.fromEntries(firstRows.map((row) => [row.aggregate_id, row.seq])) const second = yield* requestInDirectory(SyncPaths.history, tmp.directory, { method: "POST", headers, body: JSON.stringify(state), }) - const secondRows = (yield* second.json) as Array<{ id: string; aggregate_id: string; seq: number }> + const secondBody = (yield* second.json) as HistoryResponse + const secondRows = secondBody.items.filter((row): row is HistoryItem & { id: string; seq: number } => row.kind === "event") expect(secondRows.length).toBeGreaterThan(0) expect(secondRows.length).toBeLessThanOrEqual(SyncHistoryLimits.events) expect(new Set([...firstRows, ...secondRows].map((row) => row.id)).size).toBe( @@ -177,7 +462,9 @@ describe("sync HttpApi", () => { headers, body: JSON.stringify(state), }) - expect(yield* done.json).toEqual([]) + const doneBody = (yield* done.json) as HistoryResponse + expect(doneBody.items).toEqual([]) + expect(doneBody.complete).toBe(true) }), { git: true, config: { formatter: false, lsp: false } }, 15_000, @@ -189,7 +476,7 @@ describe("sync HttpApi", () => { Effect.gen(function* () { const tmp = yield* TestInstance const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } - const session = yield* Session.use.create({ title: "sync-legacy-oversized" }) + const session = yield* Session.use.create({ title: "sync-legacy-oversized", workspaceID: syncWorkspaceID }) const { db } = yield* Database.Service const sequence = yield* db .select({ seq: EventSequenceTable.seq }) @@ -204,7 +491,7 @@ describe("sync HttpApi", () => { aggregate_id: session.id, seq: sequence!.seq + 1, type: "sync.legacy.1", - data: { value: "x".repeat(SyncHistoryLimits.bytes + 1) }, + data: { value: "x".repeat(SyncHistoryLimits.dataBytes + 1) }, }) .run() .pipe(Effect.orDie) @@ -228,7 +515,7 @@ describe("sync HttpApi", () => { Effect.gen(function* () { const tmp = yield* TestInstance const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } - const session = yield* Session.use.create({ title: "sync-prefix" }) + const session = yield* Session.use.create({ title: "sync-prefix", workspaceID: syncWorkspaceID }) const { db } = yield* Database.Service const sequence = yield* db .select({ seq: EventSequenceTable.seq }) @@ -243,7 +530,7 @@ describe("sync HttpApi", () => { aggregate_id: session.id, seq: sequence!.seq + 1, type: "sync.legacy.1", - data: { value: "x".repeat(SyncHistoryLimits.bytes + 1) }, + data: { value: "x".repeat(SyncHistoryLimits.dataBytes + 1) }, }) .run() .pipe(Effect.orDie) @@ -253,7 +540,8 @@ describe("sync HttpApi", () => { headers, body: JSON.stringify({}), }) - const rows = (yield* first.json) as Array<{ aggregate_id: string; seq: number }> + const firstBody = (yield* first.json) as HistoryResponse + const rows = firstBody.items.filter((row): row is HistoryItem & { seq: number } => row.kind === "event") expect(first.status).toBe(200) expect(rows).toHaveLength(1) expect(rows[0]?.aggregate_id).toBe(session.id) @@ -272,12 +560,12 @@ describe("sync HttpApi", () => { ) it.instance( - "never crosses a UTF-8 event that exceeds the serialized page budget", + "carries a maximum data-budget UTF-8 event inside the larger wire envelope", () => Effect.gen(function* () { const tmp = yield* TestInstance const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } - const session = yield* Session.use.create({ title: "sync-wire-prefix" }) + const session = yield* Session.use.create({ title: "sync-wire-prefix", workspaceID: syncWorkspaceID }) const { db } = yield* Database.Service const sequence = yield* db .select({ seq: EventSequenceTable.seq }) @@ -286,7 +574,7 @@ describe("sync HttpApi", () => { .get() .pipe(Effect.orDie) expect(sequence).toBeDefined() - const target = SyncHistoryLimits.bytes - 100 + const target = SyncHistoryLimits.dataBytes - 100 const overhead = Buffer.byteLength(JSON.stringify({ value: "" })) const available = target - overhead const unicode = "界".repeat(Math.floor(available / 3)) + "x".repeat(available % 3) @@ -331,19 +619,15 @@ describe("sync HttpApi", () => { headers, body: JSON.stringify({ [session.id]: sequence!.seq }), }) - const rows = (yield* first.json) as Array<{ id: string; seq: number }> + const firstBody = (yield* first.json) as HistoryResponse + const rows = firstBody.items.filter((row): row is HistoryItem & { id: string; seq: number } => row.kind === "event") expect(first.status).toBe(200) - expect(rows.map((row) => row.id)).toEqual(["evt_sync_wire_small_before"]) - - const blocked = yield* requestInDirectory(SyncPaths.history, tmp.directory, { - method: "POST", - headers, - body: JSON.stringify({ [session.id]: sequence!.seq + 1 }), - }) - const body = yield* blocked.text - expect(blocked.status, body).toBe(503) - expect(body).toContain("Projected sync event evt_sync_wire_middle exceeds") - expect(body).not.toContain("evt_sync_wire_small_after") + expect(rows.map((row) => row.id)).toEqual([ + "evt_sync_wire_small_before", + "evt_sync_wire_middle", + "evt_sync_wire_small_after", + ]) + expect(Buffer.byteLength(JSON.stringify(firstBody))).toBeLessThanOrEqual(SyncHistoryLimits.wireBytes) }), { git: true, config: { formatter: false, lsp: false } }, 20_000, @@ -410,7 +694,7 @@ describe("sync HttpApi", () => { () => Effect.gen(function* () { const tmp = yield* TestInstance - const session = yield* Session.use.create({ title: "sync-max-replay" }) + const session = yield* Session.use.create({ title: "sync-max-replay", workspaceID: syncWorkspaceID }) const event = (aggregateID: string, id: EventV2.ID, seq: number) => { const base = { sessionID: aggregateID, @@ -455,7 +739,7 @@ describe("sync HttpApi", () => { }) expect(web.status, yield* web.text).toBe(200) - const nodeSession = yield* Session.use.create({ title: "sync-max-node" }) + const nodeSession = yield* Session.use.create({ title: "sync-max-node", workspaceID: syncWorkspaceID }) const nodeSequence = yield* db .select({ seq: EventSequenceTable.seq }) .from(EventSequenceTable) @@ -481,6 +765,85 @@ describe("sync HttpApi", () => { 30_000, ) + it.instance( + "publishes a maximum admitted event through paged history into a fresh projection", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "sync-max-history", workspaceID: syncWorkspaceID }) + const messageID = MessageID.ascending() + const base = { + sessionID: session.id, + info: { + id: messageID, + sessionID: session.id, + role: "user" as const, + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + system: "", + }, + } + const remaining = EventV2.MAX_ENCODED_PAYLOAD_BYTES - Buffer.byteLength(JSON.stringify(base)) + const message = { ...base.info, system: "x".repeat(remaining) } + expect(Buffer.byteLength(JSON.stringify({ sessionID: session.id, info: message }))).toBe( + EventV2.MAX_ENCODED_PAYLOAD_BYTES, + ) + yield* Session.use.updateMessage(message) + + const pages = new Array() + let cursor: string | undefined + while (true) { + const response = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1, ...(cursor ? { cursor } : {}) }), + }) + const text = yield* response.text + expect(response.status, text).toBe(200) + const page = JSON.parse(text) as HistoryResponse + expect(Buffer.byteLength(text)).toBeLessThanOrEqual(SyncHistoryLimits.wireBytes) + pages.push(page) + cursor = page.nextCursor + if (page.complete) break + } + const history = pages.flatMap((page) => page.items).filter((item) => item.aggregate_id === session.id) + expect(history.map((item) => item.type)).toEqual(["session.created.1", "message.updated.1"]) + expect(Buffer.byteLength(JSON.stringify(history[1]!.data))).toBe(EventV2.MAX_ENCODED_PAYLOAD_BYTES) + + const { db } = yield* Database.Service + yield* db.delete(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, session.id)).run().pipe(Effect.orDie) + yield* db.delete(SessionTable).where(eq(SessionTable.id, session.id)).run().pipe(Effect.orDie) + for (const page of pages) { + const events = page.items + .filter((item) => item.aggregate_id === session.id) + .map((item) => ({ + id: item.id!, + aggregateID: item.aggregate_id, + seq: item.seq!, + type: item.type!, + data: item.data!, + })) + if (events.length === 0) continue + const replay = yield* requestInDirectory(SyncPaths.replay, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ directory: tmp.directory, events }), + }) + expect(replay.status, yield* replay.text).toBe(200) + } + + expect((yield* Session.use.get(session.id)).title).toBe("sync-max-history") + expect((yield* Session.use.getClientMessage({ sessionID: session.id, messageID })).info).toMatchObject({ + id: messageID, + system: message.system, + }) + }), + { git: true, config: { formatter: false, lsp: false } }, + 30_000, + ) + it.instance( "rejects an unbounded replay stream before reading or decoding the full body", () => diff --git a/packages/deepagent-code/test/server/httpapi-workspace.test.ts b/packages/deepagent-code/test/server/httpapi-workspace.test.ts index 0f6345b7b..8d7a67240 100644 --- a/packages/deepagent-code/test/server/httpapi-workspace.test.ts +++ b/packages/deepagent-code/test/server/httpapi-workspace.test.ts @@ -534,6 +534,25 @@ describe("workspace HttpApi", () => { const workspace = (yield* created.json) as Workspace.Info const sessionResponse = yield* requestDefault("/session", dir, { method: "POST" }) const session = (yield* sessionResponse.json) as Session.Info + const { db } = yield* Database.Service + const beforeSession = yield* db + .select() + .from(SessionTable) + .where(eq(SessionTable.id, session.id)) + .get() + .pipe(Effect.orDie) + const beforeSequence = yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + const beforeEvents = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie) const warped = yield* requestDefault(WorkspacePaths.warp, dir, { method: "POST", headers: { "content-type": "application/json" }, @@ -546,6 +565,73 @@ describe("workspace HttpApi", () => { resource: `session:${session.id}`, }) + const replayed = yield* requestDefault(`${SyncPaths.replay}?workspace=${workspace.id}`, dir, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + directory: dir, + events: [ + { + id: "evt_remote_replay_disabled", + aggregateID: session.id, + seq: 1, + type: "session.next.moved.1", + data: { + sessionID: session.id, + timestamp: 1, + location: { directory: path.join(dir, ".remote-session"), workspaceID: workspace.id }, + }, + }, + ], + }), + }) + expect(replayed.status).toBe(409) + expect( + yield* db.select().from(SessionTable).where(eq(SessionTable.id, session.id)).get().pipe(Effect.orDie), + ).toEqual(beforeSession) + expect( + yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie), + ).toEqual(beforeSequence) + expect( + yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie), + ).toEqual(beforeEvents) + + const stolen = yield* requestDefault(`${SyncPaths.steal}?workspace=${workspace.id}`, dir, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sessionID: session.id }), + }) + expect(stolen.status).toBe(409) + expect( + yield* db.select().from(SessionTable).where(eq(SessionTable.id, session.id)).get().pipe(Effect.orDie), + ).toEqual(beforeSession) + expect( + yield* db + .select() + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie), + ).toEqual(beforeSequence) + expect( + yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie), + ).toEqual(beforeEvents) + try { expect( proxied.filter((item) => diff --git a/packages/deepagent-code/test/server/workspace-routing.test.ts b/packages/deepagent-code/test/server/workspace-routing.test.ts index 9ae0f3e63..569d95322 100644 --- a/packages/deepagent-code/test/server/workspace-routing.test.ts +++ b/packages/deepagent-code/test/server/workspace-routing.test.ts @@ -7,6 +7,14 @@ import { import { SessionID } from "../../src/session/schema" describe("isLocalWorkspaceRoute", () => { + test("keeps transfer-sensitive sync mutations local only at their exact POST paths", () => { + expect(isLocalWorkspaceRoute("POST", "/sync/replay")).toBe(true) + expect(isLocalWorkspaceRoute("POST", "/sync/steal")).toBe(true) + expect(isLocalWorkspaceRoute("GET", "/sync/replay")).toBe(false) + expect(isLocalWorkspaceRoute("POST", "/sync/replay/child")).toBe(false) + expect(isLocalWorkspaceRoute("POST", "/sync/steal/child")).toBe(false) + }) + test("GET /session is local", () => { expect(isLocalWorkspaceRoute("GET", "/session")).toBe(true) }) diff --git a/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts b/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts index 9869b73c6..214d48dec 100644 --- a/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts +++ b/packages/deepagent-code/test/session/legacy-provider-resolution.test.ts @@ -16,7 +16,17 @@ import { import { and, eq } from "drizzle-orm" import { Effect, Layer } from "effect" import { EventV2Bridge } from "@/event-v2-bridge" -import { EventTable } from "@deepagent-code/core/event/sql" +import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { + SessionActivityTable, + SessionContextSelectionTable, + SessionProviderAttemptRecoveryBridgeTable, + SessionProviderAttemptResolutionTable, + SessionProviderAttemptTable, +} from "@deepagent-code/core/context-federation/session-sql" +import { Prompt } from "@deepagent-code/core/session/prompt" +import { SessionMessage } from "@deepagent-code/core/session/message" +import { SessionInputTable } from "@deepagent-code/core/session/sql" import { SessionLegacyProviderResolution } from "@/session/legacy-provider-resolution" import { MessageV2 } from "@/session/message-v2" import { SessionPromptEpochTable } from "@/session/prompt-epoch.sql" @@ -113,8 +123,81 @@ it.instance( const baselineHash = authority.worldStateBaselineHash const requestHash = "final-request-hash" const receiptID = "receipt-ambiguous-provider-turn" + const providerAttemptID = "attempt-ambiguous-provider-turn" + const providerActivityID = "activity-ambiguous-provider-turn" + const providerSelectionID = "selection-ambiguous-provider-turn" const now = Date.now() + yield* db + .insert(SessionInputTable) + .values({ + id: SessionMessage.ID.make(user.id), + session_id: source.id, + prompt: new Prompt({ text: "dispatch then crash" }), + delivery: "steer", + admitted_seq: 0, + promoted_seq: 0, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionActivityTable) + .values({ + activity_id: providerActivityID, + session_id: source.id, + ordinal: 0, + trigger_input_id: user.id, + delivery: "steer", + state: "active", + created_at: now, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionContextSelectionTable) + .values({ + selection_id: providerSelectionID, + session_id: source.id, + activity_id: providerActivityID, + revision: 0, + trigger_input_id: user.id, + location_key: "local", + query_fingerprint: "query", + authorization_fingerprint: "authorization", + authorization_epoch: 0, + execution_fingerprint: "execution", + selected_source_fingerprint: "sources", + observed_location_mutation_epoch: 0, + next_revalidation_at: now + 60_000, + graph_revisions: "{}", + graph_statuses: "[]", + selected_refs: "[]", + projection: "", + projection_hash: "projection", + token_count: 0, + artifact_write_status: "degraded_unavailable", + inline_audit: "{}", + created_at: now, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionProviderAttemptTable) + .values({ + attempt_id: providerAttemptID, + session_id: source.id, + activity_id: providerActivityID, + provider_turn_seq: 0, + selection_id: providerSelectionID, + projection_hash: "projection", + request_hash: requestHash, + provider_id: model.providerID, + state: "indeterminate_after_crash", + created_at: now, + error_code: "process_recovery", + }) + .run() + .pipe(Effect.orDie) yield* db .insert(SessionToolRequestReceiptTable) .values({ @@ -123,7 +206,7 @@ it.instance( session_id: source.id, user_message_id: user.id, assistant_message_id: assistant.id, - provider_attempt_id: null, + provider_attempt_id: providerAttemptID, provider_id: model.providerID, model_id: model.modelID, protocol: "chat", @@ -147,30 +230,6 @@ it.instance( }) .run() .pipe(Effect.orDie) - yield* db - .insert(SessionToolRequestReceiptTable) - .values({ - receipt_id: "legacy-stale-indeterminate-without-prompt-authority", - request_ordinal: 2, - session_id: source.id, - user_message_id: safeUser.id, - assistant_message_id: safeAssistant.id, - provider_attempt_id: null, - provider_id: model.providerID, - model_id: model.modelID, - protocol: "chat", - registry_tool_ids: [], - permission_filtered_tool_ids: [], - final_offered_tool_ids: [], - call_ids: [], - provider_state: "indeterminate_after_crash", - terminal_at: now, - request_state: "dispatched", - request_error_code: "legacy_migration_without_prompt_authority", - created_at: now - 1, - }) - .run() - .pipe(Effect.orDie) yield* db .update(SessionPromptEpochTable) .set({ authority_state: "recovery_required", recovery_reason: "provider outcome is unknown after restart" }) @@ -257,6 +316,10 @@ it.instance( worldStateBaselineHash: baselineHash, }) if (!descriptor) return + if (!descriptor.resolutionSupported) + return yield* Effect.die( + `expected a resolvable provider recovery descriptor: ${descriptor.unsupportedReasons.join(",")}`, + ) if (!descriptor.worldStateBaselineHash) return yield* Effect.die("expected a resolvable provider recovery descriptor") const command = { @@ -289,12 +352,12 @@ it.instance( const assertRecoveryUnchanged = Effect.fnUntraced(function* () { expect( yield* db - .select() + .select({ result: SessionToolRequestResolutionCommandTable.result_resolution_id }) .from(SessionToolRequestResolutionCommandTable) .where(eq(SessionToolRequestResolutionCommandTable.session_id, source.id)) .all() .pipe(Effect.orDie), - ).toEqual([]) + ).not.toContainEqual(expect.objectContaining({ result: expect.any(String) })) expect( yield* db .select() @@ -383,9 +446,23 @@ it.instance( .pipe(Effect.orDie) yield* db.insert(SessionWorldStateBaselineTable).values(sourceBaseline).run().pipe(Effect.orDie) + const sequenceBefore = yield* db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, source.id)) + .get() + .pipe(Effect.orDie) const first = yield* recovery.resolve(command) const retry = yield* recovery.resolve(command) expect(retry).toEqual(first) + expect( + yield* db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, source.id)) + .get() + .pipe(Effect.orDie), + ).toEqual(sequenceBefore) expect( yield* db .select({ id: EventTable.id }) @@ -493,6 +570,50 @@ it.instance( .get() .pipe(Effect.orDie), ).toBeDefined() + expect( + yield* db + .select() + .from(SessionProviderAttemptRecoveryBridgeTable) + .where(eq(SessionProviderAttemptRecoveryBridgeTable.resolution_id, first.resolutionID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ + resolution_id: first.resolutionID, + attempt_id: providerAttemptID, + receipt_id: receiptID, + command_id: command.commandID, + }) + expect( + yield* db + .select() + .from(SessionProviderAttemptResolutionTable) + .where(eq(SessionProviderAttemptResolutionTable.attempt_id, providerAttemptID)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ resolution_id: first.resolutionID, decision: "abandoned" }) + expect( + (yield* db + .delete(SessionProviderAttemptResolutionTable) + .where(eq(SessionProviderAttemptResolutionTable.attempt_id, providerAttemptID)) + .run() + .pipe(Effect.orDie, Effect.exit))._tag, + ).toBe("Failure") + expect( + yield* db + .select({ state: SessionProviderAttemptTable.state }) + .from(SessionProviderAttemptTable) + .where(eq(SessionProviderAttemptTable.attempt_id, providerAttemptID)) + .get() + .pipe(Effect.orDie), + ).toEqual({ state: "resolved_abandoned" }) + expect( + yield* db + .select({ state: SessionActivityTable.state }) + .from(SessionActivityTable) + .where(eq(SessionActivityTable.activity_id, providerActivityID)) + .get() + .pipe(Effect.orDie), + ).toEqual({ state: "interrupted" }) expect( yield* db .select() @@ -569,3 +690,122 @@ it.instance( { git: true }, 30_000, ) + +it.instance( + "keeps orphaned unknown outcomes visible and blocks every cutoff after their user boundary", + () => + Effect.gen(function* () { + const sessions = yield* Session.Service + const recovery = yield* SessionLegacyProviderResolution.Service + yield* TestInstance + const { db } = yield* Database.Service + const source = yield* sessions.create({ title: "orphaned provider outcome" }) + const before = yield* addUser(source.id, "before orphan") + const beforeAssistant = yield* addAssistant(source.id, before.id, true) + const orphanUser = yield* addUser(source.id, "dispatch before legacy migration") + const orphanAssistant = yield* addAssistant(source.id, orphanUser.id) + const tail = yield* addUser(source.id, "after orphan") + const authority = yield* MessageV2.promptHistoryProjectionEffect(source.id) + const now = Date.now() + yield* db + .insert(SessionToolRequestReceiptTable) + .values({ + receipt_id: "legacy-orphan-without-prompt-authority", + request_ordinal: 1, + session_id: source.id, + user_message_id: orphanUser.id, + assistant_message_id: orphanAssistant.id, + provider_attempt_id: null, + provider_id: model.providerID, + model_id: model.modelID, + protocol: "chat", + registry_tool_ids: [], + permission_filtered_tool_ids: [], + final_offered_tool_ids: [], + call_ids: [], + provider_state: "indeterminate_after_crash", + terminal_at: now, + request_state: "dispatched", + request_error_code: "legacy_migration_without_prompt_authority", + created_at: now, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(SessionPromptEpochTable) + .set({ authority_state: "recovery_required", recovery_reason: "provider outcome is unknown after restart" }) + .where( + and(eq(SessionPromptEpochTable.session_id, source.id), eq(SessionPromptEpochTable.epoch, authority.epoch)), + ) + .run() + .pipe(Effect.orDie) + yield* db + .update(SessionHistoryStateTable) + .set({ + state: "recovery_required", + reason: "provider outcome is unknown after restart", + time_updated: now, + }) + .where(eq(SessionHistoryStateTable.session_id, source.id)) + .run() + .pipe(Effect.orDie) + + const safe = yield* MessageV2.promptHistoryCutoffProjectionEffect({ + sessionID: source.id, + cutoffMessageID: orphanUser.id, + }) + expect(safe.messages.map((message) => message.info.id)).toEqual([before.id, beforeAssistant.id]) + const fork = yield* sessions.fork({ + sessionID: source.id, + messageID: orphanUser.id, + intentID: "fork-before-orphaned-provider-outcome", + }) + expect((yield* sessions.messages({ sessionID: fork.id })).map((message) => message.info.role)).toEqual([ + "user", + "assistant", + ]) + for (const cutoffMessageID of [orphanAssistant.id, tail.id]) { + expect( + yield* sessions + .fork({ + sessionID: source.id, + messageID: cutoffMessageID, + intentID: `fork-after-orphaned-provider-outcome-${cutoffMessageID}`, + }) + .pipe(Effect.flip), + ).toMatchObject({ + _tag: "Session.ForkConflict", + reason: expect.stringContaining("fork cutoff is outside the recoverable history prefix"), + }) + } + + expect(yield* recovery.describe(source.id)).toEqual([ + expect.objectContaining({ + receiptID: "legacy-orphan-without-prompt-authority", + resolutionSupported: false, + unsupportedReasons: expect.arrayContaining([ + "legacy_receipt_authority_incomplete", + "source_world_state_baseline_missing", + ]), + }), + ]) + expect( + yield* db + .select() + .from(SessionToolRequestResolutionCommandTable) + .where(eq(SessionToolRequestResolutionCommandTable.session_id, source.id)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + expect( + yield* db + .select() + .from(SessionToolRequestResolutionTable) + .where(eq(SessionToolRequestResolutionTable.session_id, source.id)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + }), + { git: true }, + 30_000, +) diff --git a/packages/deepagent-code/test/session/prompt-intent.test.ts b/packages/deepagent-code/test/session/prompt-intent.test.ts index 3ec29d589..199f31aee 100644 --- a/packages/deepagent-code/test/session/prompt-intent.test.ts +++ b/packages/deepagent-code/test/session/prompt-intent.test.ts @@ -769,10 +769,11 @@ describe("SessionPromptIntent", () => { const first = yield* claim({ intentID: "intent_progress", messageID: MessageID.make("msg_progress_user") }) expect(first.kind).toBe("claimed") if (first.kind !== "claimed") return - yield* SessionPromptIntent.materializeTurn({ + const materialized = yield* SessionPromptIntent.materializeTurn({ receipt: first.receipt, message: message(first.receipt.messageID), }) + if (!("run" in materialized) || !materialized.run) return const activity = yield* SessionPromptIntent.activityForMessage({ sessionID, messageID: first.receipt.messageID, @@ -850,6 +851,7 @@ describe("SessionPromptIntent", () => { assistantMessageID: assistantID, providerReceiptID: "receipt-progress-final", }) + yield* SessionPromptIntent.markRunFinalizing(materialized.run) expect(yield* SessionPromptIntent.recoverActiveActivities()).toEqual([]) expect(yield* SessionPromptIntent.recoverActiveActivities("next-process-owner")).toEqual([ diff --git a/packages/deepagent-code/test/session/prompt.test.ts b/packages/deepagent-code/test/session/prompt.test.ts index cf0ae1a61..7108b511f 100644 --- a/packages/deepagent-code/test/session/prompt.test.ts +++ b/packages/deepagent-code/test/session/prompt.test.ts @@ -4421,6 +4421,112 @@ it.instance( 15_000, ) +it.instance( + "cancel keeps a finalizing follow-up busy until durable teardown completes", + () => + Effect.gen(function* () { + const { dir, llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const steer = yield* SessionSteer.Service + const runState = yield* SessionRunState.Service + const status = yield* SessionStatus.Service + const { db } = yield* Database.Service + const session = yield* sessions.create({ + title: "Finalizing cancel lifecycle", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + const gate = yield* Deferred.make() + const marker = path.join(dir, "activity-finalizing-followup.json") + const previousPoint = process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT + const previousRoot = process.env.DEEPAGENT_CODE_TEST_ROOT + const previousMarker = process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER + + const admitted = yield* Effect.acquireUseRelease( + Effect.sync(() => { + process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT = "while_finalizing_before_follow_up_drain" + process.env.DEEPAGENT_CODE_TEST_ROOT = dir + process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER = marker + }), + () => + Effect.gen(function* () { + yield* llm.hold("first answer", deferredAsPromise(gate)) + const running = yield* prompt + .prompt({ + sessionID: session.id, + agent: "build", + parts: [{ type: "text", text: "finish, then absorb my follow-up" }], + }) + .pipe(Effect.forkChild) + yield* llm.wait(1) + const admitted = yield* steer.admit({ + sessionID: session.id, + prompt: new Prompt({ text: "cancel before this follow-up drains" }), + }) + yield* Deferred.succeed(gate, undefined) + yield* pollWithTimeout( + Effect.promise(async () => ((await Bun.file(marker).exists()) ? true : undefined)), + "timed out waiting for finalizing follow-up boundary", + "5 seconds", + ) + expect(yield* runState.isBusy(session.id)).toBe(true) + expect((yield* status.get(session.id)).type).toBe("busy") + expect(yield* llm.calls).toBe(1) + + yield* awaitWithTimeout(prompt.cancel(session.id), "timed out cancelling finalizing run", "5 seconds") + yield* awaitWithTimeout(Fiber.await(running), "timed out joining finalizing run", "5 seconds") + expect(yield* runState.isBusy(session.id)).toBe(false) + expect((yield* status.get(session.id)).type).toBe("idle") + return admitted + }), + () => + Effect.sync(() => { + if (previousPoint === undefined) delete process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT + else process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_POINT = previousPoint + if (previousRoot === undefined) delete process.env.DEEPAGENT_CODE_TEST_ROOT + else process.env.DEEPAGENT_CODE_TEST_ROOT = previousRoot + if (previousMarker === undefined) delete process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER + else process.env.DEEPAGENT_CODE_TEST_ACTIVITY_CRASH_MARKER = previousMarker + }), + ) + + expect(yield* llm.calls).toBe(1) + expect( + yield* db + .select() + .from(SessionLegacyActivityTable) + .where(eq(SessionLegacyActivityTable.session_id, session.id)) + .all() + .pipe(Effect.orDie), + ).toMatchObject([{ state: "interrupted", terminal_reason: "user_cancelled" }]) + expect( + yield* db + .select() + .from(SessionLegacyActivityRunTable) + .where(eq(SessionLegacyActivityRunTable.session_id, session.id)) + .all() + .pipe(Effect.orDie), + ).toMatchObject([{ state: "interrupted", terminal_reason: "user_cancelled" }]) + expect( + yield* db + .select() + .from(SessionLegacyActivityTerminalTable) + .where(eq(SessionLegacyActivityTerminalTable.session_id, session.id)) + .all() + .pipe(Effect.orDie), + ).toMatchObject([{ state: "interrupted", reason_code: "user_cancelled", source: "cancel" }]) + expect( + yield* db + .select() + .from(SessionSteerTable) + .where(eq(SessionSteerTable.id, admitted.id)) + .get() + .pipe(Effect.orDie), + ).toMatchObject({ consumed_seq: null, superseded_at: expect.any(Number) }) + }), + 20_000, +) + // Agent variant noLLMServer.instance( diff --git a/packages/deepagent-code/test/session/session.test.ts b/packages/deepagent-code/test/session/session.test.ts index 24ae2f7ac..a361d9e60 100644 --- a/packages/deepagent-code/test/session/session.test.ts +++ b/packages/deepagent-code/test/session/session.test.ts @@ -3,6 +3,7 @@ import path from "path" import { SessionV1 } from "@deepagent-code/core/v1/session" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" +import { EventTable } from "@deepagent-code/core/event/sql" import { ModelV2 } from "@deepagent-code/core/model" import { ProviderV2 } from "@deepagent-code/core/provider" import { SessionProjector } from "@deepagent-code/core/session/projector" @@ -25,6 +26,8 @@ import { Worktree } from "@/worktree" import { Git } from "../../src/git" import { DeepAgentContext, DeepAgentDocumentStore } from "@deepagent-code/core/deepagent/index" import { contextStoreRoot, loadForkOrigin, forwardLedgerOnFork } from "@/session/context-ledger" +import { WorkspaceV2 } from "@deepagent-code/core/workspace" +import { eq } from "drizzle-orm" void Log.init({ print: false }) @@ -38,6 +41,7 @@ const it = testEffect( Layer.provide(RuntimeFlags.layer({ experimentalWorkspaces: false })), Layer.provide(BackgroundJob.defaultLayer), ), + Database.defaultLayer, CrossSpawnSpawner.defaultLayer, testInstanceStoreLayer, ), @@ -415,6 +419,65 @@ describe("Session", () => { }), ) + it.instance("rejects workspace placement changes without writing Session or EventV2 state", () => + Effect.gen(function* () { + const sessions = yield* SessionNs.Service + const { db } = yield* Database.Service + const workspaceID = WorkspaceV2.ID.make("wrk_session_placement_guard") + const workspaceSession = yield* Effect.acquireRelease( + sessions.create({ title: "workspace placement guard", workspaceID }), + (info) => sessions.remove(info.id).pipe(Effect.ignore), + ) + const localSession = yield* Effect.acquireRelease(sessions.create({ title: "local placement guard" }), (info) => + sessions.remove(info.id).pipe(Effect.ignore), + ) + const beforeWorkspace = yield* sessions.get(workspaceSession.id) + const beforeLocal = yield* sessions.get(localSession.id) + const beforeWorkspaceEvents = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, workspaceSession.id)) + .all() + .pipe(Effect.orDie) + const beforeLocalEvents = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, localSession.id)) + .all() + .pipe(Effect.orDie) + + const directoryError = yield* sessions + .setDirectory({ sessionID: workspaceSession.id, directory: path.join(workspaceSession.directory, "other") }) + .pipe(Effect.flip) + const workspaceError = yield* sessions + .setWorkspace({ sessionID: localSession.id, workspaceID }) + .pipe(Effect.flip) + + expect(directoryError).toBeInstanceOf(SessionNs.PlacementChangeUnsupportedError) + expect(directoryError.operation).toBe("directory") + expect(workspaceError).toBeInstanceOf(SessionNs.PlacementChangeUnsupportedError) + expect(workspaceError.operation).toBe("workspace") + expect(yield* sessions.get(workspaceSession.id)).toEqual(beforeWorkspace) + expect(yield* sessions.get(localSession.id)).toEqual(beforeLocal) + expect( + yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, workspaceSession.id)) + .all() + .pipe(Effect.orDie), + ).toEqual(beforeWorkspaceEvents) + expect( + yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, localSession.id)) + .all() + .pipe(Effect.orDie), + ).toEqual(beforeLocalEvents) + }), + ) + it.instance("omits metadata when not provided", () => Effect.gen(function* () { const session = yield* SessionNs.Service diff --git a/packages/sdk/js/src/gen/sdk.gen.ts b/packages/sdk/js/src/gen/sdk.gen.ts index 2a33b6a15..b8305cc02 100644 --- a/packages/sdk/js/src/gen/sdk.gen.ts +++ b/packages/sdk/js/src/gen/sdk.gen.ts @@ -442,8 +442,6 @@ import type { SyncReplayResponses, SyncStartErrors, SyncStartResponses, - SyncStealErrors, - SyncStealResponses, TextPartInput, ToolIdsErrors, ToolIdsResponses, @@ -8466,15 +8464,23 @@ export class History extends HeyApiClient { /** * List sync events * - * List a bounded page of sync events. Keys are aggregate IDs the client already knows about, values are the last known sequence ID. Oversized legacy events fail closed until maintenance migration externalizes their inline payload. + * List a bounded page with an opaque global cursor. A bounded legacy aggregate map is accepted only for the first upgrade request. Retention-floor crossings return resync_required with a canonical snapshot. */ public list( parameters?: { directory?: string workspace?: string - body?: { - [key: string]: number - } + body?: + | { + [key: string]: number + } + | { + version: 1 + cursor?: string + known?: { + [key: string]: number + } + } }, options?: Options, ) { @@ -8589,43 +8595,6 @@ export class Sync extends HeyApiClient { }) } - /** - * Steal session into workspace - * - * Update a session to belong to the current workspace through the sync event system. - */ - public steal( - parameters: { - directory?: string - workspace?: string - sessionID: string - }, - options?: Options, - ) { - const params = buildClientParams( - [parameters], - [ - { - args: [ - { in: "query", key: "directory" }, - { in: "query", key: "workspace" }, - { in: "body", key: "sessionID" }, - ], - }, - ], - ) - return (options?.client ?? this.client).post({ - url: "/sync/steal", - ...options, - ...params, - headers: { - "Content-Type": "application/json", - ...options?.headers, - ...params.headers, - }, - }) - } - private _history?: History get history(): History { return (this._history ??= new History({ client: this.client })) diff --git a/packages/sdk/js/src/gen/types.gen.ts b/packages/sdk/js/src/gen/types.gen.ts index b853e4d88..f303c8eb5 100644 --- a/packages/sdk/js/src/gen/types.gen.ts +++ b/packages/sdk/js/src/gen/types.gen.ts @@ -277,6 +277,13 @@ export type OutputFormatJsonSchema = { export type OutputFormat = OutputFormatText | OutputFormatJsonSchema +export type SnapshotDiffArtifactDescriptor = { + id: string + hash: string + codec: "legacy-message-diff.v1" + fileCount: number +} + export type UserMessage = { id: string sessionID: string @@ -290,6 +297,7 @@ export type UserMessage = { body?: string diffs: Array diffManifest?: SnapshotDiffManifestDescriptor + diffArtifact?: SnapshotDiffArtifactDescriptor } agent: string model: { @@ -5373,6 +5381,57 @@ export type ProjectCopyCopy = { directory: string } +export type SessionLegacyProviderResolutionDescriptor = + | { + receiptID: string + sessionID: string + providerID: string + modelID: string + providerState: "indeterminate_after_crash" + sessionMutationEpoch: number + resolutionSupported: true + unsupportedReasons: Array + assistantMessageID: string + promptEpoch: number + promptWindowID: string + historyHash: string + requestHash: string + continuationRecoverySupported: boolean + workspaceRecoverySupported: boolean + sourceWorldStateBaselineStatus: "available" + worldStateBaselineHash: string + } + | { + receiptID: string + sessionID: string + providerID: string + modelID: string + providerState: "indeterminate_after_crash" + sessionMutationEpoch: number + resolutionSupported: false + unsupportedReasons: Array< + | "legacy_receipt_authority_incomplete" + | "source_prompt_epoch_missing" + | "source_prompt_epoch_not_recovery_required" + | "source_prompt_epoch_binding_mismatch" + | "provider_attempt_authority_incomplete" + | "provider_attempt_resolution_requires_reconciliation" + | "compaction_continuation_requires_maintenance" + | "workspace_recovery_requires_coordination" + | "source_world_state_baseline_missing" + | "source_world_state_baseline_invalid" + > + assistantMessageID?: string + promptEpoch?: number + promptWindowID?: string + historyHash?: string + requestHash?: string + continuationRecoverySupported: boolean + workspaceRecoverySupported: boolean + sourceWorldStateBaselineStatus: "available" | "missing" | "invalid" + worldStateBaselineHash?: string + } + export type LocationInfo = { directory: string workspaceID?: string @@ -15030,23 +15089,7 @@ export type SessionProviderResolutionListResponses = { /** * Pending provider recovery decisions */ - 200: Array<{ - receiptID: string - sessionID: string - assistantMessageID: string - providerID: string - modelID: string - providerState: "indeterminate_after_crash" - promptEpoch: number - promptWindowID: string - historyHash: string - requestHash: string - sessionMutationEpoch: number - continuationRecoverySupported: boolean - workspaceRecoverySupported: boolean - sourceWorldStateBaselineStatus: "available" | "missing" | "invalid" - worldStateBaselineHash?: string - }> + 200: Array } export type SessionProviderResolutionListResponse = @@ -15175,6 +15218,10 @@ export type SyncReplayErrors = { * BadRequest | InvalidRequestError */ 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * ConflictError + */ + 409: ConflictError } export type SyncReplayError = SyncReplayErrors[keyof SyncReplayErrors] @@ -15190,19 +15237,27 @@ export type SyncReplayResponses = { export type SyncReplayResponse = SyncReplayResponses[keyof SyncReplayResponses] -export type SyncStealData = { - body?: { - sessionID: string - } +export type SyncHistoryListData = { + body?: + | { + [key: string]: number + } + | { + version: 1 + cursor?: string + known?: { + [key: string]: number + } + } path?: never query?: { directory?: string workspace?: string } - url: "/sync/steal" + url: "/sync/history" } -export type SyncStealErrors = { +export type SyncHistoryListErrors = { /** * BadRequest | InvalidRequestError */ @@ -15211,38 +15266,6 @@ export type SyncStealErrors = { * ConflictError */ 409: ConflictError -} - -export type SyncStealError = SyncStealErrors[keyof SyncStealErrors] - -export type SyncStealResponses = { - /** - * Session stolen into workspace - */ - 200: { - sessionID: string - } -} - -export type SyncStealResponse = SyncStealResponses[keyof SyncStealResponses] - -export type SyncHistoryListData = { - body?: { - [key: string]: number - } - path?: never - query?: { - directory?: string - workspace?: string - } - url: "/sync/history" -} - -export type SyncHistoryListErrors = { - /** - * BadRequest | InvalidRequestError - */ - 400: EffectHttpApiErrorBadRequest | InvalidRequestError /** * ServiceUnavailableError */ @@ -15253,17 +15276,23 @@ export type SyncHistoryListError = SyncHistoryListErrors[keyof SyncHistoryListEr export type SyncHistoryListResponses = { /** - * Sync events + * Bounded sync history page */ - 200: Array<{ - id: string - aggregate_id: string - seq: number - type: string - data: { - [key: string]: unknown - } - }> + 200: { + version: 1 + items: Array<{ + kind: "event" + id: string + aggregate_id: string + seq: number + type: string + data: { + [key: string]: unknown + } + }> + nextCursor: string + complete: boolean + } } export type SyncHistoryListResponse = SyncHistoryListResponses[keyof SyncHistoryListResponses] diff --git a/packages/sdk/js/src/v2/gen/sdk.gen.ts b/packages/sdk/js/src/v2/gen/sdk.gen.ts index 2a33b6a15..b8305cc02 100644 --- a/packages/sdk/js/src/v2/gen/sdk.gen.ts +++ b/packages/sdk/js/src/v2/gen/sdk.gen.ts @@ -442,8 +442,6 @@ import type { SyncReplayResponses, SyncStartErrors, SyncStartResponses, - SyncStealErrors, - SyncStealResponses, TextPartInput, ToolIdsErrors, ToolIdsResponses, @@ -8466,15 +8464,23 @@ export class History extends HeyApiClient { /** * List sync events * - * List a bounded page of sync events. Keys are aggregate IDs the client already knows about, values are the last known sequence ID. Oversized legacy events fail closed until maintenance migration externalizes their inline payload. + * List a bounded page with an opaque global cursor. A bounded legacy aggregate map is accepted only for the first upgrade request. Retention-floor crossings return resync_required with a canonical snapshot. */ public list( parameters?: { directory?: string workspace?: string - body?: { - [key: string]: number - } + body?: + | { + [key: string]: number + } + | { + version: 1 + cursor?: string + known?: { + [key: string]: number + } + } }, options?: Options, ) { @@ -8589,43 +8595,6 @@ export class Sync extends HeyApiClient { }) } - /** - * Steal session into workspace - * - * Update a session to belong to the current workspace through the sync event system. - */ - public steal( - parameters: { - directory?: string - workspace?: string - sessionID: string - }, - options?: Options, - ) { - const params = buildClientParams( - [parameters], - [ - { - args: [ - { in: "query", key: "directory" }, - { in: "query", key: "workspace" }, - { in: "body", key: "sessionID" }, - ], - }, - ], - ) - return (options?.client ?? this.client).post({ - url: "/sync/steal", - ...options, - ...params, - headers: { - "Content-Type": "application/json", - ...options?.headers, - ...params.headers, - }, - }) - } - private _history?: History get history(): History { return (this._history ??= new History({ client: this.client })) diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index b853e4d88..f303c8eb5 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -277,6 +277,13 @@ export type OutputFormatJsonSchema = { export type OutputFormat = OutputFormatText | OutputFormatJsonSchema +export type SnapshotDiffArtifactDescriptor = { + id: string + hash: string + codec: "legacy-message-diff.v1" + fileCount: number +} + export type UserMessage = { id: string sessionID: string @@ -290,6 +297,7 @@ export type UserMessage = { body?: string diffs: Array diffManifest?: SnapshotDiffManifestDescriptor + diffArtifact?: SnapshotDiffArtifactDescriptor } agent: string model: { @@ -5373,6 +5381,57 @@ export type ProjectCopyCopy = { directory: string } +export type SessionLegacyProviderResolutionDescriptor = + | { + receiptID: string + sessionID: string + providerID: string + modelID: string + providerState: "indeterminate_after_crash" + sessionMutationEpoch: number + resolutionSupported: true + unsupportedReasons: Array + assistantMessageID: string + promptEpoch: number + promptWindowID: string + historyHash: string + requestHash: string + continuationRecoverySupported: boolean + workspaceRecoverySupported: boolean + sourceWorldStateBaselineStatus: "available" + worldStateBaselineHash: string + } + | { + receiptID: string + sessionID: string + providerID: string + modelID: string + providerState: "indeterminate_after_crash" + sessionMutationEpoch: number + resolutionSupported: false + unsupportedReasons: Array< + | "legacy_receipt_authority_incomplete" + | "source_prompt_epoch_missing" + | "source_prompt_epoch_not_recovery_required" + | "source_prompt_epoch_binding_mismatch" + | "provider_attempt_authority_incomplete" + | "provider_attempt_resolution_requires_reconciliation" + | "compaction_continuation_requires_maintenance" + | "workspace_recovery_requires_coordination" + | "source_world_state_baseline_missing" + | "source_world_state_baseline_invalid" + > + assistantMessageID?: string + promptEpoch?: number + promptWindowID?: string + historyHash?: string + requestHash?: string + continuationRecoverySupported: boolean + workspaceRecoverySupported: boolean + sourceWorldStateBaselineStatus: "available" | "missing" | "invalid" + worldStateBaselineHash?: string + } + export type LocationInfo = { directory: string workspaceID?: string @@ -15030,23 +15089,7 @@ export type SessionProviderResolutionListResponses = { /** * Pending provider recovery decisions */ - 200: Array<{ - receiptID: string - sessionID: string - assistantMessageID: string - providerID: string - modelID: string - providerState: "indeterminate_after_crash" - promptEpoch: number - promptWindowID: string - historyHash: string - requestHash: string - sessionMutationEpoch: number - continuationRecoverySupported: boolean - workspaceRecoverySupported: boolean - sourceWorldStateBaselineStatus: "available" | "missing" | "invalid" - worldStateBaselineHash?: string - }> + 200: Array } export type SessionProviderResolutionListResponse = @@ -15175,6 +15218,10 @@ export type SyncReplayErrors = { * BadRequest | InvalidRequestError */ 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * ConflictError + */ + 409: ConflictError } export type SyncReplayError = SyncReplayErrors[keyof SyncReplayErrors] @@ -15190,19 +15237,27 @@ export type SyncReplayResponses = { export type SyncReplayResponse = SyncReplayResponses[keyof SyncReplayResponses] -export type SyncStealData = { - body?: { - sessionID: string - } +export type SyncHistoryListData = { + body?: + | { + [key: string]: number + } + | { + version: 1 + cursor?: string + known?: { + [key: string]: number + } + } path?: never query?: { directory?: string workspace?: string } - url: "/sync/steal" + url: "/sync/history" } -export type SyncStealErrors = { +export type SyncHistoryListErrors = { /** * BadRequest | InvalidRequestError */ @@ -15211,38 +15266,6 @@ export type SyncStealErrors = { * ConflictError */ 409: ConflictError -} - -export type SyncStealError = SyncStealErrors[keyof SyncStealErrors] - -export type SyncStealResponses = { - /** - * Session stolen into workspace - */ - 200: { - sessionID: string - } -} - -export type SyncStealResponse = SyncStealResponses[keyof SyncStealResponses] - -export type SyncHistoryListData = { - body?: { - [key: string]: number - } - path?: never - query?: { - directory?: string - workspace?: string - } - url: "/sync/history" -} - -export type SyncHistoryListErrors = { - /** - * BadRequest | InvalidRequestError - */ - 400: EffectHttpApiErrorBadRequest | InvalidRequestError /** * ServiceUnavailableError */ @@ -15253,17 +15276,23 @@ export type SyncHistoryListError = SyncHistoryListErrors[keyof SyncHistoryListEr export type SyncHistoryListResponses = { /** - * Sync events + * Bounded sync history page */ - 200: Array<{ - id: string - aggregate_id: string - seq: number - type: string - data: { - [key: string]: unknown - } - }> + 200: { + version: 1 + items: Array<{ + kind: "event" + id: string + aggregate_id: string + seq: number + type: string + data: { + [key: string]: unknown + } + }> + nextCursor: string + complete: boolean + } } export type SyncHistoryListResponse = SyncHistoryListResponses[keyof SyncHistoryListResponses] From 5afb707e56085c6b9faf90717e929a0af8340677 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 11:23:02 +0800 Subject: [PATCH 20/44] fix(core): bound legacy event artifact backfill --- .../core/bug407-artifact-backfill-one.gate.ts | 32 --- .../core/bug407-artifact-backfill.gate.ts | 32 --- packages/core/bug407-migration-only.gate.ts | 13 -- packages/core/src/event.ts | 208 +++++++++--------- packages/core/src/v1/session.ts | 2 +- packages/core/test/event.test.ts | 65 +++++- packages/sdk/js/src/gen/types.gen.ts | 2 +- packages/sdk/js/src/v2/gen/types.gen.ts | 2 +- 8 files changed, 165 insertions(+), 191 deletions(-) delete mode 100644 packages/core/bug407-artifact-backfill-one.gate.ts delete mode 100644 packages/core/bug407-artifact-backfill.gate.ts delete mode 100644 packages/core/bug407-migration-only.gate.ts diff --git a/packages/core/bug407-artifact-backfill-one.gate.ts b/packages/core/bug407-artifact-backfill-one.gate.ts deleted file mode 100644 index 5bf7ef35c..000000000 --- a/packages/core/bug407-artifact-backfill-one.gate.ts +++ /dev/null @@ -1,32 +0,0 @@ -const database = process.env.BUG407_GATE_DATABASE -if (!database) throw new Error("BUG407_GATE_DATABASE is required") - -process.env.DEEPAGENT_CODE_DB = database - -const { Effect } = await import("effect") -const { EventV2 } = await import("./src/event.ts") - -const started = performance.now() -const result = await Effect.runPromise( - Effect.gen(function* () { - const events = yield* EventV2.Service - let cursor: EventV2.ID | undefined - let batches = 0 - let processed = 0 - - while (true) { - const page = yield* events.canonicalizeLegacyArtifacts({ - ...(cursor ? { afterID: cursor } : {}), - limit: 1, - }) - batches += 1 - processed += page.processed - if (!page.next) break - cursor = page.next - } - - return { batches, processed } - }).pipe(Effect.provide(EventV2.defaultLayer)), -) - -process.stdout.write(`${JSON.stringify({ ...result, elapsedMs: Math.round(performance.now() - started) })}\n`) diff --git a/packages/core/bug407-artifact-backfill.gate.ts b/packages/core/bug407-artifact-backfill.gate.ts deleted file mode 100644 index bb5fbbe06..000000000 --- a/packages/core/bug407-artifact-backfill.gate.ts +++ /dev/null @@ -1,32 +0,0 @@ -const database = process.env.BUG407_GATE_DATABASE -if (!database) throw new Error("BUG407_GATE_DATABASE is required") - -process.env.DEEPAGENT_CODE_DB = database - -const { Effect } = await import("effect") -const { EventV2 } = await import("./src/event.ts") - -const started = performance.now() -const result = await Effect.runPromise( - Effect.gen(function* () { - const events = yield* EventV2.Service - let cursor: EventV2.ID | undefined - let batches = 0 - let processed = 0 - - while (true) { - const page = yield* events.canonicalizeLegacyArtifacts({ - ...(cursor ? { afterID: cursor } : {}), - limit: EventV2.LEGACY_ARTIFACT_BATCH_EVENTS, - }) - batches += 1 - processed += page.processed - if (!page.next) break - cursor = page.next - } - - return { batches, processed } - }).pipe(Effect.provide(EventV2.defaultLayer)), -) - -process.stdout.write(`${JSON.stringify({ ...result, elapsedMs: Math.round(performance.now() - started) })}\n`) diff --git a/packages/core/bug407-migration-only.gate.ts b/packages/core/bug407-migration-only.gate.ts deleted file mode 100644 index 3c0d5a929..000000000 --- a/packages/core/bug407-migration-only.gate.ts +++ /dev/null @@ -1,13 +0,0 @@ -const database = process.env.BUG407_GATE_DATABASE -if (!database) throw new Error("BUG407_GATE_DATABASE is required") - -process.env.DEEPAGENT_CODE_DB = database - -const { Effect } = await import("effect") -const { EventV2 } = await import("./src/event.ts") - -const started = performance.now() -await Effect.runPromise(Effect.gen(function* () { - yield* EventV2.Service -}).pipe(Effect.provide(EventV2.defaultLayer))) -process.stdout.write(`${JSON.stringify({ elapsedMs: Math.round(performance.now() - started) })}\n`) diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index e47a646d7..ab4135bad 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -18,6 +18,7 @@ import { externalID, type ExternalID, NonNegativeInt, withStatics } from "./sche import { Identifier } from "./util/identifier" import { isDeepStrictEqual } from "node:util" import { Hash } from "./util/hash" +import { createHash } from "node:crypto" export const ID = Schema.String.check(Schema.isStartsWith("evt_")).pipe( Schema.brand("Event.ID"), @@ -108,7 +109,7 @@ export const MAX_ENCODED_PAYLOAD_BYTES = 4 * 1024 * 1024 export const AGGREGATE_READ_BATCH_EVENTS = 100 export const AGGREGATE_READ_BATCH_BYTES = 4 * 1024 * 1024 export const SNAPSHOT_MAX_ENCODED_BYTES = 16 * 1024 * 1024 -export const LEGACY_ARTIFACT_BATCH_EVENTS = 8 +export const LEGACY_ARTIFACT_BATCH_EVENTS = 1 export const LEGACY_DIFF_DESCRIPTOR_FILES = 200 export const ARTIFACT_CHUNK_BYTES = 256 * 1024 @@ -175,48 +176,6 @@ function hashJson(value: unknown) { return Hash.sha256(stableJson(value)) } -function canonicalizeLegacyMessageDiff(eventID: ID, type: string, data: Record) { - if (type !== versionedType("message.updated", 1)) return - if (!data.info || typeof data.info !== "object" || Array.isArray(data.info)) return - const info = data.info as Record - if (!info.summary || typeof info.summary !== "object" || Array.isArray(info.summary)) return - const summary = info.summary as Record - if (!Array.isArray(summary.diffs) || summary.diffs.length === 0) return - const originalDataHash = hashJson(data) - const body = { diffs: summary.diffs } - const bodyHash = hashJson(body) - const artifactID = `evtart_${Hash.sha256(`legacy-message-diff.v1:${eventID}:${originalDataHash}:${bodyHash}`)}` - const diffs = summary.diffs.slice(0, LEGACY_DIFF_DESCRIPTOR_FILES).flatMap((item) => { - if (!item || typeof item !== "object" || Array.isArray(item)) return [] - const { patch: _, ...descriptor } = item as Record - return [descriptor] - }) - const canonicalData = { - ...data, - info: { - ...info, - summary: { - ...summary, - diffs, - diffArtifact: { - id: artifactID, - hash: bodyHash, - codec: "legacy-message-diff.v1", - fileCount: summary.diffs.length, - }, - }, - }, - } - return { - kind: "legacy_message_diff" as const, - artifactID, - originalDataHash, - canonicalDataHash: hashJson(canonicalData), - canonicalData, - body, - } -} - // Synchronized events cross a JSON boundary, so their data schemas must encode and decode without services. const syncCodec = (definition: Definition) => definition.data as Schema.Codec @@ -452,6 +411,7 @@ export const layerWith = (layerOptions?: LayerOptions) => .select({ originalDataHash: EventArtifactTable.original_data_hash, canonicalData: EventArtifactTable.canonical_data, + codecVersion: EventArtifactTable.codec_version, }) .from(EventArtifactTable) .where(eq(EventArtifactTable.event_id, stored.id)) @@ -476,7 +436,11 @@ export const layerWith = (layerOptions?: LayerOptions) => stored.type === versionedType(definition.type, sync.version) && (isDeepStrictEqual(stored.data, encoded) || isDeepStrictEqual(artifact?.canonicalData, encoded) || - artifact?.originalDataHash === hashJson(encoded))) || + (artifact && + artifact.originalDataHash === + (artifact.codecVersion >= 2 + ? Hash.sha256(JSON.stringify(encoded)) + : hashJson(encoded))))) || (dedupe?.event_id === event.id && dedupe.type === versionedType(definition.type, sync.version) && dedupe.data_hash === hashJson(encoded))) @@ -1132,18 +1096,50 @@ export const layerWith = (layerOptions?: LayerOptions) => } function canonicalizeLegacyArtifacts(input?: { readonly afterID?: ID; readonly limit?: number; readonly now?: number }) { + const bodyData = sql`json_object('diffs', json_extract(${EventTable.data}, '$.info.summary.diffs'))` + const chunk = (eventID: ID, kind: "source" | "body", index: number) => + db + .select({ + data: kind === "source" + ? sql`substr(CAST(${EventTable.data} AS BLOB), ${index * ARTIFACT_CHUNK_BYTES + 1}, ${ARTIFACT_CHUNK_BYTES})` + : sql`substr(CAST(${bodyData} AS BLOB), ${index * ARTIFACT_CHUNK_BYTES + 1}, ${ARTIFACT_CHUNK_BYTES})`, + }) + .from(EventTable) + .where(eq(EventTable.id, eventID)) + .get() + .pipe(Effect.orDie) + const hash = (eventID: ID, kind: "source" | "body", bytes: number) => + Effect.gen(function* () { + const digest = createHash("sha256") + yield* Effect.forEach( + Array.from({ length: Math.ceil(bytes / ARTIFACT_CHUNK_BYTES) }, (_, index) => index), + (index) => chunk(eventID, kind, index).pipe(Effect.tap((value) => Effect.sync(() => digest.update(value!.data)))), + { discard: true }, + ) + return digest.digest("hex") + }) return db .transaction( () => Effect.gen(function* () { const rows = yield* db - .select() + .select({ + id: EventTable.id, + aggregateID: EventTable.aggregate_id, + seq: EventTable.seq, + fileCount: sql`json_array_length(${EventTable.data}, '$.info.summary.diffs')`, + sourceBytes: sql`length(CAST(${EventTable.data} AS BLOB))`, + bodyBytes: sql`length(CAST(${bodyData} AS BLOB))`, + }) .from(EventTable) .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) .where( and( eq(EventTable.type, versionedType("message.updated", 1)), sql`length(CAST(${EventTable.data} AS BLOB)) > ${MAX_ENCODED_PAYLOAD_BYTES}`, + sql`json_valid(${EventTable.data})`, + sql`json_type(${EventTable.data}, '$.info.summary.diffs') = 'array'`, + sql`json_array_length(${EventTable.data}, '$.info.summary.diffs') > 0`, input?.afterID ? gt(EventTable.id, input.afterID) : undefined, sql`${EventArtifactTable.event_id} is null`, ), @@ -1152,69 +1148,75 @@ export const layerWith = (layerOptions?: LayerOptions) => .limit(Math.min(Math.max(input?.limit ?? LEGACY_ARTIFACT_BATCH_EVENTS, 1), LEGACY_ARTIFACT_BATCH_EVENTS)) .all() .pipe(Effect.orDie) - const candidates = rows.flatMap((row) => { - const artifact = canonicalizeLegacyMessageDiff(row.event.id, row.event.type, row.event.data) - return artifact - ? [{ - eventID: row.event.id, - aggregateID: row.event.aggregate_id, - seq: row.event.seq, - artifact, - }] - : [] - }) yield* Effect.forEach( - candidates, - ({ eventID, aggregateID, seq, artifact }) => { - const bytes = Buffer.from(JSON.stringify(artifact.body)) - const chunks = Array.from( - { length: Math.ceil(bytes.length / ARTIFACT_CHUNK_BYTES) || 1 }, - (_, index) => bytes.subarray(index * ARTIFACT_CHUNK_BYTES, (index + 1) * ARTIFACT_CHUNK_BYTES), - ) - return db - .insert(EventArtifactTable) - .values({ - artifact_id: artifact.artifactID, - event_id: eventID, - aggregate_id: aggregateID, - seq, - kind: artifact.kind, - original_data_hash: artifact.originalDataHash, - canonical_data_hash: artifact.canonicalDataHash, - canonical_data: artifact.canonicalData, - body_hash: hashJson(artifact.body), - body_bytes: bytes.length, - chunk_count: chunks.length, - codec_version: 1, - created_at: input?.now ?? Date.now(), + rows, + (row) => Effect.gen(function* () { + const originalDataHash = yield* hash(row.id, "source", row.sourceBytes) + const bodyHash = yield* hash(row.id, "body", row.bodyBytes) + const artifactID = `evtart_${Hash.sha256(`legacy-message-diff.v2:${row.id}:${originalDataHash}:${bodyHash}`)}` + const canonicalData = db + .select({ + data: sql`json_set( + ${EventTable.data}, + '$.info.summary.diffs', + json(COALESCE(( + SELECT json_group_array(json_remove(value, '$.patch')) + FROM ( + SELECT value + FROM json_each(json_extract(${EventTable.data}, '$.info.summary.diffs')) + LIMIT ${LEGACY_DIFF_DESCRIPTOR_FILES} + ) + ), '[]')), + '$.info.summary.diffArtifact', + json_object( + 'id', ${artifactID}, + 'hash', ${bodyHash}, + 'codec', 'legacy-message-diff.v2', + 'fileCount', ${row.fileCount} + ) + )`, }) - .onConflictDoNothing() - .run() - .pipe( - Effect.orDie, - Effect.andThen( - Effect.forEach( - chunks, - (chunk, index) => - db - .insert(EventArtifactChunkTable) - .values({ - artifact_id: artifact.artifactID, - chunk_index: index, - data: chunk, - chunk_hash: Hash.sha256(chunk), - }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie), - { discard: true }, + .from(EventTable) + .where(eq(EventTable.id, row.id)) + .get() + .pipe(Effect.orDie) + return yield* canonicalData.pipe( + Effect.flatMap((canonical) => { + if (!canonical) return Effect.die(new Error(`Missing artifact source event ${row.id}`)) + const chunks = Math.ceil(row.bodyBytes / ARTIFACT_CHUNK_BYTES) || 1 + return db.run(sql` + INSERT INTO event_artifact ( + artifact_id, event_id, aggregate_id, seq, kind, + original_data_hash, canonical_data_hash, canonical_data, + body_hash, body_bytes, chunk_count, codec_version, created_at + ) VALUES ( + ${artifactID}, ${row.id}, ${row.aggregateID}, ${row.seq}, 'legacy_message_diff', + ${originalDataHash}, ${Hash.sha256(canonical.data)}, ${canonical.data}, + ${bodyHash}, ${row.bodyBytes}, ${chunks}, 2, ${input?.now ?? Date.now()} + ) ON CONFLICT DO NOTHING + `).pipe( + Effect.orDie, + Effect.andThen( + Effect.forEach( + Array.from({ length: chunks }, (_, index) => index), + (index) => chunk(row.id, "body", index).pipe( + Effect.flatMap((value) => db.run(sql` + INSERT INTO event_artifact_chunk (artifact_id, chunk_index, data, chunk_hash) + VALUES (${artifactID}, ${index}, ${value!.data}, ${Hash.sha256(value!.data)}) + ON CONFLICT DO NOTHING + `)), + Effect.orDie, + ), + { discard: true }, + ), ), - ), - ) - }, + ) + }), + ) + }), { discard: true }, ) - return { processed: candidates.length, next: rows.at(-1)?.event.id } + return { processed: rows.length, next: rows.at(-1)?.id } }), { behavior: "immediate" }, ) diff --git a/packages/core/src/v1/session.ts b/packages/core/src/v1/session.ts index 6fbb375a4..9ef607de2 100644 --- a/packages/core/src/v1/session.ts +++ b/packages/core/src/v1/session.ts @@ -382,7 +382,7 @@ const DiffManifestDescriptor = Schema.Struct({ const DiffArtifactDescriptor = Schema.Struct({ id: Schema.String, hash: Schema.String, - codec: Schema.Literal("legacy-message-diff.v1"), + codec: Schema.Literals(["legacy-message-diff.v1", "legacy-message-diff.v2"]), fileCount: NonNegativeInt, }).annotate({ identifier: "SnapshotDiffArtifactDescriptor" }) diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index 5c807ea49..ea0f0982d 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -14,7 +14,8 @@ import { Location } from "@deepagent-code/core/location" import { AbsolutePath } from "@deepagent-code/core/schema" import { WorkspaceV2 } from "@deepagent-code/core/workspace" import { V2Schema } from "@deepagent-code/core/v2-schema" -import { eq } from "drizzle-orm" +import { asc, eq, sql } from "drizzle-orm" +import { createHash } from "node:crypto" import { location } from "./fixture/location" import { testEffect } from "./lib/effect" @@ -59,6 +60,18 @@ const SyncSent = EventV2.define({ }, }) +const LegacyMessageUpdated = EventV2.define({ + type: "message.updated", + sync: { + version: 1, + aggregate: "sessionID", + }, + schema: { + sessionID: Schema.String, + info: Schema.Any, + }, +}) + const GlobalMessage = EventV2.define({ type: "test.global", schema: { @@ -912,15 +925,16 @@ describe("EventV2", () => { yield* events.publish(SyncMessage, { id: aggregateID, text: "zero" }) const id = EventV2.ID.make("evt_legacy_message_diff_artifact") const patch = "x".repeat(EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1) + const data = { + sessionID: aggregateID, + info: { summary: { diffs: [{ file: "large.patch", patch, additions: 1, deletions: 0 }] } }, + } yield* db.insert(EventTable).values({ id, aggregate_id: aggregateID, seq: 1, type: EventV2.versionedType("message.updated", 1), - data: { - sessionID: aggregateID, - info: { summary: { diffs: [{ file: "large.patch", patch, additions: 1, deletions: 0 }] } }, - }, + data, }).run().pipe(Effect.orDie) yield* db.update(EventSequenceTable).set({ seq: 1 }).where(eq(EventSequenceTable.aggregate_id, aggregateID)).run().pipe(Effect.orDie) @@ -931,12 +945,47 @@ describe("EventV2", () => { expect(artifact?.kind).toBe("legacy_message_diff") expect(artifact?.body_bytes).toBeGreaterThan(EventV2.MAX_ENCODED_PAYLOAD_BYTES) expect(artifact?.chunk_count).toBeGreaterThan(1) - const summary = (artifact?.canonical_data.info as { summary: { diffs: Array<{ patch?: string }>; diffArtifact: { hash: string } } }).summary + const summary = (artifact?.canonical_data.info as { + summary: { + diffs: Array<{ patch?: string }> + diffArtifact: { codec: string; hash: string } + } + }).summary expect(summary.diffs[0]?.patch).toBeUndefined() + expect(summary.diffArtifact.codec).toBe("legacy-message-diff.v2") expect(summary.diffArtifact.hash).toBe(artifact!.body_hash) - const chunks = yield* db.select().from(EventArtifactChunkTable).where(eq(EventArtifactChunkTable.artifact_id, artifact!.artifact_id)).all().pipe(Effect.orDie) + const chunks = yield* db + .select() + .from(EventArtifactChunkTable) + .where(eq(EventArtifactChunkTable.artifact_id, artifact!.artifact_id)) + .orderBy(asc(EventArtifactChunkTable.chunk_index)) + .all() + .pipe(Effect.orDie) expect(chunks).toHaveLength(artifact?.chunk_count ?? 0) - expect(Buffer.concat(chunks.map((chunk) => chunk.data)).length).toBe(artifact!.body_bytes) + expect(artifact?.codec_version).toBe(2) + const body = Buffer.concat(chunks.map((chunk) => chunk.data)) + expect(body.length).toBe(artifact!.body_bytes) + expect(createHash("sha256").update(body).digest("hex")).toBe(artifact!.body_hash) + for (const chunk of chunks) { + expect(createHash("sha256").update(chunk.data).digest("hex")).toBe(chunk.chunk_hash) + } + const source = yield* db + .select({ data: sql`CAST(${EventTable.data} AS TEXT)` }) + .from(EventTable) + .where(eq(EventTable.id, id)) + .get() + .pipe(Effect.orDie) + expect(createHash("sha256").update(source!.data).digest("hex")).toBe(artifact!.original_data_hash) + expect(createHash("sha256").update(JSON.stringify(artifact!.canonical_data)).digest("hex")).toBe( + artifact!.canonical_data_hash, + ) + yield* events.replay({ + id, + type: EventV2.versionedType(LegacyMessageUpdated.type, 1), + seq: 1, + aggregateID, + data: artifact!.canonical_data, + }) }), ) diff --git a/packages/sdk/js/src/gen/types.gen.ts b/packages/sdk/js/src/gen/types.gen.ts index f303c8eb5..ccee0dbd1 100644 --- a/packages/sdk/js/src/gen/types.gen.ts +++ b/packages/sdk/js/src/gen/types.gen.ts @@ -280,7 +280,7 @@ export type OutputFormat = OutputFormatText | OutputFormatJsonSchema export type SnapshotDiffArtifactDescriptor = { id: string hash: string - codec: "legacy-message-diff.v1" + codec: "legacy-message-diff.v1" | "legacy-message-diff.v2" fileCount: number } diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index f303c8eb5..ccee0dbd1 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -280,7 +280,7 @@ export type OutputFormat = OutputFormatText | OutputFormatJsonSchema export type SnapshotDiffArtifactDescriptor = { id: string hash: string - codec: "legacy-message-diff.v1" + codec: "legacy-message-diff.v1" | "legacy-message-diff.v2" fileCount: number } From 9b8285ab0265431dad0f61124b9d789de22c917d Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 11:58:50 +0800 Subject: [PATCH 21/44] fix(core): stop scoped install retry timers --- packages/core/src/npm.ts | 20 +++++++++---- packages/core/test/npm.test.ts | 28 +++++++++++++++++++ .../routes/instance/httpapi/handlers/im.ts | 10 ++++--- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/packages/core/src/npm.ts b/packages/core/src/npm.ts index d77756d61..c00e0572c 100644 --- a/packages/core/src/npm.ts +++ b/packages/core/src/npm.ts @@ -83,22 +83,30 @@ export const layer = Layer.effect( const { Arborist } = yield* Effect.promise(() => import("@npmcli/arborist")) const add = input.add ?? [] const npmOptions = yield* NpmConfig.load(input.dir) + const retry = { retries: 0, factor: 1, minTimeout: 0, maxTimeout: 0 } + const reifyOptions = { + ...npmOptions, + add, + fetchRetries: 0, + retry, + save: true, + saveType: "prod" as const, + } return yield* Effect.tryPromise({ try: (signal) => new Arborist({ ...npmOptions, path: input.dir, binLinks: true, + // Scoped installs are retried on the next config load. npm's internal retry timers do not + // observe AbortSignal and can otherwise keep a disposed desktop process alive for 10s+. + fetchRetries: 0, + retry, progress: false, savePrefix: "", ignoreScripts: true, signal, - }).reify({ - ...npmOptions, - add, - save: true, - saveType: "prod", - }), + }).reify(reifyOptions), catch: (cause) => new InstallFailedError({ cause, diff --git a/packages/core/test/npm.test.ts b/packages/core/test/npm.test.ts index 806c8f8ca..4c31d5d26 100644 --- a/packages/core/test/npm.test.ts +++ b/packages/core/test/npm.test.ts @@ -127,4 +127,32 @@ describe("Npm.install", () => { server.stop(true) } }) + + test("does not retain npm retry timers after a scoped registry failure", async () => { + await using tmp = await tmpdir() + let requests = 0 + const server = Bun.serve({ + port: 0, + fetch: () => { + requests++ + return new Response("temporary failure", { status: 500 }) + }, + }) + + try { + await writePackage(tmp.path, { name: "retry-fixture" }) + await Bun.write(path.join(tmp.path, ".npmrc"), `registry=${server.url.origin}/\n`) + const exit = await Effect.gen(function* () { + const npm = yield* Npm.Service + return yield* npm + .install(tmp.path, { add: [{ name: "retry-fixture-package", version: "1.0.0" }] }) + .pipe(Effect.exit, Effect.timeout("2 seconds")) + }).pipe(Effect.scoped, Effect.provide(npmLayer(path.join(tmp.path, "cache"))), Effect.runPromise) + + expect(requests).toBe(1) + expect(exit._tag).toBe("Failure") + } finally { + server.stop(true) + } + }) }) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/im.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/im.ts index 6950bd1e9..686c88775 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/im.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/im.ts @@ -74,9 +74,14 @@ const toAttachmentResponse = (a: IMAttachment) => ({ // Simple in-memory rate limiter class RateLimiter { private buckets = new Map() + private nextCleanupAt = Date.now() + 5 * 60 * 1000 check(key: string, limit: number, windowMs: number): boolean { const now = Date.now() + if (now >= this.nextCleanupAt) { + this.cleanup(now) + this.nextCleanupAt = now + 5 * 60 * 1000 + } const bucket = this.buckets.get(key) if (!bucket || now >= bucket.resetAt) { @@ -92,8 +97,7 @@ class RateLimiter { return true } - cleanup() { - const now = Date.now() + private cleanup(now: number) { for (const [key, bucket] of this.buckets.entries()) { if (now >= bucket.resetAt) { this.buckets.delete(key) @@ -103,8 +107,6 @@ class RateLimiter { } const rateLimiter = new RateLimiter() -// Cleanup every 5 minutes -setInterval(() => rateLimiter.cleanup(), 5 * 60 * 1000) const mapRepositoryError = (effect: Effect.Effect) => effect.pipe( From 432e61eda5205584ae42f66af14ba72914d86f89 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 16:29:54 +0800 Subject: [PATCH 22/44] fix(core): complete bounded event maintenance --- .../session/message-timeline.data.test.ts | 34 + .../pages/session/message-timeline.data.ts | 16 +- .../src/pages/session/message-timeline.tsx | 184 ++- packages/core/src/database/database.ts | 14 + packages/core/src/database/migration.gen.ts | 7 + .../20260813074240_bug_407_010_maintenance.ts | 11 + ...20260813100000_event_snapshot_authority.ts | 64 +- ...813125000_event_sync_backfill_authority.ts | 81 ++ .../20260813130000_file_part_artifact.ts | 84 ++ .../20260813131000_event_snapshot_chunks.ts | 124 ++ .../20260813132000_session_diff_artifact.ts | 86 ++ ...260813133000_session_transfer_authority.ts | 147 +++ .../20260813134000_database_capability.ts | 39 + packages/core/src/event.ts | 1100 ++++++++++++++--- packages/core/src/event/sql.ts | 88 +- packages/core/src/file-part-artifact.sql.ts | 67 + packages/core/src/file-part-artifact.ts | 839 +++++++++++++ packages/core/src/session/projector.ts | 343 ++++- packages/core/src/session/transfer.sql.ts | 49 + packages/core/src/v1/session.ts | 10 + .../core/test/database-capability.test.ts | 55 + packages/core/test/database-migration.test.ts | 27 +- packages/core/test/event-snapshot.test.ts | 195 +++ .../event-sync-backfill-migration.test.ts | 117 ++ packages/core/test/event.test.ts | 11 +- packages/core/test/file-part-artifact.test.ts | 334 +++++ .../test/session-transfer-authority.test.ts | 73 ++ .../script/bug-407-010-release-gate.ts | 241 ++++ .../src/control-plane/workspace.ts | 247 +++- .../deepagent-code/src/effect/app-runtime.ts | 2 + .../routes/instance/httpapi/groups/session.ts | 86 ++ .../routes/instance/httpapi/groups/sync.ts | 212 +++- .../instance/httpapi/handlers/session.ts | 49 + .../routes/instance/httpapi/handlers/sync.ts | 385 +++++- .../server/routes/instance/httpapi/server.ts | 2 + .../src/session/diff-artifact-schema.ts | 43 + .../src/session/diff-artifact.sql.ts | 61 + .../src/session/diff-artifact.ts | 804 ++++++++++++ .../deepagent-code/src/session/message-v2.ts | 35 +- packages/deepagent-code/src/session/revert.ts | 7 +- .../src/session/session-projector.ts | 18 + .../test/control-plane/workspace.test.ts | 145 ++- .../test/server/httpapi-sync.test.ts | 334 ++++- .../test/session/diff-artifact.test.ts | 389 ++++++ packages/sdk/js/src/gen/sdk.gen.ts | 294 +++++ packages/sdk/js/src/gen/types.gen.ts | 362 +++++- packages/sdk/js/src/v2/gen/sdk.gen.ts | 294 +++++ packages/sdk/js/src/v2/gen/types.gen.ts | 362 +++++- 48 files changed, 8286 insertions(+), 285 deletions(-) create mode 100644 packages/core/src/database/migration/20260813074240_bug_407_010_maintenance.ts create mode 100644 packages/core/src/database/migration/20260813125000_event_sync_backfill_authority.ts create mode 100644 packages/core/src/database/migration/20260813130000_file_part_artifact.ts create mode 100644 packages/core/src/database/migration/20260813131000_event_snapshot_chunks.ts create mode 100644 packages/core/src/database/migration/20260813132000_session_diff_artifact.ts create mode 100644 packages/core/src/database/migration/20260813133000_session_transfer_authority.ts create mode 100644 packages/core/src/database/migration/20260813134000_database_capability.ts create mode 100644 packages/core/src/file-part-artifact.sql.ts create mode 100644 packages/core/src/file-part-artifact.ts create mode 100644 packages/core/src/session/transfer.sql.ts create mode 100644 packages/core/test/database-capability.test.ts create mode 100644 packages/core/test/event-snapshot.test.ts create mode 100644 packages/core/test/event-sync-backfill-migration.test.ts create mode 100644 packages/core/test/file-part-artifact.test.ts create mode 100644 packages/core/test/session-transfer-authority.test.ts create mode 100644 packages/deepagent-code/script/bug-407-010-release-gate.ts create mode 100644 packages/deepagent-code/src/session/diff-artifact-schema.ts create mode 100644 packages/deepagent-code/src/session/diff-artifact.sql.ts create mode 100644 packages/deepagent-code/src/session/diff-artifact.ts create mode 100644 packages/deepagent-code/src/session/session-projector.ts create mode 100644 packages/deepagent-code/test/session/diff-artifact.test.ts diff --git a/packages/app/src/pages/session/message-timeline.data.test.ts b/packages/app/src/pages/session/message-timeline.data.test.ts index 2e7e40770..778899ad9 100644 --- a/packages/app/src/pages/session/message-timeline.data.test.ts +++ b/packages/app/src/pages/session/message-timeline.data.test.ts @@ -43,6 +43,40 @@ describe("message timeline compaction", () => { }), ) }) + + test("keeps a legacy diff artifact as metadata without materializing an inline patch", async () => { + const { Timeline } = await import("./message-timeline.data") + const message = { + id: "msg_diff_artifact", + sessionID: "ses_1", + role: "user", + agent: "build", + model: { providerID: "deepseek", modelID: "deepseek-chat" }, + time: { created: 1 }, + summary: { + diffs: [], + diffArtifact: { + id: "evtart_legacy", + hash: "a".repeat(64), + codec: "legacy-message-diff.v2", + fileCount: 4_500, + }, + }, + } as UserMessage + + const rows = Timeline.constructMessageRows(message, () => [], [], 0, false, "idle", false) + const diff = rows.find((row) => row._tag === "DiffSummary") + + expect(diff).toEqual( + expect.objectContaining({ + _tag: "DiffSummary", + userMessageID: message.id, + diffs: [], + artifact: message.summary?.diffArtifact, + }), + ) + expect(JSON.stringify(rows)).not.toContain('"patch"') + }) }) describe("message timeline activity progress", () => { diff --git a/packages/app/src/pages/session/message-timeline.data.ts b/packages/app/src/pages/session/message-timeline.data.ts index 77b91613d..98688c54f 100644 --- a/packages/app/src/pages/session/message-timeline.data.ts +++ b/packages/app/src/pages/session/message-timeline.data.ts @@ -1,5 +1,12 @@ import { parseCommentNote, readCommentMetadata } from "@/utils/comment-note" -import { AssistantMessage, Part, SessionStatus, SnapshotFileDiff, UserMessage } from "@deepagent-code/sdk/v2" +import { + AssistantMessage, + Part, + SessionStatus, + SnapshotDiffArtifactDescriptor, + SnapshotFileDiff, + UserMessage, +} from "@deepagent-code/sdk/v2" import { groupParts, PartGroup, renderable } from "@deepagent-code/ui/message-part" import { Data, Equal } from "effect" @@ -37,7 +44,7 @@ export type TimelineRowMap = { } Thinking: { userMessageID: string; reasoningHeading?: string } Retry: { userMessageID: string } - DiffSummary: { userMessageID: string; diffs: SummaryDiff[] } + DiffSummary: { userMessageID: string; diffs: SummaryDiff[]; artifact?: SnapshotDiffArtifactDescriptor } Error: { userMessageID: string; text: string } BottomSpacer: {} } @@ -75,6 +82,7 @@ export namespace TimelineRow { export class DiffSummary extends Data.TaggedClass("DiffSummary")<{ userMessageID: string diffs: SummaryDiff[] + artifact?: SnapshotDiffArtifactDescriptor }> {} export class Error extends Data.TaggedClass("Error")<{ userMessageID: string @@ -255,11 +263,13 @@ export namespace Timeline { return result }, []) .reverse() - if (diffs.length > 0 && (status === "idle" || !isActive)) { + const artifact = userMessage.summary?.diffArtifact + if ((diffs.length > 0 || artifact) && (status === "idle" || !isActive)) { rows.push( new TimelineRow.DiffSummary({ userMessageID: userMessage.id, diffs, + ...(artifact ? { artifact } : {}), }), ) } diff --git a/packages/app/src/pages/session/message-timeline.tsx b/packages/app/src/pages/session/message-timeline.tsx index f3db8342e..9615ff6d5 100644 --- a/packages/app/src/pages/session/message-timeline.tsx +++ b/packages/app/src/pages/session/message-timeline.tsx @@ -45,6 +45,9 @@ import type { AssistantMessage, Message as MessageType, Part as PartType, + SessionDiffArtifactFileResponse, + SessionDiffArtifactManifestResponse, + SnapshotDiffArtifactDescriptor, ToolPart, UserMessage, } from "@deepagent-code/sdk/v2" @@ -180,17 +183,85 @@ function TimelineThinkingRow(props: { reasoningHeading?: string; showReasoningSu ) } -function TimelineDiffSummaryRow(props: { diffs: SummaryDiff[] }) { +function TimelineDiffSummaryRow(props: { + messageID: string + diffs: SummaryDiff[] + artifact?: SnapshotDiffArtifactDescriptor +}) { const language = useLanguage() + const sdk = useSDK() + const { params } = useSessionKey() const maxFiles = 10 const [state, setState] = createStore({ showAll: false, expanded: [] as string[], + artifactOpen: false, + artifactLoading: false, + artifactFiles: [] as SummaryDiff[], + artifactCursor: undefined as string | undefined, + artifactComplete: false, }) + let manifestRequest: AbortController | undefined + onCleanup(() => manifestRequest?.abort()) + const loadManifest = async () => { + if (!props.artifact || state.artifactLoading || state.artifactComplete) return + const sessionID = params.id + if (!sessionID) return + manifestRequest?.abort() + manifestRequest = new AbortController() + setState("artifactLoading", true) + await sdk.client.session + .diffArtifactManifest( + { + sessionID, + messageID: props.messageID, + artifactID: props.artifact.id, + ...(state.artifactCursor ? { cursor: state.artifactCursor } : {}), + limit: "100", + }, + { signal: manifestRequest.signal }, + ) + .then((result) => { + const data = result.data as SessionDiffArtifactManifestResponse | undefined + if (!data || data.artifact.hash !== props.artifact?.hash) throw new Error("diff artifact hash mismatch") + const existing = new Set(state.artifactFiles.map((file) => file.file)) + setState({ + artifactFiles: [ + ...state.artifactFiles, + ...data.files.filter((file) => !existing.has(file.file)).map((file) => ({ + file: file.file, + additions: file.additions, + deletions: file.deletions, + ...(file.status ? { status: file.status } : {}), + })), + ], + artifactCursor: data.nextCursor, + artifactComplete: data.complete, + }) + }) + .catch((error) => { + if (error instanceof DOMException && error.name === "AbortError") return + showToast({ title: language.t("common.requestFailed") }) + }) + .finally(() => setState("artifactLoading", false)) + } + const toggleArtifact = () => { + if (!props.artifact) return + const open = !state.artifactOpen + setState("artifactOpen", open) + if (open) { + void loadManifest() + return + } + manifestRequest?.abort() + setState("expanded", []) + } const showAll = () => state.showAll const expanded = () => state.expanded - const overflow = createMemo(() => Math.max(0, props.diffs.length - maxFiles)) - const visible = createMemo(() => (showAll() ? props.diffs : props.diffs.slice(0, maxFiles))) + const diffs = createMemo(() => (props.artifact ? state.artifactFiles : props.diffs)) + const count = createMemo(() => props.artifact?.fileCount ?? props.diffs.length) + const overflow = createMemo(() => Math.max(0, diffs().length - maxFiles)) + const visible = createMemo(() => (showAll() ? diffs() : diffs().slice(0, maxFiles))) return (
- {props.diffs.length} {language.t("ui.sessionTurn.diffs.changed")}{" "} - {language.t(props.diffs.length === 1 ? "ui.common.file.one" : "ui.common.file.other")} + {count()} {language.t("ui.sessionTurn.diffs.changed")}{" "} + {language.t(count() === 1 ? "ui.common.file.one" : "ui.common.file.other")} - + + + + 0}> setState("showAll", !showAll())}> {showAll() ? language.t("ui.sessionTurn.diffs.showLess") : language.t("ui.sessionTurn.diffs.showAll")}
+
setState("expanded", Array.isArray(value) ? value : value ? [value] : [])} + onChange={(value) => { + const next = Array.isArray(value) ? value : value ? [value] : [] + setState("expanded", next.slice(-3)) + }} > {(diff) => { @@ -250,7 +338,15 @@ function TimelineDiffSummaryRow(props: { diffs: SummaryDiff[] }) { - + }> + {(artifact) => ( + + )} + @@ -263,11 +359,77 @@ function TimelineDiffSummaryRow(props: { diffs: SummaryDiff[] }) { {language.t("ui.sessionTurn.diffs.more", { count: String(overflow()) })}
+ +
void loadManifest()}> + {language.t("ui.sessionTurn.diffs.showAll")} +
+ + +
+
+
) } +function TimelineArtifactDiffView(props: { + messageID: string + artifact: SnapshotDiffArtifactDescriptor + diff: SummaryDiff +}) { + const language = useLanguage() + const sdk = useSDK() + const { params } = useSessionKey() + const [state, setState] = createStore({ + loading: true, + value: undefined as SessionDiffArtifactFileResponse | undefined, + }) + const controller = new AbortController() + onCleanup(() => controller.abort()) + createEffect(() => { + const sessionID = params.id + if (!sessionID) return + void sdk.client.session + .diffArtifactFile( + { + sessionID, + messageID: props.messageID, + artifactID: props.artifact.id, + path: props.diff.file, + maxBytes: String(1024 * 1024), + }, + { signal: controller.signal }, + ) + .then((result) => { + const data = result.data as SessionDiffArtifactFileResponse | undefined + if (!data || data.artifactID !== props.artifact.id || data.file !== props.diff.file) + throw new Error("diff artifact file identity mismatch") + setState("value", data) + }) + .catch((error) => { + if (error instanceof DOMException && error.name === "AbortError") return + showToast({ title: language.t("common.requestFailed") }) + }) + .finally(() => setState("loading", false)) + }) + return ( + }> + {(value) => ( + + )} + + ) +} + function TimelineDiffView(props: { diff: SummaryDiff }) { const fileComponent = useFileComponent() const view = normalize(props.diff) @@ -1414,7 +1576,11 @@ export function MessageTimeline(props: { return (
- +
) diff --git a/packages/core/src/database/database.ts b/packages/core/src/database/database.ts index 3bc75f830..edc3acb78 100644 --- a/packages/core/src/database/database.ts +++ b/packages/core/src/database/database.ts @@ -37,6 +37,20 @@ export const layer = Layer.effect( yield* db.run("PRAGMA wal_autocheckpoint = 200") yield* DatabaseMigration.apply(db) + const capabilities = yield* db.all<{ + capability: string + minimum_reader_protocol: number + minimum_writer_protocol: number + }>("SELECT capability, minimum_reader_protocol, minimum_writer_protocol FROM database_capability") + for (const capability of capabilities) { + if (capability.minimum_reader_protocol > 2 || capability.minimum_writer_protocol > 2) + return yield* Effect.die( + new Error( + `Database capability ${capability.capability} requires reader protocol ${capability.minimum_reader_protocol} and writer protocol ${capability.minimum_writer_protocol}; this runtime supports protocol 2`, + ), + ) + } + return { db } }).pipe(Effect.orDie), ) diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index b0a93104a..dd9609e29 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -83,8 +83,15 @@ export const migrations = ( import("./migration/20260812120000_legacy_provider_recovery"), import("./migration/20260812130000_legacy_activity_lifecycle_expand"), import("./migration/20260812140000_session_diff_manifest"), + import("./migration/20260813074240_bug_407_010_maintenance"), import("./migration/20260813100000_event_snapshot_authority"), import("./migration/20260813110000_provider_recovery_authority_bridge"), import("./migration/20260813120000_legacy_provider_receipt_supersession"), + import("./migration/20260813125000_event_sync_backfill_authority"), + import("./migration/20260813130000_file_part_artifact"), + import("./migration/20260813131000_event_snapshot_chunks"), + import("./migration/20260813132000_session_diff_artifact"), + import("./migration/20260813133000_session_transfer_authority"), + import("./migration/20260813134000_database_capability"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260813074240_bug_407_010_maintenance.ts b/packages/core/src/database/migration/20260813074240_bug_407_010_maintenance.ts new file mode 100644 index 000000000..46d501da8 --- /dev/null +++ b/packages/core/src/database/migration/20260813074240_bug_407_010_maintenance.ts @@ -0,0 +1,11 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +// Drizzle records the declared end-state here. The following custom migrations install the +// same schema together with bounded backfill receipts, integrity triggers, and crash-safe state. +export default { + id: "20260813074240_bug_407_010_maintenance", + up() { + return Effect.void + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts b/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts index 09d2d5dba..31735f9d7 100644 --- a/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts +++ b/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts @@ -8,17 +8,57 @@ export default { yield* tx.run("ALTER TABLE event_sequence ADD COLUMN retention_floor_seq INTEGER") yield* tx.run("ALTER TABLE event_sequence ADD COLUMN snapshot_id TEXT") yield* tx.run("ALTER TABLE event ADD COLUMN sync_seq INTEGER") - yield* tx.run("UPDATE event SET sync_seq = rowid WHERE sync_seq IS NULL") - yield* tx.run("CREATE UNIQUE INDEX event_sync_seq_idx ON event(sync_seq)") yield* tx.run(` CREATE TABLE event_sync_sequence ( id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), seq INTEGER NOT NULL CHECK (seq >= -1), generation TEXT NOT NULL CHECK (length(generation) = 32), - cursor_secret TEXT NOT NULL CHECK (length(cursor_secret) = 64) + cursor_secret TEXT NOT NULL CHECK (length(cursor_secret) = 64), + backfill_complete INTEGER NOT NULL CHECK (backfill_complete IN (0, 1)) ) `) - yield* tx.run("INSERT INTO event_sync_sequence(id, seq, generation, cursor_secret) SELECT 1, COALESCE(MAX(sync_seq), -1), lower(hex(randomblob(16))), lower(hex(randomblob(32))) FROM event") + // Reserve the existing rowid range without rewriting legacy event payload pages. A bounded, + // resumable maintenance job materializes the sidecar index before history sync is enabled. + yield* tx.run(` + INSERT INTO event_sync_sequence(id, seq, generation, cursor_secret, backfill_complete) + SELECT 1, COALESCE(MAX(rowid), -1), lower(hex(randomblob(16))), lower(hex(randomblob(32))), + CASE WHEN MAX(rowid) IS NULL THEN 1 ELSE 0 END + FROM event + `) + yield* tx.run(` + CREATE TABLE event_sync_backfill ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + state TEXT NOT NULL CHECK (state IN ('pending', 'complete')), + cursor_rowid INTEGER NOT NULL CHECK (cursor_rowid >= 0), + high_water_rowid INTEGER NOT NULL CHECK (high_water_rowid >= 0), + processed_count INTEGER NOT NULL CHECK (processed_count >= 0), + started_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + completed_at INTEGER + ) + `) + yield* tx.run(` + INSERT INTO event_sync_backfill( + id, state, cursor_rowid, high_water_rowid, processed_count, + started_at, updated_at, completed_at + ) + SELECT 1, + CASE WHEN COALESCE(MAX(rowid), 0) = 0 THEN 'complete' ELSE 'pending' END, + 0, COALESCE(MAX(rowid), 0), 0, + unixepoch('subsec') * 1000, unixepoch('subsec') * 1000, + CASE WHEN COALESCE(MAX(rowid), 0) = 0 THEN unixepoch('subsec') * 1000 END + FROM event + `) + yield* tx.run(` + CREATE TABLE event_sync_index ( + sync_seq INTEGER NOT NULL PRIMARY KEY CHECK (sync_seq >= 0), + event_id TEXT NOT NULL UNIQUE REFERENCES event(id) ON DELETE CASCADE, + aggregate_id TEXT NOT NULL, + seq INTEGER NOT NULL CHECK (seq >= 0), + UNIQUE (aggregate_id, seq) + ) + `) + yield* tx.run("CREATE INDEX event_sync_index_aggregate_seq_idx ON event_sync_index(aggregate_id, seq)") yield* tx.run(` CREATE TABLE workspace_sync_cursor ( workspace_id TEXT NOT NULL, @@ -85,12 +125,20 @@ export default { ) `) yield* tx.run(` - CREATE TRIGGER event_sync_seq_legacy_allocator - AFTER INSERT ON event + CREATE TRIGGER event_sync_seq_required + BEFORE INSERT ON event WHEN NEW.sync_seq IS NULL BEGIN - UPDATE event_sync_sequence SET seq = seq + 1 WHERE id = 1; - UPDATE event SET sync_seq = (SELECT seq FROM event_sync_sequence WHERE id = 1) WHERE id = NEW.id; + SELECT RAISE(ABORT, 'event_sync_seq_required'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_sync_index_explicit_insert + AFTER INSERT ON event + WHEN NEW.sync_seq IS NOT NULL + BEGIN + INSERT INTO event_sync_index(sync_seq, event_id, aggregate_id, seq) + VALUES (NEW.sync_seq, NEW.id, NEW.aggregate_id, NEW.seq); END `) yield* tx.run(` diff --git a/packages/core/src/database/migration/20260813125000_event_sync_backfill_authority.ts b/packages/core/src/database/migration/20260813125000_event_sync_backfill_authority.ts new file mode 100644 index 000000000..93c87de14 --- /dev/null +++ b/packages/core/src/database/migration/20260813125000_event_sync_backfill_authority.ts @@ -0,0 +1,81 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813125000_event_sync_backfill_authority", + up(tx) { + return Effect.gen(function* () { + const columns = yield* tx.all<{ name: string }>("PRAGMA table_info('event_sync_sequence')") + if (!columns.some((column) => column.name === "backfill_complete")) + yield* tx.run( + "ALTER TABLE event_sync_sequence ADD COLUMN backfill_complete INTEGER NOT NULL DEFAULT 0 CHECK (backfill_complete IN (0, 1))", + ) + + yield* tx.run(` + CREATE TABLE IF NOT EXISTS event_sync_backfill ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + state TEXT NOT NULL CHECK (state IN ('pending', 'complete')), + cursor_rowid INTEGER NOT NULL CHECK (cursor_rowid >= 0), + high_water_rowid INTEGER NOT NULL CHECK (high_water_rowid >= 0), + processed_count INTEGER NOT NULL CHECK (processed_count >= 0), + started_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + completed_at INTEGER + ) + `) + yield* tx.run(` + CREATE TABLE IF NOT EXISTS event_sync_index ( + sync_seq INTEGER NOT NULL PRIMARY KEY CHECK (sync_seq >= 0), + event_id TEXT NOT NULL UNIQUE REFERENCES event(id) ON DELETE CASCADE, + aggregate_id TEXT NOT NULL, + seq INTEGER NOT NULL CHECK (seq >= 0), + UNIQUE (aggregate_id, seq) + ) + `) + yield* tx.run( + "CREATE INDEX IF NOT EXISTS event_sync_index_aggregate_seq_idx ON event_sync_index(aggregate_id, seq)", + ) + yield* tx.run(` + INSERT OR IGNORE INTO event_sync_backfill( + id, state, cursor_rowid, high_water_rowid, processed_count, + started_at, updated_at, completed_at + ) + SELECT 1, + CASE WHEN MAX(rowid) IS NULL THEN 'complete' ELSE 'pending' END, + 0, COALESCE(MAX(rowid), 0), 0, + unixepoch('subsec') * 1000, unixepoch('subsec') * 1000, + CASE WHEN MAX(rowid) IS NULL THEN unixepoch('subsec') * 1000 END + FROM event + `) + yield* tx.run(` + UPDATE event_sync_sequence + SET seq = max(seq, (SELECT high_water_rowid FROM event_sync_backfill WHERE id = 1)), + backfill_complete = CASE + WHEN (SELECT state FROM event_sync_backfill WHERE id = 1) = 'complete' THEN 1 + ELSE 0 + END + WHERE id = 1 + `) + yield* tx.run("DROP TRIGGER IF EXISTS event_sync_index_explicit_insert") + yield* tx.run("DROP TRIGGER IF EXISTS event_sync_seq_legacy_allocator") + yield* tx.run("DROP TRIGGER IF EXISTS event_sync_seq_required") + yield* tx.run(` + CREATE TRIGGER event_sync_index_explicit_insert + AFTER INSERT ON event + WHEN NEW.sync_seq IS NOT NULL + BEGIN + INSERT INTO event_sync_index(sync_seq, event_id, aggregate_id, seq) + VALUES (NEW.sync_seq, NEW.id, NEW.aggregate_id, NEW.seq); + END + `) + yield* tx.run(` + CREATE TRIGGER event_sync_seq_required + BEFORE INSERT ON event + WHEN NEW.sync_seq IS NULL + BEGIN + SELECT RAISE(ABORT, 'event_sync_seq_required'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813130000_file_part_artifact.ts b/packages/core/src/database/migration/20260813130000_file_part_artifact.ts new file mode 100644 index 000000000..8f71c0425 --- /dev/null +++ b/packages/core/src/database/migration/20260813130000_file_part_artifact.ts @@ -0,0 +1,84 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813130000_file_part_artifact", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE file_part_artifact ( + artifact_id TEXT NOT NULL PRIMARY KEY, + body_hash TEXT NOT NULL UNIQUE CHECK (length(body_hash) = 64), + body_bytes INTEGER NOT NULL CHECK (body_bytes >= 0), + chunk_bytes INTEGER NOT NULL CHECK (chunk_bytes = 262144), + chunk_count INTEGER NOT NULL CHECK (chunk_count >= 1 AND chunk_count <= 128), + codec_version INTEGER NOT NULL CHECK (codec_version = 1), + complete INTEGER NOT NULL CHECK (complete IN (0, 1)), + created_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + CREATE TABLE file_part_artifact_chunk ( + artifact_id TEXT NOT NULL REFERENCES file_part_artifact(artifact_id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL CHECK (chunk_index >= 0 AND chunk_index < 128), + data BLOB NOT NULL CHECK (length(data) <= 262144), + chunk_hash TEXT NOT NULL CHECK (length(chunk_hash) = 64), + PRIMARY KEY (artifact_id, chunk_index) + ) + `) + yield* tx.run(` + CREATE TABLE file_part_artifact_binding ( + event_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + seq INTEGER NOT NULL CHECK (seq >= 0), + part_id TEXT NOT NULL CHECK (part_id LIKE 'prt%'), + artifact_id TEXT NOT NULL REFERENCES file_part_artifact(artifact_id) ON DELETE RESTRICT, + original_data_hash TEXT NOT NULL CHECK (length(original_data_hash) = 64), + canonical_data_hash TEXT NOT NULL CHECK (length(canonical_data_hash) = 64), + canonical_data TEXT NOT NULL CHECK (json_valid(canonical_data)), + created_at INTEGER NOT NULL, + UNIQUE (aggregate_id, seq) + ) + `) + yield* tx.run(` + CREATE INDEX file_part_artifact_binding_part_idx + ON file_part_artifact_binding(aggregate_id, part_id, seq) + `) + yield* tx.run(` + CREATE TABLE file_part_artifact_import ( + event_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL, + seq INTEGER NOT NULL CHECK (seq >= 0), + artifact_id TEXT NOT NULL REFERENCES file_part_artifact(artifact_id) ON DELETE CASCADE, + original_data_hash TEXT NOT NULL CHECK (length(original_data_hash) = 64), + canonical_data_hash TEXT NOT NULL CHECK (length(canonical_data_hash) = 64), + canonical_data TEXT NOT NULL CHECK (json_valid(canonical_data)), + created_at INTEGER NOT NULL, + UNIQUE (aggregate_id, seq) + ) + `) + yield* tx.run(` + CREATE TRIGGER file_part_artifact_binding_source_guard + BEFORE INSERT ON file_part_artifact_binding + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM event source + WHERE source.id = NEW.event_id + AND source.aggregate_id = NEW.aggregate_id + AND source.seq = NEW.seq + AND source.type = 'message.part.updated.1' + ) AND NOT EXISTS ( + SELECT 1 FROM file_part_artifact_import imported + WHERE imported.event_id = NEW.event_id + AND imported.aggregate_id = NEW.aggregate_id + AND imported.seq = NEW.seq + AND imported.artifact_id = NEW.artifact_id + AND imported.original_data_hash = NEW.original_data_hash + AND imported.canonical_data_hash = NEW.canonical_data_hash + AND imported.canonical_data = NEW.canonical_data + ) THEN RAISE(ABORT, 'file_part_artifact_source_mismatch') END; + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813131000_event_snapshot_chunks.ts b/packages/core/src/database/migration/20260813131000_event_snapshot_chunks.ts new file mode 100644 index 000000000..c96b3d955 --- /dev/null +++ b/packages/core/src/database/migration/20260813131000_event_snapshot_chunks.ts @@ -0,0 +1,124 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813131000_event_snapshot_chunks", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE event_snapshot_row ( + snapshot_id TEXT NOT NULL, + row_index INTEGER NOT NULL CHECK (row_index >= 0), + table_name TEXT NOT NULL CHECK (length(table_name) > 0), + row_key TEXT NOT NULL CHECK (length(row_key) > 0), + row_hash TEXT NOT NULL CHECK (length(row_hash) = 64), + row_bytes INTEGER NOT NULL CHECK (row_bytes > 0), + chunk_count INTEGER NOT NULL CHECK (chunk_count > 0), + chain_hash TEXT NOT NULL CHECK (length(chain_hash) = 64), + PRIMARY KEY (snapshot_id, row_index), + UNIQUE (snapshot_id, table_name, row_key) + ) + `) + yield* tx.run(` + CREATE TABLE event_snapshot_chunk ( + row_hash TEXT NOT NULL CHECK (length(row_hash) = 64), + chunk_index INTEGER NOT NULL CHECK (chunk_index >= 0), + data BLOB NOT NULL, + chunk_hash TEXT NOT NULL CHECK (length(chunk_hash) = 64), + PRIMARY KEY (row_hash, chunk_index) + ) + `) + yield* tx.run(` + CREATE TABLE event_snapshot_attempt ( + snapshot_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL, + through_seq INTEGER NOT NULL CHECK (through_seq >= 0), + expected_latest INTEGER NOT NULL CHECK (expected_latest >= 0), + owner_id TEXT, + codec TEXT NOT NULL CHECK (length(codec) > 0), + schema_version INTEGER NOT NULL CHECK (schema_version > 0), + projection_revision TEXT NOT NULL, + cursor TEXT, + row_count INTEGER NOT NULL CHECK (row_count >= 0), + encoded_bytes INTEGER NOT NULL CHECK (encoded_bytes >= 0), + content_hash TEXT NOT NULL CHECK (length(content_hash) = 64), + tables TEXT NOT NULL CHECK (json_valid(tables)), + state TEXT NOT NULL CHECK (state IN ('prepared', 'staged', 'complete')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_row_immutable + BEFORE UPDATE ON event_snapshot_row + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_row_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_row_delete_guard + BEFORE DELETE ON event_snapshot_row + WHEN EXISTS (SELECT 1 FROM event_snapshot snapshot WHERE snapshot.snapshot_id = OLD.snapshot_id) + OR EXISTS ( + SELECT 1 FROM event_snapshot_attempt attempt + WHERE attempt.snapshot_id = OLD.snapshot_id + AND attempt.state != 'prepared' + ) + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_row_active'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_chunk_immutable + BEFORE UPDATE ON event_snapshot_chunk + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_chunk_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_chunk_delete_guard + BEFORE DELETE ON event_snapshot_chunk + WHEN EXISTS (SELECT 1 FROM event_snapshot_row row WHERE row.row_hash = OLD.row_hash) + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_chunk_referenced'); + END + `) + yield* tx.run("ALTER TABLE event_dedupe ADD COLUMN source_data TEXT CHECK (source_data IS NULL OR json_valid(source_data))") + yield* tx.run("DROP TRIGGER IF EXISTS event_delete_requires_dedupe") + yield* tx.run(` + CREATE TRIGGER event_delete_requires_dedupe + BEFORE DELETE ON event + WHEN EXISTS ( + SELECT 1 FROM event_sequence sequence + WHERE sequence.aggregate_id = OLD.aggregate_id + AND sequence.retention_floor_seq IS NOT NULL + AND OLD.seq <= sequence.retention_floor_seq + ) AND NOT EXISTS ( + SELECT 1 FROM event_dedupe dedupe + WHERE dedupe.aggregate_id = OLD.aggregate_id + AND dedupe.seq = OLD.seq + AND dedupe.event_id = OLD.id + AND dedupe.type = OLD.type + AND length(dedupe.data_hash) = 64 + AND dedupe.source_data = OLD.data + ) + BEGIN + SELECT RAISE(ABORT, 'event_compaction_dedupe_missing'); + END + `) + yield* tx.run(` + CREATE TABLE event_compaction_receipt ( + aggregate_id TEXT NOT NULL PRIMARY KEY REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + snapshot_id TEXT NOT NULL, + through_seq INTEGER NOT NULL CHECK (through_seq >= 0), + codec TEXT NOT NULL, + schema_version INTEGER NOT NULL CHECK (schema_version > 0), + cursor_seq INTEGER NOT NULL CHECK (cursor_seq >= -1), + deleted_count INTEGER NOT NULL CHECK (deleted_count >= 0), + state TEXT NOT NULL CHECK (state IN ('running', 'complete')), + updated_at INTEGER NOT NULL + ) + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813132000_session_diff_artifact.ts b/packages/core/src/database/migration/20260813132000_session_diff_artifact.ts new file mode 100644 index 000000000..aa728c1da --- /dev/null +++ b/packages/core/src/database/migration/20260813132000_session_diff_artifact.ts @@ -0,0 +1,86 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813132000_session_diff_artifact", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE session_diff_migration_receipt ( + message_id TEXT NOT NULL PRIMARY KEY REFERENCES message(id) ON DELETE CASCADE, + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + artifact_id TEXT NOT NULL UNIQUE REFERENCES event_artifact(artifact_id) ON DELETE RESTRICT, + source_event_id TEXT NOT NULL, + expected_message_data_hash TEXT NOT NULL CHECK (length(expected_message_data_hash) = 64), + committed_message_data_hash TEXT CHECK (committed_message_data_hash IS NULL OR length(committed_message_data_hash) = 64), + expected_session_summary_hash TEXT NOT NULL CHECK (length(expected_session_summary_hash) = 64), + committed_session_summary_hash TEXT CHECK (committed_session_summary_hash IS NULL OR length(committed_session_summary_hash) = 64), + canonicalizer_version INTEGER NOT NULL CHECK (canonicalizer_version >= 1), + canonicalization_version INTEGER NOT NULL CHECK (canonicalization_version >= 1), + epoch_hashes TEXT NOT NULL CHECK (json_valid(epoch_hashes)), + state TEXT NOT NULL CHECK (state IN ('prepared', 'committed', 'migration_validation_failed')), + failure_reason TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + committed_at INTEGER, + CHECK ( + (state = 'committed' AND committed_message_data_hash IS NOT NULL AND committed_session_summary_hash IS NOT NULL AND committed_at IS NOT NULL AND failure_reason IS NULL) + OR (state = 'migration_validation_failed' AND committed_message_data_hash IS NULL AND committed_session_summary_hash IS NULL AND committed_at IS NULL AND failure_reason IS NOT NULL) + OR (state = 'prepared' AND committed_message_data_hash IS NULL AND committed_session_summary_hash IS NULL AND committed_at IS NULL AND failure_reason IS NULL) + ) + ) + `) + yield* tx.run( + "CREATE INDEX session_diff_migration_receipt_session_state_idx ON session_diff_migration_receipt(session_id, state)", + ) + yield* tx.run(` + CREATE TABLE session_diff_artifact_file ( + artifact_id TEXT NOT NULL REFERENCES event_artifact(artifact_id) ON DELETE CASCADE, + file_index INTEGER NOT NULL CHECK (file_index >= 0), + path TEXT NOT NULL CHECK (length(path) > 0), + path_key TEXT NOT NULL CHECK (length(path_key) > 0), + additions INTEGER NOT NULL CHECK (additions >= 0), + deletions INTEGER NOT NULL CHECK (deletions >= 0), + status TEXT CHECK (status IS NULL OR status IN ('added', 'deleted', 'modified')), + patch_hash TEXT NOT NULL CHECK (length(patch_hash) = 64), + patch_bytes INTEGER NOT NULL CHECK (patch_bytes >= 0), + patch_chunk_count INTEGER NOT NULL CHECK (patch_chunk_count >= 1), + PRIMARY KEY (artifact_id, file_index), + UNIQUE (artifact_id, path_key) + ) + `) + yield* tx.run(` + CREATE TABLE session_diff_artifact_file_chunk ( + artifact_id TEXT NOT NULL, + file_index INTEGER NOT NULL, + chunk_index INTEGER NOT NULL CHECK (chunk_index >= 0), + data BLOB NOT NULL, + chunk_hash TEXT NOT NULL CHECK (length(chunk_hash) = 64), + PRIMARY KEY (artifact_id, file_index, chunk_index), + FOREIGN KEY (artifact_id, file_index) + REFERENCES session_diff_artifact_file(artifact_id, file_index) ON DELETE CASCADE + ) + `) + yield* tx.run(` + CREATE TRIGGER session_diff_migration_receipt_committed_immutable + BEFORE UPDATE ON session_diff_migration_receipt + WHEN OLD.state = 'committed' + BEGIN + SELECT RAISE(ABORT, 'session_diff_migration_receipt_committed_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_diff_artifact_file_receipt_guard + BEFORE INSERT ON session_diff_artifact_file + WHEN NOT EXISTS ( + SELECT 1 FROM session_diff_migration_receipt receipt + WHERE receipt.artifact_id = NEW.artifact_id + AND receipt.state = 'prepared' + ) + BEGIN + SELECT RAISE(ABORT, 'session_diff_artifact_file_requires_prepared_receipt'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813133000_session_transfer_authority.ts b/packages/core/src/database/migration/20260813133000_session_transfer_authority.ts new file mode 100644 index 000000000..e609f3a55 --- /dev/null +++ b/packages/core/src/database/migration/20260813133000_session_transfer_authority.ts @@ -0,0 +1,147 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813133000_session_transfer_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run("ALTER TABLE event_sequence ADD COLUMN write_fence_transfer_id TEXT") + yield* tx.run(` + CREATE TABLE session_transfer_operation ( + transfer_id TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + source_workspace_id TEXT, + target_workspace_id TEXT, + source_owner_id TEXT, + target_owner_id TEXT, + source_event_seq INTEGER NOT NULL CHECK (source_event_seq >= 0), + source_mutation_epoch INTEGER NOT NULL CHECK (source_mutation_epoch >= 0), + snapshot_id TEXT, + snapshot_hash TEXT, + state TEXT NOT NULL CHECK (state IN ( + 'admitted', 'source_frozen', 'target_staged', 'owner_committed', 'target_activated', 'aborted' + )), + request_hash TEXT NOT NULL CHECK (length(request_hash) = 64), + error_code TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + completed_at INTEGER, + UNIQUE (session_id, request_hash) + ) + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_transfer_operation_active_idx + ON session_transfer_operation(session_id) + WHERE state NOT IN ('target_activated', 'aborted') + `) + yield* tx.run(` + CREATE TABLE session_transfer_target_receipt ( + transfer_id TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + source_snapshot_id TEXT NOT NULL, + source_snapshot_hash TEXT NOT NULL CHECK (length(source_snapshot_hash) = 64), + source_event_seq INTEGER NOT NULL CHECK (source_event_seq >= 0), + target_workspace_id TEXT, + target_owner_id TEXT, + state TEXT NOT NULL CHECK (state IN ('staged', 'activated')), + activated_snapshot_id TEXT, + activated_at INTEGER, + created_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + CREATE TRIGGER session_transfer_operation_state_guard + BEFORE UPDATE ON session_transfer_operation + BEGIN + SELECT CASE + WHEN NEW.transfer_id != OLD.transfer_id + OR NEW.session_id != OLD.session_id + OR NEW.request_hash != OLD.request_hash + OR NEW.source_event_seq != OLD.source_event_seq + OR NEW.source_mutation_epoch != OLD.source_mutation_epoch + OR NEW.source_workspace_id IS NOT OLD.source_workspace_id + OR NEW.target_workspace_id IS NOT OLD.target_workspace_id + OR NEW.source_owner_id IS NOT OLD.source_owner_id + OR NEW.target_owner_id IS NOT OLD.target_owner_id + THEN RAISE(ABORT, 'session_transfer_identity_immutable') + WHEN NOT ( + (OLD.state = 'admitted' AND NEW.state IN ('source_frozen', 'aborted')) + OR (OLD.state = 'source_frozen' AND NEW.state IN ('target_staged', 'aborted')) + OR (OLD.state = 'target_staged' AND NEW.state = 'owner_committed') + OR (OLD.state = 'owner_committed' AND NEW.state = 'target_activated') + OR OLD.state = NEW.state + ) THEN RAISE(ABORT, 'session_transfer_transition_invalid') + WHEN OLD.state IN ('target_activated', 'aborted') AND NEW.state != OLD.state + THEN RAISE(ABORT, 'session_transfer_terminal') + WHEN NEW.state IN ('source_frozen', 'target_staged', 'owner_committed', 'target_activated') + AND (NEW.snapshot_id IS NULL OR length(NEW.snapshot_hash) != 64) + THEN RAISE(ABORT, 'session_transfer_snapshot_required') + WHEN OLD.snapshot_id IS NOT NULL + AND (NEW.snapshot_id != OLD.snapshot_id OR NEW.snapshot_hash != OLD.snapshot_hash) + THEN RAISE(ABORT, 'session_transfer_snapshot_immutable') + END; + END + `) + yield* tx.run(` + CREATE TRIGGER event_sequence_transfer_fence_guard + BEFORE UPDATE OF write_fence_transfer_id ON event_sequence + BEGIN + SELECT CASE + WHEN OLD.write_fence_transfer_id IS NOT NULL + AND NEW.write_fence_transfer_id IS NOT OLD.write_fence_transfer_id + THEN RAISE(ABORT, 'event_sequence_transfer_fence_immutable') + WHEN NEW.write_fence_transfer_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session_transfer_operation operation + WHERE operation.transfer_id = NEW.write_fence_transfer_id + AND operation.session_id = NEW.aggregate_id + AND operation.state IN ('admitted', 'source_frozen') + ) THEN RAISE(ABORT, 'event_sequence_transfer_fence_authority_missing') + END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_input_transfer_fence + BEFORE INSERT ON session_input + WHEN EXISTS ( + SELECT 1 FROM event_sequence sequence + WHERE sequence.aggregate_id = NEW.session_id + AND sequence.write_fence_transfer_id IS NOT NULL + ) + BEGIN + SELECT RAISE(ABORT, 'session_transfer_source_frozen'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_transfer_target_receipt_state_guard + BEFORE UPDATE ON session_transfer_target_receipt + BEGIN + SELECT CASE + WHEN NEW.transfer_id != OLD.transfer_id + OR NEW.session_id != OLD.session_id + OR NEW.source_snapshot_id != OLD.source_snapshot_id + OR NEW.source_snapshot_hash != OLD.source_snapshot_hash + OR NEW.source_event_seq != OLD.source_event_seq + OR NEW.target_workspace_id IS NOT OLD.target_workspace_id + OR NEW.target_owner_id IS NOT OLD.target_owner_id + THEN RAISE(ABORT, 'session_transfer_target_identity_immutable') + WHEN OLD.state = 'activated' AND NEW.state != 'activated' + THEN RAISE(ABORT, 'session_transfer_target_terminal') + WHEN OLD.state = 'staged' AND NEW.state NOT IN ('staged', 'activated') + THEN RAISE(ABORT, 'session_transfer_target_transition_invalid') + WHEN NEW.state = 'activated' + AND (NEW.activated_snapshot_id IS NULL OR NEW.activated_at IS NULL) + THEN RAISE(ABORT, 'session_transfer_target_activation_incomplete') + END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_transfer_target_receipt_immutable_delete + BEFORE DELETE ON session_transfer_target_receipt + WHEN EXISTS (SELECT 1 FROM session WHERE id = OLD.session_id) + BEGIN + SELECT RAISE(ABORT, 'session_transfer_target_receipt_immutable'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813134000_database_capability.ts b/packages/core/src/database/migration/20260813134000_database_capability.ts new file mode 100644 index 000000000..a59da4f8f --- /dev/null +++ b/packages/core/src/database/migration/20260813134000_database_capability.ts @@ -0,0 +1,39 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813134000_database_capability", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE database_capability ( + capability TEXT NOT NULL PRIMARY KEY, + minimum_reader_protocol INTEGER NOT NULL CHECK (minimum_reader_protocol >= 1), + minimum_writer_protocol INTEGER NOT NULL CHECK (minimum_writer_protocol >= minimum_reader_protocol), + installed_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + INSERT INTO database_capability( + capability, minimum_reader_protocol, minimum_writer_protocol, installed_at + ) VALUES ( + 'bounded_event_snapshot_v1', 2, 2, unixepoch('subsec') * 1000 + ) + `) + yield* tx.run(` + CREATE TRIGGER database_capability_immutable_update + BEFORE UPDATE ON database_capability + BEGIN + SELECT RAISE(ABORT, 'database_capability_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER database_capability_immutable_delete + BEFORE DELETE ON database_capability + BEGIN + SELECT RAISE(ABORT, 'database_capability_immutable'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index ab4135bad..85917311a 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -1,14 +1,20 @@ export * as EventV2 from "./event" import { Cause, Context, Effect, Layer, Option, PubSub, Schema, Stream } from "effect" -import { and, asc, eq, gt, inArray, lte, sql } from "drizzle-orm" +import { and, asc, eq, gt, inArray, lte, or, sql } from "drizzle-orm" import { Database } from "./database/database" import { EventArtifactTable, EventArtifactChunkTable, + EventCompactionReceiptTable, EventDedupeTable, EventSequenceTable, + EventSnapshotChunkTable, + EventSnapshotAttemptTable, + EventSnapshotRowTable, EventSnapshotTable, + EventSyncBackfillTable, + EventSyncIndexTable, EventSyncSequenceTable, EventTable, } from "./event/sql" @@ -19,6 +25,8 @@ import { Identifier } from "./util/identifier" import { isDeepStrictEqual } from "node:util" import { Hash } from "./util/hash" import { createHash } from "node:crypto" +import { FilePartArtifact } from "./file-part-artifact" +import { FilePartArtifactBindingTable } from "./file-part-artifact.sql" export const ID = Schema.String.check(Schema.isStartsWith("evt_")).pipe( Schema.brand("Event.ID"), @@ -92,6 +100,70 @@ export type SerializedSnapshot = { readonly createdAt: number } +export const SnapshotManifest = Schema.Struct({ + format: Schema.Literal("chunked-rows.v1"), + projectionRevision: Schema.String, + contentHash: Schema.String, + rowCount: NonNegativeInt, + encodedBytes: NonNegativeInt, + tables: Schema.Record(Schema.String, NonNegativeInt), +}) +export type SnapshotManifest = typeof SnapshotManifest.Type + +export type SerializedSnapshotRow = { + readonly snapshotID: string + readonly rowIndex: number + readonly tableName: string + readonly rowKey: string + readonly rowHash: string + readonly rowBytes: number + readonly chunkCount: number + readonly chainHash: string +} + +export type SerializedSnapshotChunk = { + readonly rowHash: string + readonly chunkIndex: number + readonly data: Buffer + readonly chunkHash: string +} + +export type SnapshotProjectionRow = { + readonly cursor: string + readonly tableName: string + readonly rowKey: string + readonly value: Record +} + +export type SnapshotCodec = { + readonly codec: string + readonly schemaVersion: number + readonly rebuildEventTypes: ReadonlySet + readonly revision: (aggregateID: string) => Effect.Effect + readonly next: (aggregateID: string, cursor?: string) => Effect.Effect + readonly clear: (aggregateID: string, snapshotID: string, manifest: SnapshotManifest) => Effect.Effect + readonly import: ( + aggregateID: string, + row: SnapshotProjectionRow, + ownerID: string | undefined, + ) => Effect.Effect +} + +export type SnapshotAttempt = { + readonly snapshotID: string + readonly aggregateID: string + readonly throughSeq: number + readonly expectedLatest: number + readonly ownerID?: string + readonly codec: string + readonly schemaVersion: number + readonly cursor?: string + readonly rowCount: number + readonly encodedBytes: number + readonly state: "prepared" | "staged" | "complete" + readonly hasMore: boolean +} + export type CursorEvent = { readonly cursor: Cursor readonly event: E @@ -109,6 +181,13 @@ export const MAX_ENCODED_PAYLOAD_BYTES = 4 * 1024 * 1024 export const AGGREGATE_READ_BATCH_EVENTS = 100 export const AGGREGATE_READ_BATCH_BYTES = 4 * 1024 * 1024 export const SNAPSHOT_MAX_ENCODED_BYTES = 16 * 1024 * 1024 +export const SNAPSHOT_ROW_MAX_ENCODED_BYTES = 32 * 1024 * 1024 +export const SNAPSHOT_MAX_TOTAL_BYTES = 64 * 1024 * 1024 +export const SNAPSHOT_MAX_ROWS = 1_000_000 +export const SNAPSHOT_CHUNK_BYTES = 256 * 1024 +export const SNAPSHOT_TRANSFER_ROWS = 100 +export const SNAPSHOT_STAGE_BATCH_BYTES = 8 * 1024 * 1024 +export const SNAPSHOT_TRANSFER_CHUNKS = 16 export const LEGACY_ARTIFACT_BATCH_EVENTS = 1 export const LEGACY_DIFF_DESCRIPTOR_FILES = 200 export const ARTIFACT_CHUNK_BYTES = 256 * 1024 @@ -135,6 +214,15 @@ export class ResyncRequiredError extends Schema.TaggedErrorClass()( + "EventV2.MaintenanceRequired", + { + operation: Schema.String, + reason: Schema.String, + message: Schema.String, + }, +) {} + export function versionedType(type: string, version: number) { return `${type}.${version}` } @@ -163,13 +251,14 @@ function admitEncodedPayload(definition: SyncDefinition, data: unknown) { } function stableJson(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]` + if (Array.isArray(value)) return `[${value.map((item) => item === undefined ? "null" : stableJson(item)).join(",")}]` if (value && typeof value === "object") return `{${Object.entries(value as Record) + .filter(([, item]) => item !== undefined) .sort(([left], [right]) => left.localeCompare(right)) .map(([key, item]) => `${JSON.stringify(key)}:${stableJson(item)}`) .join(",")}}` - return JSON.stringify(value) + return JSON.stringify(value) ?? "null" } function hashJson(value: unknown) { @@ -252,6 +341,7 @@ export interface Interface { readonly listen: (listener: Listener) => Effect.Effect readonly beforeCommit: (guard: CommitGuard) => Effect.Effect readonly project: (definition: D, projector: Projector) => Effect.Effect + readonly registerSnapshotCodec?: (codec: SnapshotCodec) => Effect.Effect readonly replay: ( event: SerializedEvent, options?: { readonly publish?: boolean; readonly ownerID?: string; readonly strictOwner?: boolean }, @@ -261,17 +351,57 @@ export interface Interface { options?: { readonly publish?: boolean; readonly ownerID?: string; readonly strictOwner?: boolean }, ) => Effect.Effect readonly snapshot: (aggregateID: string) => Effect.Effect + readonly prepareCheckpoint?: (input: { + readonly aggregateID: string + readonly throughSeq: Cursor + readonly codec: string + readonly schemaVersion: number + readonly expectedLatest: Cursor + readonly ownerID?: string + readonly now?: number + }) => Effect.Effect + readonly stageCheckpoint?: (input: { + readonly snapshotID: string + readonly limit?: number + readonly now?: number + }) => Effect.Effect + readonly finalizeCheckpoint?: (input: { + readonly snapshotID: string + readonly now?: number + }) => Effect.Effect + readonly discardCheckpoint?: (input: { + readonly snapshotID: string + readonly limit?: number + }) => Effect.Effect<{ readonly deletedRows: number; readonly complete: boolean }> readonly checkpoint: (input: { readonly aggregateID: string readonly throughSeq: Cursor readonly codec: string readonly schemaVersion: number - readonly body: Record readonly expectedLatest: Cursor readonly ownerID?: string readonly now?: number }) => Effect.Effect readonly importSnapshot: (snapshot: SerializedSnapshot) => Effect.Effect + readonly snapshotRows?: (input: { + readonly snapshotID: string + readonly after?: number + readonly limit?: number + }) => Effect.Effect + readonly snapshotChunks?: (input: { + readonly rowHash: string + readonly after?: number + readonly limit?: number + }) => Effect.Effect + readonly stageSnapshotRows?: ( + snapshot: SerializedSnapshot, + rows: readonly SerializedSnapshotRow[], + ) => Effect.Effect + readonly stageSnapshotChunks?: ( + snapshot: SerializedSnapshot, + row: SerializedSnapshotRow, + chunks: readonly SerializedSnapshotChunk[], + ) => Effect.Effect readonly compact: (input: { readonly aggregateID: string readonly throughSeq: Cursor @@ -283,6 +413,10 @@ export interface Interface { readonly limit?: number readonly now?: number }) => Effect.Effect<{ readonly processed: number; readonly next?: ID }> + readonly backfillSyncIndex?: (input?: { + readonly limit?: number + readonly now?: number + }) => Effect.Effect<{ readonly processed: number; readonly complete: boolean }> readonly remove: (aggregateID: string) => Effect.Effect readonly claim: (aggregateID: string, ownerID: string) => Effect.Effect } @@ -304,6 +438,7 @@ export const layerWith = (layerOptions?: LayerOptions) => const synchronized = new Map>>() const typed = new Map>() const projectors = new Map() + const snapshotCodecs = new Map() const commitGuards = new Array() const listeners = new Array() const syncHandlers = new Array() @@ -371,10 +506,16 @@ export const layerWith = (layerOptions?: LayerOptions) => }), ) } - const admission = admitEncodedPayload( - syncRegistry.get(versionedType(definition.type, sync.version))!, - event.data, + const codec = syncRegistry.get(versionedType(definition.type, sync.version))! + const original = codec.encode(event.data) as Record + const prepared = FilePartArtifact.prepare( + definition.type, + original, + Buffer.byteLength(JSON.stringify(original)), + MAX_ENCODED_PAYLOAD_BYTES, ) + const canonicalEvent = { ...event, data: codec.decode(prepared.data) } as Payload + const admission = admitEncodedPayload(codec, canonicalEvent.data) if ("error" in admission) return yield* Effect.die(admission.error) const encoded = admission.encoded const list = projectors.get(event.type) ?? [] @@ -385,7 +526,11 @@ export const layerWith = (layerOptions?: LayerOptions) => () => Effect.gen(function* () { const row = yield* db - .select({ seq: EventSequenceTable.seq, ownerID: EventSequenceTable.owner_id }) + .select({ + seq: EventSequenceTable.seq, + ownerID: EventSequenceTable.owner_id, + writeFence: EventSequenceTable.write_fence_transfer_id, + }) .from(EventSequenceTable) .where(eq(EventSequenceTable.aggregate_id, aggregateID)) .get() @@ -418,6 +563,17 @@ export const layerWith = (layerOptions?: LayerOptions) => .get() .pipe(Effect.orDie) : undefined + const fileArtifact = stored + ? yield* db + .select({ + originalDataHash: FilePartArtifactBindingTable.original_data_hash, + canonicalData: FilePartArtifactBindingTable.canonical_data, + }) + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, stored.id)) + .get() + .pipe(Effect.orDie) + : undefined const dedupe = !stored ? yield* db .select() @@ -436,6 +592,9 @@ export const layerWith = (layerOptions?: LayerOptions) => stored.type === versionedType(definition.type, sync.version) && (isDeepStrictEqual(stored.data, encoded) || isDeepStrictEqual(artifact?.canonicalData, encoded) || + (isDeepStrictEqual(fileArtifact?.canonicalData, encoded) && + (fileArtifact?.originalDataHash === Hash.sha256(JSON.stringify(original)) || + isDeepStrictEqual(stored.data, original))) || (artifact && artifact.originalDataHash === (artifact.codecVersion >= 2 @@ -465,6 +624,13 @@ export const layerWith = (layerOptions?: LayerOptions) => }), ) } + if (row?.writeFence) + return yield* Effect.die( + new InvalidSyncEventError({ + type: event.type, + message: `Aggregate ${aggregateID} is fenced by transfer ${row.writeFence}`, + }), + ) if (input && row?.ownerID && row.ownerID !== input.ownerID) { return } @@ -505,11 +671,18 @@ export const layerWith = (layerOptions?: LayerOptions) => message: `Event ${event.id} already exists at aggregate ${stored.aggregateID} sequence ${stored.seq}`, }), ) + yield* Effect.forEach(prepared.artifacts, (artifact) => FilePartArtifact.put(db, artifact), { + discard: true, + }) + if (prepared.artifacts.length === 0) { + const descriptor = FilePartArtifact.descriptor(encoded) + if (descriptor) yield* FilePartArtifact.requireAvailable(db, descriptor) + } for (const guard of commitGuards) { - yield* guard(event) + yield* guard(canonicalEvent) } for (const projector of list) { - yield* projector({ ...event, seq } as Payload) + yield* projector({ ...canonicalEvent, seq } as Payload) } if (commit) yield* commit(seq) yield* db @@ -549,7 +722,16 @@ export const layerWith = (layerOptions?: LayerOptions) => ]) .run() .pipe(Effect.orDie) - return { aggregateID, seq, inserted: true } + yield* FilePartArtifact.bind(db, { + eventID: event.id, + aggregateID, + seq, + type: definition.type, + data: encoded, + originalData: original, + requireImport: Boolean(canonicalEvent.replay), + }) + return { aggregateID, seq, inserted: true, event: canonicalEvent } }), { behavior: "immediate" }, ) @@ -586,7 +768,7 @@ export const layerWith = (layerOptions?: LayerOptions) => if (durable) { const committed = yield* commitSyncEvent(event as Payload, undefined, commit, idempotent) if (committed) { - event = { ...event, seq: committed.seq } + event = { ...(committed.event ?? event), seq: committed.seq } as Payload if (committed.inserted) { yield* Effect.forEach(syncHandlers, (sync) => observe(event as Payload, "sync", sync), { discard: true, @@ -694,9 +876,9 @@ export const layerWith = (layerOptions?: LayerOptions) => false, options?.deferDurableWake, ) - if (committed) options?.onCommitted?.({ ...payload, seq: committed.seq }) + if (committed) options?.onCommitted?.({ ...(committed.event ?? payload), seq: committed.seq }) if (committed && options?.publish) { - yield* notify({ ...payload, seq: committed.seq }, true) + yield* notify({ ...(committed.event ?? payload), seq: committed.seq }, true) } } }) @@ -818,141 +1000,295 @@ export const layerWith = (layerOptions?: LayerOptions) => readonly throughSeq: Cursor readonly codec: string readonly schemaVersion: number - readonly body: Record readonly expectedLatest: Cursor readonly ownerID?: string readonly now?: number }) { return Effect.gen(function* () { - return yield* Effect.die( - new InvalidSyncEventError({ + const attempt = yield* prepareCheckpoint(input) + if (attempt.state === "complete") { + const existing = yield* snapshot(input.aggregateID) + if (existing?.snapshotID === attempt.snapshotID) return existing + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", - message: "Snapshot checkpointing is disabled until a registered canonical projection codec is available", - }), - ) - /* c8 ignore start - retained design is unreachable until a projection codec is registered */ - if (input.schemaVersion < 1 || input.codec.length === 0) - return yield* Effect.die( - new InvalidSyncEventError({ type: "snapshot", message: "Snapshot codec and schema version are required" }), - ) - const encodedBytes = Buffer.byteLength(JSON.stringify(input.body)) - if (encodedBytes > SNAPSHOT_MAX_ENCODED_BYTES) - return yield* Effect.die( - new EncodedPayloadTooLargeError({ - type: "event.snapshot", - encodedBytes, - limitBytes: SNAPSHOT_MAX_ENCODED_BYTES, - message: `Snapshot body is ${encodedBytes} bytes; limit is ${SNAPSHOT_MAX_ENCODED_BYTES} bytes`, - }), - ) - const snapshotHash = hashJson(input.body) - const snapshotID = `evtsnap_${Hash.sha256( - `${input.codec}:${input.schemaVersion}:${input.aggregateID}:${input.throughSeq}:${snapshotHash}`, - )}` - const result = { - snapshotID, - aggregateID: input.aggregateID, - throughSeq: input.throughSeq, - syncSeq: 0, - codec: input.codec, - schemaVersion: input.schemaVersion, - snapshotHash, - body: input.body, - ...(input.ownerID ? { ownerID: input.ownerID } : {}), - createdAt: input.now ?? Date.now(), + message: `Completed snapshot attempt ${attempt.snapshotID} is not active`, + })) } - const syncSeq = yield* db - .transaction( - () => - Effect.gen(function* () { - const current = yield* db - .select({ - seq: EventSequenceTable.seq, - ownerID: EventSequenceTable.owner_id, - floor: EventSequenceTable.retention_floor_seq, - }) - .from(EventSequenceTable) - .where(eq(EventSequenceTable.aggregate_id, input.aggregateID)) - .get() - .pipe(Effect.orDie) - if ( - !current || - current.seq !== input.expectedLatest || - input.throughSeq > current.seq || - input.throughSeq < (current.floor ?? -1) || - (input.ownerID !== undefined && current.ownerID !== input.ownerID) - ) - return yield* Effect.die( - new InvalidSyncEventError({ - type: "snapshot", - message: `Snapshot authority changed for aggregate ${input.aggregateID}`, - }), - ) - const syncSequence = yield* db - .update(EventSyncSequenceTable) - .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) - .where(eq(EventSyncSequenceTable.id, 1)) - .returning({ seq: EventSyncSequenceTable.seq }) - .get() - .pipe(Effect.orDie) - if (!syncSequence) - return yield* Effect.die( - new InvalidSyncEventError({ type: "snapshot", message: "Sync sequence authority missing" }), - ) - yield* db - .insert(EventSnapshotTable) - .values({ - snapshot_id: snapshotID, - aggregate_id: input.aggregateID, - through_seq: input.throughSeq, - sync_seq: syncSequence.seq, - codec: input.codec, - schema_version: input.schemaVersion, - snapshot_hash: snapshotHash, - body: input.body, - owner_id: input.ownerID, - created_at: result.createdAt, - }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie) - yield* db - .update(EventSequenceTable) - .set({ retention_floor_seq: input.throughSeq, snapshot_id: snapshotID }) - .where( - and( - eq(EventSequenceTable.aggregate_id, input.aggregateID), - eq(EventSequenceTable.seq, input.expectedLatest), - ), - ) - .run() - .pipe(Effect.orDie) - return syncSequence.seq - }), - { behavior: "immediate" }, - ) - .pipe(Effect.orDie) - return { ...result, syncSeq } - /* c8 ignore stop */ + let staged = attempt + while (staged.state !== "staged") staged = yield* stageCheckpoint({ snapshotID: staged.snapshotID }) + return yield* finalizeCheckpoint({ snapshotID: staged.snapshotID, now: input.now }) }) } + const snapshotAttempt = (row: typeof EventSnapshotAttemptTable.$inferSelect): SnapshotAttempt => ({ + snapshotID: row.snapshot_id, + aggregateID: row.aggregate_id, + throughSeq: row.through_seq, + expectedLatest: row.expected_latest, + ...(row.owner_id ? { ownerID: row.owner_id } : {}), + codec: row.codec, + schemaVersion: row.schema_version, + ...(row.cursor ? { cursor: row.cursor } : {}), + rowCount: row.row_count, + encodedBytes: row.encoded_bytes, + state: row.state, + hasMore: row.state === "prepared", + }) + + function prepareCheckpoint(input: { + readonly aggregateID: string + readonly throughSeq: Cursor + readonly codec: string + readonly schemaVersion: number + readonly expectedLatest: Cursor + readonly ownerID?: string + readonly now?: number + }) { + return db.transaction( + () => Effect.gen(function* () { + const codec = snapshotCodecs.get(`${input.codec}@${input.schemaVersion}`) + if (!codec) + return yield* Effect.die(new InvalidSyncEventError({ + type: "snapshot", + message: `Unsupported snapshot codec ${input.codec}@${input.schemaVersion}`, + })) + const projectionRevision = yield* codec.revision(input.aggregateID) + const snapshotID = `evtsnap_${Hash.sha256( + `${input.codec}:${input.schemaVersion}:${input.aggregateID}:${input.throughSeq}:${projectionRevision}`, + )}` + const syncAuthority = yield* db.select({ complete: EventSyncSequenceTable.backfill_complete }) + .from(EventSyncSequenceTable).where(eq(EventSyncSequenceTable.id, 1)).get().pipe(Effect.orDie) + if (!syncAuthority?.complete) + return yield* Effect.die(new MaintenanceRequiredError({ + operation: "snapshot.checkpoint", + reason: "event_sync_backfill_required", + message: "Event sync index backfill must complete before checkpointing", + })) + const current = yield* db.select({ + seq: EventSequenceTable.seq, + ownerID: EventSequenceTable.owner_id, + floor: EventSequenceTable.retention_floor_seq, + fence: EventSequenceTable.write_fence_transfer_id, + }).from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, input.aggregateID)).get().pipe(Effect.orDie) + if (!current || current.seq !== input.expectedLatest || input.throughSeq !== current.seq || + input.throughSeq < (current.floor ?? -1) || current.fence !== null || + (input.ownerID !== undefined && current.ownerID !== input.ownerID)) + return yield* Effect.die(new InvalidSyncEventError({ + type: "snapshot", + message: `Snapshot authority changed or is fenced for aggregate ${input.aggregateID}`, + })) + const existing = yield* db.select().from(EventSnapshotAttemptTable) + .where(eq(EventSnapshotAttemptTable.snapshot_id, snapshotID)).get().pipe(Effect.orDie) + if (existing) return snapshotAttempt(existing) + const now = input.now ?? Date.now() + const inserted = yield* db.insert(EventSnapshotAttemptTable).values({ + snapshot_id: snapshotID, + aggregate_id: input.aggregateID, + through_seq: input.throughSeq, + expected_latest: input.expectedLatest, + owner_id: input.ownerID, + codec: input.codec, + schema_version: input.schemaVersion, + projection_revision: projectionRevision, + row_count: 0, + encoded_bytes: 0, + content_hash: Hash.sha256(""), + tables: {}, + state: "prepared", + created_at: now, + updated_at: now, + }).returning().get().pipe(Effect.orDie) + return snapshotAttempt(inserted!) + }), + { behavior: "immediate" }, + ).pipe(Effect.orDie) + } + + function stageCheckpoint(input: { readonly snapshotID: string; readonly limit?: number; readonly now?: number }) { + const limit = Math.min(Math.max(input.limit ?? SNAPSHOT_TRANSFER_ROWS, 1), SNAPSHOT_TRANSFER_ROWS) + return db.transaction( + () => Effect.gen(function* () { + const attempt = yield* db.select().from(EventSnapshotAttemptTable) + .where(eq(EventSnapshotAttemptTable.snapshot_id, input.snapshotID)).get().pipe(Effect.orDie) + if (!attempt) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Unknown snapshot attempt ${input.snapshotID}` })) + if (attempt.state !== "prepared") return snapshotAttempt(attempt) + const codec = snapshotCodecs.get(`${attempt.codec}@${attempt.schema_version}`) + if (!codec) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Unsupported snapshot codec ${attempt.codec}@${attempt.schema_version}` })) + let cursor = attempt.cursor ?? undefined + let rowCount = attempt.row_count + let encodedBytes = attempt.encoded_bytes + let batchBytes = 0 + const tables = { ...attempt.tables } + let staged = 0 + let exhausted = false + while (staged < limit) { + const row = yield* codec.next(attempt.aggregate_id, cursor) + if (!row) { + exhausted = true + break + } + if (rowCount >= SNAPSHOT_MAX_ROWS) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot exceeds ${SNAPSHOT_MAX_ROWS} projection rows` })) + const data = Buffer.from(stableJson(row.value)) + if (data.length > SNAPSHOT_ROW_MAX_ENCODED_BYTES) return yield* Effect.die(new EncodedPayloadTooLargeError({ + type: `event.snapshot.${row.tableName}`, encodedBytes: data.length, limitBytes: SNAPSHOT_ROW_MAX_ENCODED_BYTES, + message: `Snapshot row ${row.tableName}:${row.rowKey} exceeds the per-row limit`, + })) + if (encodedBytes + data.length > SNAPSHOT_MAX_TOTAL_BYTES) return yield* Effect.die(new EncodedPayloadTooLargeError({ + type: "event.snapshot", encodedBytes: encodedBytes + data.length, limitBytes: SNAPSHOT_MAX_TOTAL_BYTES, + message: `Snapshot exceeds the ${SNAPSHOT_MAX_TOTAL_BYTES} byte total limit`, + })) + if (staged > 0 && batchBytes + data.length > SNAPSHOT_STAGE_BATCH_BYTES) break + encodedBytes += data.length + const rowHash = Hash.sha256(data) + const chunks = Math.ceil(data.length / SNAPSHOT_CHUNK_BYTES) + const chainHash = Hash.sha256(`${attempt.content_hash}\0${row.tableName}\0${row.rowKey}\0${rowHash}\0${data.length}`) + const insertedRow = yield* db.insert(EventSnapshotRowTable).values({ + snapshot_id: input.snapshotID, row_index: rowCount, table_name: row.tableName, row_key: row.rowKey, + row_hash: rowHash, row_bytes: data.length, chunk_count: chunks, chain_hash: chainHash, + }).onConflictDoNothing().returning().get().pipe(Effect.orDie) + if (!insertedRow) { + const existing = yield* db.select().from(EventSnapshotRowTable).where(and( + eq(EventSnapshotRowTable.snapshot_id, input.snapshotID), + eq(EventSnapshotRowTable.row_index, rowCount), + )).get().pipe(Effect.orDie) + if (!existing || existing.table_name !== row.tableName || existing.row_key !== row.rowKey || + existing.row_hash !== rowHash || existing.row_bytes !== data.length || existing.chunk_count !== chunks || + existing.chain_hash !== chainHash) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot row ${rowCount} conflicts with staged content` })) + } + yield* Effect.forEach(Array.from({ length: chunks }, (_, index) => index), (index) => { + const chunk = data.subarray(index * SNAPSHOT_CHUNK_BYTES, (index + 1) * SNAPSHOT_CHUNK_BYTES) + return db.insert(EventSnapshotChunkTable).values({ row_hash: rowHash, chunk_index: index, data: chunk, chunk_hash: Hash.sha256(chunk) }) + .onConflictDoNothing().returning().get().pipe( + Effect.orDie, + Effect.flatMap((inserted) => { + if (inserted) return Effect.void + return db.select().from(EventSnapshotChunkTable).where(and( + eq(EventSnapshotChunkTable.row_hash, rowHash), + eq(EventSnapshotChunkTable.chunk_index, index), + )).get().pipe( + Effect.orDie, + Effect.flatMap((existing) => existing && existing.chunk_hash === Hash.sha256(chunk) && Buffer.from(existing.data).equals(chunk) + ? Effect.void + : Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot chunk ${rowHash}:${index} conflicts with staged content` }))), + ) + }), + ) + }, { discard: true }) + tables[row.tableName] = (tables[row.tableName] ?? 0) + 1 + attempt.content_hash = chainHash + cursor = row.cursor + rowCount += 1 + batchBytes += data.length + staged += 1 + } + const updated = yield* db.update(EventSnapshotAttemptTable).set({ + cursor, + row_count: rowCount, + encoded_bytes: encodedBytes, + tables, + content_hash: attempt.content_hash, + state: exhausted ? "staged" : "prepared", + updated_at: input.now ?? Date.now(), + }).where(and(eq(EventSnapshotAttemptTable.snapshot_id, input.snapshotID), eq(EventSnapshotAttemptTable.row_count, attempt.row_count))) + .returning().get().pipe(Effect.orDie) + if (!updated) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot attempt ${input.snapshotID} changed concurrently` })) + return snapshotAttempt(updated) + }), + { behavior: "immediate" }, + ).pipe(Effect.orDie) + } + + function finalizeCheckpoint(input: { readonly snapshotID: string; readonly now?: number }) { + return db.transaction( + () => Effect.gen(function* () { + const attempt = yield* db.select().from(EventSnapshotAttemptTable) + .where(eq(EventSnapshotAttemptTable.snapshot_id, input.snapshotID)).get().pipe(Effect.orDie) + if (!attempt || attempt.state !== "staged") return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot attempt ${input.snapshotID} is not fully staged` })) + if (attempt.tables.session !== 1) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Canonical projection for ${attempt.aggregate_id} must contain one Session root` })) + const current = yield* db.select({ seq: EventSequenceTable.seq, ownerID: EventSequenceTable.owner_id, fence: EventSequenceTable.write_fence_transfer_id }) + .from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, attempt.aggregate_id)).get().pipe(Effect.orDie) + if (!current || current.seq !== attempt.expected_latest || current.ownerID !== attempt.owner_id || current.fence !== null) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot finalize authority changed for aggregate ${attempt.aggregate_id}` })) + const codec = snapshotCodecs.get(`${attempt.codec}@${attempt.schema_version}`) + if (!codec || (yield* codec.revision(attempt.aggregate_id)) !== attempt.projection_revision) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot projection changed for aggregate ${attempt.aggregate_id}` })) + const rowCount = yield* db.$count(EventSnapshotRowTable, eq(EventSnapshotRowTable.snapshot_id, input.snapshotID)).pipe(Effect.orDie) + const lastRow = yield* db.select({ index: EventSnapshotRowTable.row_index, chainHash: EventSnapshotRowTable.chain_hash }) + .from(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID)) + .orderBy(sql`${EventSnapshotRowTable.row_index} DESC`).limit(1).get().pipe(Effect.orDie) + const body = SnapshotManifest.make({ format: "chunked-rows.v1", projectionRevision: attempt.projection_revision, + contentHash: attempt.content_hash, rowCount: attempt.row_count, encodedBytes: attempt.encoded_bytes, tables: attempt.tables }) + if (rowCount !== attempt.row_count) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot attempt ${input.snapshotID} lost staged rows` })) + if (!lastRow || lastRow.index !== attempt.row_count - 1 || lastRow.chainHash !== attempt.content_hash) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot attempt ${input.snapshotID} metadata chain is incomplete` })) + const syncSequence = yield* db.update(EventSyncSequenceTable).set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)).returning({ seq: EventSyncSequenceTable.seq }).get().pipe(Effect.orDie) + if (!syncSequence) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: "Sync sequence authority missing" })) + const createdAt = input.now ?? Date.now() + const snapshotHash = hashJson(body) + yield* db.insert(EventSnapshotTable).values({ snapshot_id: input.snapshotID, aggregate_id: attempt.aggregate_id, through_seq: attempt.through_seq, + sync_seq: syncSequence.seq, codec: attempt.codec, schema_version: attempt.schema_version, snapshot_hash: snapshotHash, + body, owner_id: attempt.owner_id, created_at: createdAt }).run().pipe(Effect.orDie) + yield* db.update(EventSequenceTable).set({ retention_floor_seq: attempt.through_seq, snapshot_id: input.snapshotID }) + .where(and(eq(EventSequenceTable.aggregate_id, attempt.aggregate_id), eq(EventSequenceTable.seq, attempt.expected_latest))).run().pipe(Effect.orDie) + yield* db.update(EventSnapshotAttemptTable).set({ state: "complete", content_hash: body.contentHash, updated_at: createdAt }) + .where(eq(EventSnapshotAttemptTable.snapshot_id, input.snapshotID)).run().pipe(Effect.orDie) + return { snapshotID: input.snapshotID, aggregateID: attempt.aggregate_id, throughSeq: attempt.through_seq, + syncSeq: syncSequence.seq, codec: attempt.codec, schemaVersion: attempt.schema_version, snapshotHash, body, + ...(attempt.owner_id ? { ownerID: attempt.owner_id } : {}), createdAt } + }), + { behavior: "immediate" }, + ).pipe(Effect.orDie) + } + + function discardCheckpoint(input: { readonly snapshotID: string; readonly limit?: number }) { + const limit = Math.min(Math.max(input.limit ?? SNAPSHOT_TRANSFER_ROWS, 1), SNAPSHOT_TRANSFER_ROWS) + return db.transaction( + () => Effect.gen(function* () { + const attempt = yield* db.select({ state: EventSnapshotAttemptTable.state }).from(EventSnapshotAttemptTable) + .where(eq(EventSnapshotAttemptTable.snapshot_id, input.snapshotID)).get().pipe(Effect.orDie) + if (!attempt || attempt.state !== "prepared") return yield* Effect.die(new InvalidSyncEventError({ + type: "snapshot", message: `Only a prepared snapshot attempt can be discarded: ${input.snapshotID}`, + })) + const rows = yield* db.select({ rowIndex: EventSnapshotRowTable.row_index, rowHash: EventSnapshotRowTable.row_hash }) + .from(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID)) + .orderBy(asc(EventSnapshotRowTable.row_index)).limit(limit).all().pipe(Effect.orDie) + if (rows.length > 0) yield* db.delete(EventSnapshotRowTable).where(and(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID), inArray(EventSnapshotRowTable.row_index, rows.map((row) => row.rowIndex)))).run().pipe(Effect.orDie) + yield* Effect.forEach(rows, (row) => db.delete(EventSnapshotChunkTable).where(and( + eq(EventSnapshotChunkTable.row_hash, row.rowHash), + sql`NOT EXISTS (SELECT 1 FROM ${EventSnapshotRowTable} WHERE ${EventSnapshotRowTable.row_hash} = ${row.rowHash})`, + )).run().pipe(Effect.orDie), { discard: true }) + const remaining = yield* db.select({ rowIndex: EventSnapshotRowTable.row_index }).from(EventSnapshotRowTable) + .where(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID)).limit(1).get().pipe(Effect.orDie) + if (!remaining) yield* db.delete(EventSnapshotAttemptTable).where(and(eq(EventSnapshotAttemptTable.snapshot_id, input.snapshotID), eq(EventSnapshotAttemptTable.state, "prepared"))).run().pipe(Effect.orDie) + return { deletedRows: rows.length, complete: !remaining } + }), + { behavior: "immediate" }, + ).pipe(Effect.orDie) + } + function importSnapshot(input: SerializedSnapshot) { return Effect.gen(function* () { - return yield* Effect.die( - new InvalidSyncEventError({ - type: "snapshot", - message: `Unsupported snapshot codec ${input.codec}@${input.schemaVersion}; canonical projection import is disabled`, - }), - ) - /* c8 ignore start - retained design is unreachable until a projection codec is registered */ - if (hashJson(input.body) !== input.snapshotHash) + const codec = snapshotCodecs.get(`${input.codec}@${input.schemaVersion}`) + if (!codec) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Unsupported snapshot codec ${input.codec}@${input.schemaVersion}`, + }), + ) + const manifest = Schema.decodeUnknownSync(SnapshotManifest)(input.body) + if (hashJson(manifest) !== input.snapshotHash) return yield* Effect.die( new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} hash mismatch` }), ) - if (Buffer.byteLength(JSON.stringify(input.body)) > SNAPSHOT_MAX_ENCODED_BYTES) + const expectedSnapshotID = `evtsnap_${Hash.sha256( + `${input.codec}:${input.schemaVersion}:${input.aggregateID}:${input.throughSeq}:${manifest.projectionRevision}`, + )}` + if (input.snapshotID !== expectedSnapshotID) return yield* Effect.die( - new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} exceeds limit` }), + new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} identity mismatch` }), ) yield* db .transaction( @@ -964,11 +1300,106 @@ export const layerWith = (layerOptions?: LayerOptions) => .where(eq(EventSequenceTable.aggregate_id, input.aggregateID)) .get() .pipe(Effect.orDie) - if (current && current.seq !== input.throughSeq) + const syncAuthority = yield* db.select({ complete: EventSyncSequenceTable.backfill_complete }) + .from(EventSyncSequenceTable).where(eq(EventSyncSequenceTable.id, 1)).get().pipe(Effect.orDie) + if (!syncAuthority?.complete) return yield* Effect.die(new MaintenanceRequiredError({ + operation: "snapshot.import", reason: "event_sync_backfill_required", + message: "Event sync index backfill must complete before importing a snapshot", + })) + if ( + current && + (current.seq > input.throughSeq || + current.write_fence_transfer_id !== null || + (current.owner_id !== null && current.owner_id !== input.ownerID)) + ) return yield* Effect.die( new InvalidSyncEventError({ type: "snapshot", - message: `Snapshot import requires an empty or exact aggregate ${input.aggregateID}`, + message: `Snapshot import authority conflicts with aggregate ${input.aggregateID}`, + }), + ) + if (current?.snapshot_id === input.snapshotID && current.seq === input.throughSeq) return + let contentHash = Hash.sha256("") + const tables: Record = {} + let after = -1 + let rowCount = 0 + let encodedBytes = 0 + while (true) { + const rows = yield* db + .select() + .from(EventSnapshotRowTable) + .where( + and( + eq(EventSnapshotRowTable.snapshot_id, input.snapshotID), + gt(EventSnapshotRowTable.row_index, after), + ), + ) + .orderBy(asc(EventSnapshotRowTable.row_index)) + .limit(SNAPSHOT_TRANSFER_ROWS) + .all() + .pipe(Effect.orDie) + if (rows.length === 0) break + for (const row of rows) { + if (row.row_index !== rowCount) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Snapshot ${input.snapshotID} row sequence is incomplete at ${rowCount}`, + }), + ) + const chunks = yield* db + .select() + .from(EventSnapshotChunkTable) + .where(eq(EventSnapshotChunkTable.row_hash, row.row_hash)) + .orderBy(asc(EventSnapshotChunkTable.chunk_index)) + .all() + .pipe(Effect.orDie) + if ( + chunks.length !== row.chunk_count || + chunks.some( + (chunk, index) => + chunk.chunk_index !== index || Hash.sha256(chunk.data) !== chunk.chunk_hash, + ) + ) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Snapshot ${input.snapshotID} row ${row.row_index} chunks are incomplete`, + }), + ) + const data = Buffer.concat(chunks.map((chunk) => chunk.data)) + if (data.length !== row.row_bytes || Hash.sha256(data) !== row.row_hash) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Snapshot ${input.snapshotID} row ${row.row_index} hash mismatch`, + }), + ) + const value = Schema.decodeUnknownSync(Schema.UnknownFromJsonString)(data.toString()) + if (!value || typeof value !== "object" || Array.isArray(value)) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Snapshot ${input.snapshotID} row ${row.row_index} is not an object`, + }), + ) + contentHash = Hash.sha256(`${contentHash}\0${row.table_name}\0${row.row_key}\0${row.row_hash}\0${row.row_bytes}`) + tables[row.table_name] = (tables[row.table_name] ?? 0) + 1 + encodedBytes += row.row_bytes + rowCount += 1 + after = row.row_index + } + } + if ( + rowCount !== manifest.rowCount || + encodedBytes !== manifest.encodedBytes || + !isDeepStrictEqual(tables, manifest.tables) || + contentHash !== manifest.contentHash + ) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "snapshot", + message: `Snapshot ${input.snapshotID} manifest does not match its staged rows`, }), ) yield* db @@ -978,22 +1409,52 @@ export const layerWith = (layerOptions?: LayerOptions) => seq: input.throughSeq, owner_id: input.ownerID, }) - .onConflictDoNothing() + .onConflictDoUpdate({ + target: EventSequenceTable.aggregate_id, + set: { seq: input.throughSeq, owner_id: input.ownerID }, + }) .run() .pipe(Effect.orDie) - yield* db + yield* codec.clear(input.aggregateID, input.snapshotID, manifest) + let importAfter = -1 + while (true) { + const rows = yield* db.select().from(EventSnapshotRowTable).where(and( + eq(EventSnapshotRowTable.snapshot_id, input.snapshotID), + gt(EventSnapshotRowTable.row_index, importAfter), + )).orderBy(asc(EventSnapshotRowTable.row_index)).limit(SNAPSHOT_TRANSFER_ROWS).all().pipe(Effect.orDie) + if (rows.length === 0) break + for (const row of rows) { + const chunks = yield* db.select().from(EventSnapshotChunkTable) + .where(eq(EventSnapshotChunkTable.row_hash, row.row_hash)) + .orderBy(asc(EventSnapshotChunkTable.chunk_index)).all().pipe(Effect.orDie) + const value = Schema.decodeUnknownSync(Schema.UnknownFromJsonString)(Buffer.concat(chunks.map((chunk) => chunk.data)).toString()) + yield* codec.import(input.aggregateID, { + cursor: String(row.row_index), + tableName: row.table_name, + rowKey: row.row_key, + value: value as Record, + }, input.ownerID) + importAfter = row.row_index + } + } + const syncSequence = yield* db .update(EventSyncSequenceTable) - .set({ seq: sql`max(${EventSyncSequenceTable.seq}, ${input.syncSeq})` }) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) .where(eq(EventSyncSequenceTable.id, 1)) - .run() + .returning({ seq: EventSyncSequenceTable.seq }) + .get() .pipe(Effect.orDie) + if (!syncSequence) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: "Sync sequence authority missing" }), + ) yield* db .insert(EventSnapshotTable) .values({ snapshot_id: input.snapshotID, aggregate_id: input.aggregateID, through_seq: input.throughSeq, - sync_seq: input.syncSeq, + sync_seq: syncSequence.seq, codec: input.codec, schema_version: input.schemaVersion, snapshot_hash: input.snapshotHash, @@ -1001,7 +1462,6 @@ export const layerWith = (layerOptions?: LayerOptions) => owner_id: input.ownerID, created_at: input.createdAt, }) - .onConflictDoNothing() .run() .pipe(Effect.orDie) yield* db @@ -1014,33 +1474,165 @@ export const layerWith = (layerOptions?: LayerOptions) => { behavior: "immediate" }, ) .pipe(Effect.orDie) - /* c8 ignore stop */ }) } + function snapshotRows(input: { readonly snapshotID: string; readonly after?: number; readonly limit?: number }) { + return db + .select({ + snapshotID: EventSnapshotRowTable.snapshot_id, + rowIndex: EventSnapshotRowTable.row_index, + tableName: EventSnapshotRowTable.table_name, + rowKey: EventSnapshotRowTable.row_key, + rowHash: EventSnapshotRowTable.row_hash, + rowBytes: EventSnapshotRowTable.row_bytes, + chunkCount: EventSnapshotRowTable.chunk_count, + chainHash: EventSnapshotRowTable.chain_hash, + }) + .from(EventSnapshotRowTable) + .where( + and( + eq(EventSnapshotRowTable.snapshot_id, input.snapshotID), + gt(EventSnapshotRowTable.row_index, input.after ?? -1), + ), + ) + .orderBy(asc(EventSnapshotRowTable.row_index)) + .limit(Math.min(Math.max(input.limit ?? SNAPSHOT_TRANSFER_ROWS, 1), SNAPSHOT_TRANSFER_ROWS)) + .all() + .pipe(Effect.orDie) + } + + function snapshotChunks(input: { readonly rowHash: string; readonly after?: number; readonly limit?: number }) { + return db + .select({ + rowHash: EventSnapshotChunkTable.row_hash, + chunkIndex: EventSnapshotChunkTable.chunk_index, + data: EventSnapshotChunkTable.data, + chunkHash: EventSnapshotChunkTable.chunk_hash, + }) + .from(EventSnapshotChunkTable) + .where( + and( + eq(EventSnapshotChunkTable.row_hash, input.rowHash), + gt(EventSnapshotChunkTable.chunk_index, input.after ?? -1), + ), + ) + .orderBy(asc(EventSnapshotChunkTable.chunk_index)) + .limit(Math.min(Math.max(input.limit ?? SNAPSHOT_TRANSFER_CHUNKS, 1), SNAPSHOT_TRANSFER_CHUNKS)) + .all() + .pipe(Effect.orDie) + } + + function stageSnapshotRows(input: SerializedSnapshot, rows: readonly SerializedSnapshotRow[]) { + return db.transaction(() => Effect.gen(function* () { + if (rows.length > SNAPSHOT_TRANSFER_ROWS) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot row page exceeds ${SNAPSHOT_TRANSFER_ROWS}` })) + if (hashJson(Schema.decodeUnknownSync(SnapshotManifest)(input.body)) !== input.snapshotHash) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} hash mismatch` }), + ) + yield* Effect.forEach( + rows, + (row) => Effect.gen(function* () { + if (row.snapshotID !== input.snapshotID) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: "Snapshot row identity mismatch" }), + ) + const inserted = yield* db + .insert(EventSnapshotRowTable) + .values({ + snapshot_id: row.snapshotID, + row_index: row.rowIndex, + table_name: row.tableName, + row_key: row.rowKey, + row_hash: row.rowHash, + row_bytes: row.rowBytes, + chunk_count: row.chunkCount, + chain_hash: row.chainHash, + }) + .onConflictDoNothing() + .returning().get() + .pipe(Effect.orDie) + if (inserted) return + const existing = yield* db.select().from(EventSnapshotRowTable).where(and( + eq(EventSnapshotRowTable.snapshot_id, row.snapshotID), + eq(EventSnapshotRowTable.row_index, row.rowIndex), + )).get().pipe(Effect.orDie) + if (!existing || existing.table_name !== row.tableName || existing.row_key !== row.rowKey || + existing.row_hash !== row.rowHash || existing.row_bytes !== row.rowBytes || existing.chunk_count !== row.chunkCount || + existing.chain_hash !== row.chainHash) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot row ${row.rowIndex} conflicts with durable CAS` })) + }), + { discard: true }, + ) + }), { behavior: "immediate" }).pipe(Effect.orDie) + } + + function stageSnapshotChunks( + input: SerializedSnapshot, + row: SerializedSnapshotRow, + chunks: readonly SerializedSnapshotChunk[], + ) { + return db.transaction(() => Effect.gen(function* () { + if (chunks.length > SNAPSHOT_TRANSFER_CHUNKS) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot chunk page exceeds ${SNAPSHOT_TRANSFER_CHUNKS}` })) + if (row.snapshotID !== input.snapshotID || chunks.some((chunk) => chunk.rowHash !== row.rowHash)) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: "Snapshot chunk identity mismatch" }), + ) + yield* Effect.forEach( + chunks, + (chunk) => Effect.gen(function* () { + if (chunk.data.length > SNAPSHOT_CHUNK_BYTES || Hash.sha256(chunk.data) !== chunk.chunkHash) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: "Snapshot chunk hash mismatch" }), + ) + const inserted = yield* db + .insert(EventSnapshotChunkTable) + .values({ + row_hash: chunk.rowHash, + chunk_index: chunk.chunkIndex, + data: chunk.data, + chunk_hash: chunk.chunkHash, + }) + .onConflictDoNothing() + .returning().get() + .pipe(Effect.orDie) + if (inserted) return + const existing = yield* db.select().from(EventSnapshotChunkTable).where(and( + eq(EventSnapshotChunkTable.row_hash, chunk.rowHash), + eq(EventSnapshotChunkTable.chunk_index, chunk.chunkIndex), + )).get().pipe(Effect.orDie) + if (!existing || existing.chunk_hash !== chunk.chunkHash || !Buffer.from(existing.data).equals(chunk.data)) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot chunk ${chunk.rowHash}:${chunk.chunkIndex} conflicts with durable CAS` })) + }), + { discard: true }, + ) + }), { behavior: "immediate" }).pipe(Effect.orDie) + } + function compact(input: { readonly aggregateID: string readonly throughSeq: Cursor readonly limit?: number readonly now?: number }) { - return Effect.die( - new InvalidSyncEventError({ - type: "snapshot", - message: `Event compaction is disabled until canonical projection snapshots can be imported for ${input.aggregateID}`, - }), - ) - /* c8 ignore start - retained design is unreachable until a projection codec is registered */ return db .transaction( () => Effect.gen(function* () { const sequence = yield* db - .select({ floor: EventSequenceTable.retention_floor_seq }) + .select({ floor: EventSequenceTable.retention_floor_seq, snapshotID: EventSequenceTable.snapshot_id }) .from(EventSequenceTable) .where(eq(EventSequenceTable.aggregate_id, input.aggregateID)) .get() .pipe(Effect.orDie) + const syncAuthority = yield* db.select({ complete: EventSyncSequenceTable.backfill_complete }) + .from(EventSyncSequenceTable).where(eq(EventSyncSequenceTable.id, 1)).get().pipe(Effect.orDie) + if (!syncAuthority?.complete) return yield* Effect.die(new MaintenanceRequiredError({ + operation: "event.compact", reason: "event_sync_backfill_required", + message: "Event sync index backfill must complete before compaction", + })) if (sequence?.floor === undefined || sequence.floor === null || input.throughSeq > sequence.floor) return yield* Effect.die( new InvalidSyncEventError({ @@ -1048,51 +1640,70 @@ export const layerWith = (layerOptions?: LayerOptions) => message: `Compaction exceeds the durable retention floor for ${input.aggregateID}`, }), ) - const rows = yield* db - .select() + const active = sequence.snapshotID ? yield* db.select().from(EventSnapshotTable) + .where(and(eq(EventSnapshotTable.snapshot_id, sequence.snapshotID), eq(EventSnapshotTable.aggregate_id, input.aggregateID))) + .get().pipe(Effect.orDie) : undefined + const codec = active ? snapshotCodecs.get(`${active.codec}@${active.schema_version}`) : undefined + if (!active || !codec || active.through_seq < input.throughSeq) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Compaction requires an active importable snapshot for ${input.aggregateID}` })) + const receipt = yield* db.select().from(EventCompactionReceiptTable) + .where(eq(EventCompactionReceiptTable.aggregate_id, input.aggregateID)).get().pipe(Effect.orDie) + if (receipt && (receipt.snapshot_id !== active.snapshot_id || receipt.through_seq !== input.throughSeq || + receipt.codec !== active.codec || receipt.schema_version !== active.schema_version)) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Compaction receipt conflicts with active snapshot ${active.snapshot_id}` })) + const cursor = receipt?.cursor_seq ?? -1 + const candidates = yield* db + .select({ + row: EventTable, + canonicalData: sql | null>`COALESCE(${FilePartArtifactBindingTable.canonical_data}, ${EventArtifactTable.canonical_data})`, + }) .from(EventTable) - .where(and(eq(EventTable.aggregate_id, input.aggregateID), lte(EventTable.seq, input.throughSeq))) + .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) + .leftJoin(FilePartArtifactBindingTable, eq(FilePartArtifactBindingTable.event_id, EventTable.id)) + .where(and(eq(EventTable.aggregate_id, input.aggregateID), gt(EventTable.seq, cursor), lte(EventTable.seq, input.throughSeq))) .orderBy(asc(EventTable.seq)) - .limit(Math.min(Math.max(input.limit ?? 100, 1), 100)) + .limit(Math.min(Math.max(input.limit ?? 100, 1), 100) + 1) .all() .pipe(Effect.orDie) - yield* Effect.forEach( - rows, - (row) => - db - .insert(EventDedupeTable) - .values({ - aggregate_id: row.aggregate_id, - seq: row.seq, - event_id: row.id, - type: row.type, - data_hash: hashJson(row.data), - compacted_at: input.now ?? Date.now(), - }) - .onConflictDoNothing() - .run() - .pipe(Effect.orDie), - { discard: true }, - ) - if (rows.length > 0) - yield* db - .delete(EventTable) - .where(inArray(EventTable.id, rows.map((row) => row.id))) - .run() - .pipe(Effect.orDie) - const remaining = yield* db - .select({ id: EventTable.id }) - .from(EventTable) - .where(and(eq(EventTable.aggregate_id, input.aggregateID), lte(EventTable.seq, input.throughSeq))) - .limit(1) - .get() + const rows = candidates.slice(0, Math.min(Math.max(input.limit ?? 100, 1), 100)) + const compactable = rows.filter((item) => codec.rebuildEventTypes.has(item.row.type)) + yield* Effect.forEach(compactable, (item) => Effect.gen(function* () { + const dataHash = hashJson(item.canonicalData ?? item.row.data) + const existing = yield* db.select().from(EventDedupeTable).where(or( + and(eq(EventDedupeTable.aggregate_id, item.row.aggregate_id), eq(EventDedupeTable.seq, item.row.seq)), + eq(EventDedupeTable.event_id, item.row.id), + )).get().pipe(Effect.orDie) + if (existing && (existing.aggregate_id !== item.row.aggregate_id || existing.seq !== item.row.seq || + existing.event_id !== item.row.id || existing.type !== item.row.type || existing.data_hash !== dataHash)) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Compaction dedupe conflicts at ${item.row.aggregate_id}:${item.row.seq}` })) + if (!existing) yield* db.insert(EventDedupeTable).values({ + aggregate_id: item.row.aggregate_id, seq: item.row.seq, event_id: item.row.id, type: item.row.type, + data_hash: dataHash, source_data: item.row.data, compacted_at: input.now ?? Date.now(), + }).run().pipe(Effect.orDie) + }), { discard: true }) + if (compactable.length > 0) yield* db.delete(EventTable) + .where(inArray(EventTable.id, compactable.map((item) => item.row.id))).run().pipe(Effect.orDie) + if (compactable.length > 0) yield* db.update(EventDedupeTable) + .set({ source_data: null }) + .where(inArray(EventDedupeTable.event_id, compactable.map((item) => item.row.id))) + .run() .pipe(Effect.orDie) - return { deleted: rows.length, complete: !remaining } + const nextCursor = rows.at(-1)?.row.seq ?? cursor + const complete = rows.length === 0 || (nextCursor >= input.throughSeq && candidates.length <= rows.length) + yield* db.insert(EventCompactionReceiptTable).values({ aggregate_id: input.aggregateID, + snapshot_id: active.snapshot_id, through_seq: input.throughSeq, codec: active.codec, + schema_version: active.schema_version, cursor_seq: nextCursor, + deleted_count: (receipt?.deleted_count ?? 0) + compactable.length, + state: complete ? "complete" : "running", updated_at: input.now ?? Date.now(), + }).onConflictDoUpdate({ target: EventCompactionReceiptTable.aggregate_id, set: { + cursor_seq: nextCursor, deleted_count: (receipt?.deleted_count ?? 0) + compactable.length, + state: complete ? "complete" : "running", updated_at: input.now ?? Date.now(), + }}).run().pipe(Effect.orDie) + return { deleted: compactable.length, complete } }), { behavior: "immediate" }, ) .pipe(Effect.orDie) - /* c8 ignore stop */ } function canonicalizeLegacyArtifacts(input?: { readonly afterID?: ID; readonly limit?: number; readonly now?: number }) { @@ -1315,9 +1926,10 @@ export const layerWith = (layerOptions?: LayerOptions) => ${EventTable.id} AS id, ${EventTable.seq} AS seq, ${EventTable.type} AS type, - length(CAST(COALESCE(${EventArtifactTable.canonical_data}, ${EventTable.data}) AS BLOB)) AS bytes + length(CAST(COALESCE(${FilePartArtifactBindingTable.canonical_data}, ${EventArtifactTable.canonical_data}, ${EventTable.data}) AS BLOB)) AS bytes FROM ${EventTable} LEFT JOIN ${EventArtifactTable} ON ${EventArtifactTable.event_id} = ${EventTable.id} + LEFT JOIN ${FilePartArtifactBindingTable} ON ${FilePartArtifactBindingTable.event_id} = ${EventTable.id} WHERE ${EventTable.aggregate_id} = ${aggregateID} AND ${EventTable.seq} > ${after} ORDER BY ${EventTable.seq} ASC @@ -1357,10 +1969,11 @@ export const layerWith = (layerOptions?: LayerOptions) => const rows = yield* db .select({ id: EventTable.id, - data: sql`COALESCE(${EventArtifactTable.canonical_data}, ${EventTable.data})`, + data: sql`COALESCE(${FilePartArtifactBindingTable.canonical_data}, ${EventArtifactTable.canonical_data}, ${EventTable.data})`, }) .from(EventTable) .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) + .leftJoin(FilePartArtifactBindingTable, eq(FilePartArtifactBindingTable.event_id, EventTable.id)) .where(inArray(EventTable.id, metadata.map((event) => event.id))) .all() .pipe(Effect.orDie) @@ -1464,6 +2077,107 @@ export const layerWith = (layerOptions?: LayerOptions) => projectors.set(definition.type, list) }) + const registerSnapshotCodec = (codec: SnapshotCodec) => + Effect.sync(() => { + const key = `${codec.codec}@${codec.schemaVersion}` + if (snapshotCodecs.has(key)) + throw new InvalidSyncEventError({ type: "snapshot", message: `Snapshot codec ${key} is already registered` }) + snapshotCodecs.set(key, codec) + }) + + function backfillSyncIndex(input?: { readonly limit?: number; readonly now?: number }) { + const limit = Math.min(Math.max(input?.limit ?? 500, 1), 5000) + return db + .transaction( + () => + Effect.gen(function* () { + const authority = yield* db + .select() + .from(EventSyncBackfillTable) + .where(eq(EventSyncBackfillTable.id, 1)) + .get() + .pipe(Effect.orDie) + const syncAuthority = yield* db + .select({ complete: EventSyncSequenceTable.backfill_complete }) + .from(EventSyncSequenceTable) + .where(eq(EventSyncSequenceTable.id, 1)) + .get() + .pipe(Effect.orDie) + if (!authority || !syncAuthority) + return yield* Effect.die( + new MaintenanceRequiredError({ + operation: "event.sync.backfill", + reason: "event_sync_backfill_authority_missing", + message: "Event sync backfill authority is unavailable", + }), + ) + if (authority.state === "complete" && syncAuthority.complete) return { processed: 0, complete: true } + const rows = yield* db + .all<{ + rowid: number + id: string + aggregate_id: string + seq: number + sync_seq: number | null + }>(sql` + SELECT rowid, id, aggregate_id, seq, sync_seq + FROM event + WHERE rowid > ${authority.cursor_rowid} AND rowid <= ${authority.high_water_rowid} + ORDER BY rowid ASC + LIMIT ${limit} + `) + .pipe(Effect.orDie) + yield* Effect.forEach( + rows, + (row) => Effect.gen(function* () { + const syncSeq = row.sync_seq ?? row.rowid + const existing = yield* db.select().from(EventSyncIndexTable).where(or( + eq(EventSyncIndexTable.sync_seq, syncSeq), + eq(EventSyncIndexTable.event_id, ID.make(row.id)), + and(eq(EventSyncIndexTable.aggregate_id, row.aggregate_id), eq(EventSyncIndexTable.seq, row.seq)), + )).get().pipe(Effect.orDie) + if (existing) { + if (existing.sync_seq === syncSeq && existing.event_id === row.id && + existing.aggregate_id === row.aggregate_id && existing.seq === row.seq) return + return yield* Effect.die(new InvalidSyncEventError({ + type: "event.sync.backfill", + message: `Sync index identity conflict at legacy event rowid ${row.rowid}`, + })) + } + yield* db.insert(EventSyncIndexTable).values({ sync_seq: syncSeq, event_id: ID.make(row.id), + aggregate_id: row.aggregate_id, seq: row.seq }).run().pipe(Effect.orDie) + }), + { discard: true }, + ) + const cursor = rows.at(-1)?.rowid ?? authority.cursor_rowid + const complete = cursor >= authority.high_water_rowid + const now = input?.now ?? Date.now() + yield* db + .update(EventSyncBackfillTable) + .set({ + state: complete ? "complete" : "pending", + cursor_rowid: cursor, + processed_count: sql`${EventSyncBackfillTable.processed_count} + ${rows.length}`, + updated_at: now, + ...(complete ? { completed_at: now } : {}), + }) + .where(eq(EventSyncBackfillTable.id, 1)) + .run() + .pipe(Effect.orDie) + if (complete) + yield* db + .update(EventSyncSequenceTable) + .set({ backfill_complete: true }) + .where(eq(EventSyncSequenceTable.id, 1)) + .run() + .pipe(Effect.orDie) + return { processed: rows.length, complete } + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie) + } + return Service.of({ publish, subscribe, @@ -1473,13 +2187,23 @@ export const layerWith = (layerOptions?: LayerOptions) => listen, beforeCommit, project, + registerSnapshotCodec, replay, replayAll, snapshot, + prepareCheckpoint, + stageCheckpoint, + finalizeCheckpoint, + discardCheckpoint, checkpoint, importSnapshot, compact, canonicalizeLegacyArtifacts, + snapshotRows, + snapshotChunks, + stageSnapshotRows, + stageSnapshotChunks, + backfillSyncIndex, remove, claim, }) diff --git a/packages/core/src/event/sql.ts b/packages/core/src/event/sql.ts index 35ed94722..0bda2c21f 100644 --- a/packages/core/src/event/sql.ts +++ b/packages/core/src/event/sql.ts @@ -7,6 +7,7 @@ export const EventSequenceTable = sqliteTable("event_sequence", { owner_id: text(), retention_floor_seq: integer(), snapshot_id: text(), + write_fence_transfer_id: text(), }) export const EventTable = sqliteTable( @@ -24,7 +25,6 @@ export const EventTable = sqliteTable( (table) => [ uniqueIndex("event_aggregate_seq_idx").on(table.aggregate_id, table.seq), index("event_aggregate_type_seq_idx").on(table.aggregate_id, table.type, table.seq), - uniqueIndex("event_sync_seq_idx").on(table.sync_seq), ], ) @@ -51,13 +51,84 @@ export const EventSnapshotTable = sqliteTable( ], ) +export const EventSnapshotRowTable = sqliteTable( + "event_snapshot_row", + { + snapshot_id: text().notNull(), + row_index: integer().notNull(), + table_name: text().notNull(), + row_key: text().notNull(), + row_hash: text().notNull(), + row_bytes: integer().notNull(), + chunk_count: integer().notNull(), + chain_hash: text().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.snapshot_id, table.row_index] }), + uniqueIndex("event_snapshot_row_identity_idx").on(table.snapshot_id, table.table_name, table.row_key), + ], +) + +export const EventSnapshotChunkTable = sqliteTable( + "event_snapshot_chunk", + { + row_hash: text().notNull(), + chunk_index: integer().notNull(), + data: blob({ mode: "buffer" }).notNull(), + chunk_hash: text().notNull(), + }, + (table) => [primaryKey({ columns: [table.row_hash, table.chunk_index] })], +) + +export const EventSnapshotAttemptTable = sqliteTable("event_snapshot_attempt", { + snapshot_id: text().primaryKey(), + aggregate_id: text().notNull(), + through_seq: integer().notNull(), + expected_latest: integer().notNull(), + owner_id: text(), + codec: text().notNull(), + schema_version: integer().notNull(), + projection_revision: text().notNull(), + cursor: text(), + row_count: integer().notNull(), + encoded_bytes: integer().notNull(), + content_hash: text().notNull(), + tables: text({ mode: "json" }).$type>().notNull(), + state: text().$type<"prepared" | "staged" | "complete">().notNull(), + created_at: integer().notNull(), + updated_at: integer().notNull(), +}) + export const EventSyncSequenceTable = sqliteTable("event_sync_sequence", { id: integer().primaryKey(), seq: integer().notNull(), generation: text().notNull(), cursor_secret: text().notNull(), + backfill_complete: integer({ mode: "boolean" }).notNull(), }) +export const EventSyncBackfillTable = sqliteTable("event_sync_backfill", { + id: integer().primaryKey(), + state: text().$type<"pending" | "complete">().notNull(), + cursor_rowid: integer().notNull(), + high_water_rowid: integer().notNull(), + processed_count: integer().notNull(), + started_at: integer().notNull(), + updated_at: integer().notNull(), + completed_at: integer(), +}) + +export const EventSyncIndexTable = sqliteTable( + "event_sync_index", + { + sync_seq: integer().primaryKey(), + event_id: text().$type().notNull().unique(), + aggregate_id: text().notNull(), + seq: integer().notNull(), + }, + (table) => [index("event_sync_index_aggregate_seq_idx").on(table.aggregate_id, table.seq)], +) + export const WorkspaceSyncCursorTable = sqliteTable( "workspace_sync_cursor", { @@ -79,6 +150,7 @@ export const EventDedupeTable = sqliteTable( event_id: text().$type().notNull(), type: text().notNull(), data_hash: text().notNull(), + source_data: text({ mode: "json" }).$type>(), compacted_at: integer().notNull(), }, (table) => [ @@ -87,6 +159,20 @@ export const EventDedupeTable = sqliteTable( ], ) +export const EventCompactionReceiptTable = sqliteTable("event_compaction_receipt", { + aggregate_id: text() + .primaryKey() + .references(() => EventSequenceTable.aggregate_id, { onDelete: "cascade" }), + snapshot_id: text().notNull(), + through_seq: integer().notNull(), + codec: text().notNull(), + schema_version: integer().notNull(), + cursor_seq: integer().notNull(), + deleted_count: integer().notNull(), + state: text().$type<"running" | "complete">().notNull(), + updated_at: integer().notNull(), +}) + export const EventArtifactTable = sqliteTable( "event_artifact", { diff --git a/packages/core/src/file-part-artifact.sql.ts b/packages/core/src/file-part-artifact.sql.ts new file mode 100644 index 000000000..4baef7992 --- /dev/null +++ b/packages/core/src/file-part-artifact.sql.ts @@ -0,0 +1,67 @@ +import { blob, index, integer, primaryKey, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import type { EventV2 } from "./event" +import { EventSequenceTable } from "./event/sql" + +export const FilePartArtifactTable = sqliteTable("file_part_artifact", { + artifact_id: text().primaryKey(), + body_hash: text().notNull().unique(), + body_bytes: integer().notNull(), + chunk_bytes: integer().notNull(), + chunk_count: integer().notNull(), + codec_version: integer().notNull(), + complete: integer({ mode: "boolean" }).notNull(), + created_at: integer().notNull(), +}) + +export const FilePartArtifactChunkTable = sqliteTable( + "file_part_artifact_chunk", + { + artifact_id: text() + .notNull() + .references(() => FilePartArtifactTable.artifact_id, { onDelete: "cascade" }), + chunk_index: integer().notNull(), + data: blob({ mode: "buffer" }).notNull(), + chunk_hash: text().notNull(), + }, + (table) => [primaryKey({ columns: [table.artifact_id, table.chunk_index] })], +) + +export const FilePartArtifactBindingTable = sqliteTable( + "file_part_artifact_binding", + { + event_id: text().$type().primaryKey(), + aggregate_id: text() + .notNull() + .references(() => EventSequenceTable.aggregate_id, { onDelete: "cascade" }), + seq: integer().notNull(), + part_id: text().notNull(), + artifact_id: text() + .notNull() + .references(() => FilePartArtifactTable.artifact_id, { onDelete: "restrict" }), + original_data_hash: text().notNull(), + canonical_data_hash: text().notNull(), + canonical_data: text({ mode: "json" }).$type>().notNull(), + created_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("file_part_artifact_binding_aggregate_seq_idx").on(table.aggregate_id, table.seq), + index("file_part_artifact_binding_part_idx").on(table.aggregate_id, table.part_id, table.seq), + ], +) + +export const FilePartArtifactImportTable = sqliteTable( + "file_part_artifact_import", + { + event_id: text().$type().primaryKey(), + aggregate_id: text().notNull(), + seq: integer().notNull(), + artifact_id: text() + .notNull() + .references(() => FilePartArtifactTable.artifact_id, { onDelete: "cascade" }), + original_data_hash: text().notNull(), + canonical_data_hash: text().notNull(), + canonical_data: text({ mode: "json" }).$type>().notNull(), + created_at: integer().notNull(), + }, + (table) => [uniqueIndex("file_part_artifact_import_aggregate_seq_idx").on(table.aggregate_id, table.seq)], +) diff --git a/packages/core/src/file-part-artifact.ts b/packages/core/src/file-part-artifact.ts new file mode 100644 index 000000000..7db78e111 --- /dev/null +++ b/packages/core/src/file-part-artifact.ts @@ -0,0 +1,839 @@ +export * as FilePartArtifact from "./file-part-artifact" + +import { createHash } from "node:crypto" +import { isDeepStrictEqual } from "node:util" +import { and, asc, desc, eq, gt, isNull, sql } from "drizzle-orm" +import { Effect, Schema } from "effect" +import { Database } from "./database/database" +import type { EventV2 } from "./event" +import { + FilePartArtifactBindingTable, + FilePartArtifactChunkTable, + FilePartArtifactImportTable, + FilePartArtifactTable, +} from "./file-part-artifact.sql" +import { NonNegativeInt } from "./schema" +import { Hash } from "./util/hash" +import { EventTable } from "./event/sql" +import { PartTable } from "./session/sql" + +export const CHUNK_BYTES = 262_144 as const +export const MAX_BYTES = 32 * 1024 * 1024 +export const MAX_CHUNKS = MAX_BYTES / CHUNK_BYTES +export const CODEC = "file-part.v1" as const + +const Digest = Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)) +const EventID = Schema.String.check(Schema.isStartsWith("evt_")) +export const ID = Schema.String.check(Schema.isPattern(/^fpart_[a-f0-9]{64}$/)) +export type ID = typeof ID.Type + +export const Descriptor = Schema.Struct({ + codec: Schema.Literal(CODEC), + id: ID, + hash: Digest, + bytes: NonNegativeInt.check(Schema.isLessThanOrEqualTo(MAX_BYTES)), + chunkBytes: Schema.Literal(CHUNK_BYTES), + chunks: NonNegativeInt.check(Schema.isGreaterThanOrEqualTo(1), Schema.isLessThanOrEqualTo(MAX_CHUNKS)), +}) + .annotate({ identifier: "FilePartArtifact.Descriptor" }) +export type Descriptor = Schema.Schema.Type + +export const Metadata = Schema.Struct({ + eventID: EventID, + aggregateID: Schema.String, + seq: NonNegativeInt, + originalDataHash: Digest, + canonicalDataHash: Digest, + canonicalData: Schema.Record(Schema.String, Schema.Unknown), + descriptor: Descriptor, + chunkHashes: Schema.Array(Digest).check(Schema.isMaxLength(MAX_CHUNKS)), +}) + .annotate({ identifier: "FilePartArtifact.Metadata" }) +export type Metadata = Schema.Schema.Type + +export class IntegrityError extends Schema.TaggedErrorClass()("FilePartArtifact.IntegrityError", { + artifactID: Schema.String, + message: Schema.String, +}) {} + +export type Prepared = { + readonly descriptor: Descriptor + readonly chunks: readonly Buffer[] +} + +type Binding = { + readonly descriptor: Descriptor + readonly aggregateID: string + readonly partID: string +} + +const descriptorFromData = (type: string, data: Record): Binding | undefined => { + if (type !== "message.part.updated") return undefined + const part = data.part + if (!part || typeof part !== "object" || Array.isArray(part)) return undefined + const value = part as Record + if (value.type !== "file" || !Schema.is(Descriptor)(value.artifact)) return undefined + if ( + value.url !== `artifact:${value.artifact.id}` || + data.sessionID !== value.sessionID || + typeof value.id !== "string" || + !value.id.startsWith("prt") + ) + return undefined + return { descriptor: value.artifact, aggregateID: String(data.sessionID), partID: value.id } +} + +const fail = (artifactID: string, message: string) => + Effect.die(new IntegrityError({ artifactID, message })) + +export function prepare(type: string, data: Record, encodedBytes: number, limitBytes: number) { + if (type !== "message.part.updated") return { data, artifacts: [] as Prepared[] } + const part = data.part + if (!part || typeof part !== "object" || Array.isArray(part)) return { data, artifacts: [] as Prepared[] } + const value = part as Record + if (value.type !== "file" || typeof value.url !== "string") return { data, artifacts: [] as Prepared[] } + const match = /^data:([^;,]*);base64,([A-Za-z0-9+/]*={0,2})$/.exec(value.url) + if (!match) return { data, artifacts: [] as Prepared[] } + const body = Buffer.from(match[2]!, "base64") + if (body.toString("base64").replace(/=+$/, "") !== match[2]!.replace(/=+$/, "")) + return { data, artifacts: [] as Prepared[] } + if (body.byteLength > MAX_BYTES) + throw new IntegrityError({ + artifactID: "pending", + message: `File part body is ${body.byteLength} bytes; limit is ${MAX_BYTES} bytes`, + }) + if (encodedBytes <= limitBytes && body.byteLength <= CHUNK_BYTES) return { data, artifacts: [] as Prepared[] } + const hash = Hash.sha256(body) + const descriptor = Descriptor.make({ + codec: CODEC, + id: ID.make(`fpart_${hash}`), + hash, + bytes: body.byteLength, + chunkBytes: CHUNK_BYTES, + chunks: Math.max(1, Math.ceil(body.byteLength / CHUNK_BYTES)), + }) + return { + data: { + ...data, + part: { ...value, url: `artifact:${descriptor.id}`, artifact: descriptor }, + }, + artifacts: [ + { + descriptor, + chunks: Array.from({ length: descriptor.chunks }, (_, index) => + Buffer.from(body.subarray(index * CHUNK_BYTES, (index + 1) * CHUNK_BYTES)), + ), + }, + ], + } +} + +export function put(db: Database.Interface["db"], prepared: Prepared, now = Date.now()) { + return Effect.gen(function* () { + yield* validate(prepared.descriptor, prepared.chunks) + const existing = yield* db + .select() + .from(FilePartArtifactTable) + .where(eq(FilePartArtifactTable.artifact_id, prepared.descriptor.id)) + .get() + .pipe(Effect.orDie) + if (existing) { + if ( + existing.body_hash !== prepared.descriptor.hash || + existing.body_bytes !== prepared.descriptor.bytes || + existing.chunk_bytes !== prepared.descriptor.chunkBytes || + existing.chunk_count !== prepared.descriptor.chunks || + existing.codec_version !== 1 || + !existing.complete + ) + return yield* fail(prepared.descriptor.id, "Artifact identity is bound to different metadata") + return + } + yield* db + .insert(FilePartArtifactTable) + .values({ + artifact_id: prepared.descriptor.id, + body_hash: prepared.descriptor.hash, + body_bytes: prepared.descriptor.bytes, + chunk_bytes: prepared.descriptor.chunkBytes, + chunk_count: prepared.descriptor.chunks, + codec_version: 1, + complete: true, + created_at: now, + }) + .run() + .pipe(Effect.orDie) + yield* Effect.forEach( + prepared.chunks, + (chunk, index) => + db + .insert(FilePartArtifactChunkTable) + .values({ artifact_id: prepared.descriptor.id, chunk_index: index, data: chunk, chunk_hash: Hash.sha256(chunk) }) + .run() + .pipe(Effect.orDie), + { discard: true }, + ) + }) +} + +export function requireAvailable(db: Database.Interface["db"], descriptor: Descriptor) { + return Effect.gen(function* () { + const row = yield* db + .select({ + hash: FilePartArtifactTable.body_hash, + bytes: FilePartArtifactTable.body_bytes, + chunkBytes: FilePartArtifactTable.chunk_bytes, + chunks: FilePartArtifactTable.chunk_count, + complete: FilePartArtifactTable.complete, + }) + .from(FilePartArtifactTable) + .where(eq(FilePartArtifactTable.artifact_id, descriptor.id)) + .get() + .pipe(Effect.orDie) + if ( + !row || + row.hash !== descriptor.hash || + row.bytes !== descriptor.bytes || + row.chunkBytes !== descriptor.chunkBytes || + row.chunks !== descriptor.chunks || + !row.complete + ) + return yield* fail(descriptor.id, "Artifact body is missing or does not match its descriptor") + }) +} + +export function bind( + db: Database.Interface["db"], + input: { + readonly eventID: EventV2.ID + readonly aggregateID: string + readonly seq: number + readonly type: string + readonly data: Record + readonly originalData?: Record + readonly requireImport?: boolean + readonly now?: number + }, +) { + return Effect.gen(function* () { + const binding = descriptorFromData(input.type, input.data) + if (!binding) return + if (binding.aggregateID !== input.aggregateID) + return yield* fail(binding.descriptor.id, "Artifact aggregate does not match its event") + yield* requireAvailable(db, binding.descriptor) + const canonicalDataHash = Hash.sha256(JSON.stringify(input.data)) + const imported = input.requireImport + ? yield* db + .select() + .from(FilePartArtifactImportTable) + .where(eq(FilePartArtifactImportTable.event_id, input.eventID)) + .get() + .pipe(Effect.orDie) + : undefined + if ( + input.requireImport && + (!imported || + imported.aggregate_id !== input.aggregateID || + imported.seq !== input.seq || + imported.artifact_id !== binding.descriptor.id || + imported.canonical_data_hash !== canonicalDataHash || + !isDeepStrictEqual(imported.canonical_data, input.data)) + ) + return yield* fail(binding.descriptor.id, "Replayed artifact has no exact staged import receipt") + const originalDataHash = imported?.original_data_hash ?? Hash.sha256(JSON.stringify(input.originalData ?? input.data)) + const existing = yield* db + .select() + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, input.eventID)) + .get() + .pipe(Effect.orDie) + if (existing) { + if ( + existing.aggregate_id !== input.aggregateID || + existing.seq !== input.seq || + existing.part_id !== binding.partID || + existing.artifact_id !== binding.descriptor.id || + existing.original_data_hash !== originalDataHash || + existing.canonical_data_hash !== canonicalDataHash || + !isDeepStrictEqual(existing.canonical_data, input.data) + ) + return yield* fail(binding.descriptor.id, "Event identity is bound to a different artifact") + return + } + yield* db + .insert(FilePartArtifactBindingTable) + .values({ + event_id: input.eventID, + aggregate_id: input.aggregateID, + seq: input.seq, + part_id: binding.partID, + artifact_id: binding.descriptor.id, + original_data_hash: originalDataHash, + canonical_data_hash: canonicalDataHash, + canonical_data: input.data, + created_at: input.now ?? Date.now(), + }) + .run() + .pipe(Effect.orDie) + if (input.requireImport) + yield* db + .delete(FilePartArtifactImportTable) + .where(eq(FilePartArtifactImportTable.event_id, input.eventID)) + .run() + .pipe(Effect.orDie) + }) +} + +export function metadata( + db: Database.Interface["db"], + input: { readonly eventID: EventV2.ID; readonly aggregateID: string; readonly seq: number; readonly artifactID: ID }, +) { + return Effect.gen(function* () { + const row = yield* db + .select({ + originalDataHash: FilePartArtifactBindingTable.original_data_hash, + canonicalDataHash: FilePartArtifactBindingTable.canonical_data_hash, + canonicalData: FilePartArtifactBindingTable.canonical_data, + hash: FilePartArtifactTable.body_hash, + bytes: FilePartArtifactTable.body_bytes, + chunkBytes: FilePartArtifactTable.chunk_bytes, + chunks: FilePartArtifactTable.chunk_count, + complete: FilePartArtifactTable.complete, + }) + .from(FilePartArtifactBindingTable) + .innerJoin( + FilePartArtifactTable, + eq(FilePartArtifactTable.artifact_id, FilePartArtifactBindingTable.artifact_id), + ) + .where( + and( + eq(FilePartArtifactBindingTable.event_id, input.eventID), + eq(FilePartArtifactBindingTable.aggregate_id, input.aggregateID), + eq(FilePartArtifactBindingTable.seq, input.seq), + eq(FilePartArtifactBindingTable.artifact_id, input.artifactID), + ), + ) + .get() + .pipe(Effect.orDie) + if (!row || !row.complete) + return yield* fail(input.artifactID, "Artifact metadata is not bound to a complete requested event") + const chunks = yield* db + .select({ index: FilePartArtifactChunkTable.chunk_index, hash: FilePartArtifactChunkTable.chunk_hash }) + .from(FilePartArtifactChunkTable) + .where(eq(FilePartArtifactChunkTable.artifact_id, input.artifactID)) + .orderBy(asc(FilePartArtifactChunkTable.chunk_index)) + .all() + .pipe(Effect.orDie) + const descriptor = Descriptor.make({ + codec: CODEC, + id: input.artifactID, + hash: row.hash, + bytes: row.bytes, + chunkBytes: CHUNK_BYTES, + chunks: row.chunks, + }) + if (chunks.length !== descriptor.chunks || chunks.some((chunk, index) => chunk.index !== index)) + return yield* fail(input.artifactID, "Artifact chunk metadata is incomplete") + return Metadata.make({ + eventID: input.eventID, + aggregateID: input.aggregateID, + seq: input.seq, + originalDataHash: row.originalDataHash, + canonicalDataHash: row.canonicalDataHash, + canonicalData: row.canonicalData, + descriptor, + chunkHashes: chunks.map((chunk) => chunk.hash), + }) + }) +} + +export function chunk( + db: Database.Interface["db"], + input: { readonly artifactID: ID; readonly index: number; readonly expectedHash: string }, +) { + return Effect.gen(function* () { + const row = yield* db + .select({ data: FilePartArtifactChunkTable.data, hash: FilePartArtifactChunkTable.chunk_hash }) + .from(FilePartArtifactChunkTable) + .where( + and( + eq(FilePartArtifactChunkTable.artifact_id, input.artifactID), + eq(FilePartArtifactChunkTable.chunk_index, input.index), + ), + ) + .get() + .pipe(Effect.orDie) + if (!row || row.hash !== input.expectedHash || Hash.sha256(row.data) !== row.hash) + return yield* fail(input.artifactID, `Artifact chunk ${input.index} is missing or corrupt`) + return row.data + }) +} + +export function has(descriptor: Descriptor) { + return Database.Service.use(({ db }) => + db + .select({ id: FilePartArtifactTable.artifact_id }) + .from(FilePartArtifactTable) + .where( + and( + eq(FilePartArtifactTable.artifact_id, descriptor.id), + eq(FilePartArtifactTable.body_hash, descriptor.hash), + eq(FilePartArtifactTable.body_bytes, descriptor.bytes), + eq(FilePartArtifactTable.chunk_count, descriptor.chunks), + eq(FilePartArtifactTable.complete, true), + ), + ) + .get() + .pipe(Effect.orDie, Effect.map(Boolean)), + ) +} + +export function importChunks(input: { readonly descriptor: Descriptor; readonly chunks: readonly Uint8Array[] }) { + return Database.Service.use(({ db }) => + db.transaction( + () => put(db, { descriptor: input.descriptor, chunks: input.chunks.map((chunk) => Buffer.from(chunk)) }), + { behavior: "immediate" }, + ).pipe(Effect.orDie), + ) +} + +export function importChunk(input: { + readonly metadata: Metadata + readonly index: number + readonly hash: string + readonly data: Uint8Array +}) { + return Database.Service.use(({ db }) => + db + .transaction( + () => + Effect.gen(function* () { + const descriptor = input.metadata.descriptor + if ( + input.index < 0 || + input.index >= descriptor.chunks || + input.metadata.chunkHashes[input.index] !== input.hash || + Hash.sha256(Buffer.from(input.data)) !== input.hash + ) + return yield* fail(descriptor.id, `Artifact import chunk ${input.index} does not match metadata`) + const existing = yield* db + .select() + .from(FilePartArtifactTable) + .where(eq(FilePartArtifactTable.artifact_id, descriptor.id)) + .get() + .pipe(Effect.orDie) + if ( + existing && + (existing.body_hash !== descriptor.hash || + existing.body_bytes !== descriptor.bytes || + existing.chunk_bytes !== descriptor.chunkBytes || + existing.chunk_count !== descriptor.chunks || + existing.codec_version !== 1) + ) + return yield* fail(descriptor.id, "Artifact identity is bound to different import metadata") + if (!existing) + yield* db + .insert(FilePartArtifactTable) + .values({ + artifact_id: descriptor.id, + body_hash: descriptor.hash, + body_bytes: descriptor.bytes, + chunk_bytes: descriptor.chunkBytes, + chunk_count: descriptor.chunks, + codec_version: 1, + complete: false, + created_at: Date.now(), + }) + .run() + .pipe(Effect.orDie) + const storedChunk = yield* db + .select({ hash: FilePartArtifactChunkTable.chunk_hash, data: FilePartArtifactChunkTable.data }) + .from(FilePartArtifactChunkTable) + .where( + and( + eq(FilePartArtifactChunkTable.artifact_id, descriptor.id), + eq(FilePartArtifactChunkTable.chunk_index, input.index), + ), + ) + .get() + .pipe(Effect.orDie) + if (storedChunk && (storedChunk.hash !== input.hash || !storedChunk.data.equals(Buffer.from(input.data)))) + return yield* fail(descriptor.id, `Artifact import chunk ${input.index} conflicts with durable CAS`) + if (!storedChunk) + yield* db + .insert(FilePartArtifactChunkTable) + .values({ artifact_id: descriptor.id, chunk_index: input.index, data: Buffer.from(input.data), chunk_hash: input.hash }) + .run() + .pipe(Effect.orDie) + const chunks = yield* db + .select({ data: FilePartArtifactChunkTable.data }) + .from(FilePartArtifactChunkTable) + .where(eq(FilePartArtifactChunkTable.artifact_id, descriptor.id)) + .orderBy(asc(FilePartArtifactChunkTable.chunk_index)) + .all() + .pipe(Effect.orDie) + if (chunks.length !== descriptor.chunks) return false + yield* validate(descriptor, chunks.map((chunk) => chunk.data)) + yield* db + .update(FilePartArtifactTable) + .set({ complete: true }) + .where(eq(FilePartArtifactTable.artifact_id, descriptor.id)) + .run() + .pipe(Effect.orDie) + yield* stage(db, input.metadata) + return true + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie), + ) +} + +export function stageImport(metadata: Metadata) { + return Database.Service.use(({ db }) => + db.transaction(() => stage(db, metadata), { behavior: "immediate" }).pipe(Effect.orDie), + ) +} + +export function read(input: { readonly aggregateID: string; readonly descriptor: Descriptor }) { + return Database.Service.use(({ db }) => + Effect.gen(function* () { + const binding = yield* db + .select({ id: FilePartArtifactBindingTable.event_id }) + .from(FilePartArtifactBindingTable) + .where( + and( + eq(FilePartArtifactBindingTable.aggregate_id, input.aggregateID), + eq(FilePartArtifactBindingTable.artifact_id, input.descriptor.id), + ), + ) + .get() + .pipe(Effect.orDie) + if (!binding) return yield* fail(input.descriptor.id, "Artifact is not bound to the requested aggregate") + const chunks = yield* db + .select({ data: FilePartArtifactChunkTable.data, hash: FilePartArtifactChunkTable.chunk_hash }) + .from(FilePartArtifactChunkTable) + .where(eq(FilePartArtifactChunkTable.artifact_id, input.descriptor.id)) + .orderBy(asc(FilePartArtifactChunkTable.chunk_index)) + .all() + .pipe(Effect.orDie) + if (chunks.some((chunk) => Hash.sha256(chunk.data) !== chunk.hash)) + return yield* fail(input.descriptor.id, "Artifact chunk hash is corrupt") + yield* validate(input.descriptor, chunks.map((chunk) => chunk.data)) + return Buffer.concat(chunks.map((chunk) => chunk.data), input.descriptor.bytes) + }), + ) +} + +export function descriptor(data: Record) { + return descriptorFromData("message.part.updated", data)?.descriptor +} + +export function snapshotRef( + db: Database.Interface["db"], + input: { readonly aggregateID: string; readonly partID: string; readonly descriptor: Descriptor }, +) { + return Effect.gen(function* () { + const binding = yield* db + .select({ + eventID: FilePartArtifactBindingTable.event_id, + seq: FilePartArtifactBindingTable.seq, + }) + .from(FilePartArtifactBindingTable) + .where( + and( + eq(FilePartArtifactBindingTable.aggregate_id, input.aggregateID), + eq(FilePartArtifactBindingTable.part_id, input.partID), + eq(FilePartArtifactBindingTable.artifact_id, input.descriptor.id), + ), + ) + .orderBy(desc(FilePartArtifactBindingTable.seq)) + .limit(1) + .get() + .pipe(Effect.orDie) + if (!binding) return yield* fail(input.descriptor.id, "Snapshot part has no durable artifact binding") + const result = yield* metadata(db, { + eventID: binding.eventID, + aggregateID: input.aggregateID, + seq: binding.seq, + artifactID: input.descriptor.id, + }) + if (JSON.stringify(result.descriptor) !== JSON.stringify(input.descriptor)) + return yield* fail(input.descriptor.id, "Snapshot part descriptor diverges from its durable binding") + return result + }) +} + +export function bindSnapshotRef(input: { readonly metadata: Metadata; readonly partID: string }) { + return Database.Service.use(({ db }) => + db + .transaction( + () => + Effect.gen(function* () { + if (!input.partID.startsWith("prt")) + return yield* fail(input.metadata.descriptor.id, "Snapshot artifact part identity is invalid") + yield* stage(db, input.metadata) + const existing = yield* db + .select() + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, input.metadata.eventID as EventV2.ID)) + .get() + .pipe(Effect.orDie) + if (existing) { + if ( + existing.aggregate_id !== input.metadata.aggregateID || + existing.seq !== input.metadata.seq || + existing.part_id !== input.partID || + existing.artifact_id !== input.metadata.descriptor.id || + existing.original_data_hash !== input.metadata.originalDataHash || + existing.canonical_data_hash !== input.metadata.canonicalDataHash || + !isDeepStrictEqual(existing.canonical_data, input.metadata.canonicalData) + ) + return yield* fail(input.metadata.descriptor.id, "Snapshot artifact binding conflicts with durable CAS") + } else { + yield* db + .insert(FilePartArtifactBindingTable) + .values({ + event_id: input.metadata.eventID as EventV2.ID, + aggregate_id: input.metadata.aggregateID, + seq: input.metadata.seq, + part_id: input.partID, + artifact_id: input.metadata.descriptor.id, + original_data_hash: input.metadata.originalDataHash, + canonical_data_hash: input.metadata.canonicalDataHash, + canonical_data: input.metadata.canonicalData, + created_at: Date.now(), + }) + .run() + .pipe(Effect.orDie) + } + yield* db + .delete(FilePartArtifactImportTable) + .where(eq(FilePartArtifactImportTable.event_id, input.metadata.eventID as EventV2.ID)) + .run() + .pipe(Effect.orDie) + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie), + ) +} + +export function canonicalizeLegacy( + db: Database.Interface["db"], + input?: { readonly afterID?: EventV2.ID; readonly limit?: number; readonly now?: number }, +) { + const limit = Math.min(Math.max(input?.limit ?? 1, 1), 1) + return db + .transaction( + () => + Effect.gen(function* () { + const rows = yield* db + .select({ + id: EventTable.id, + aggregateID: EventTable.aggregate_id, + seq: EventTable.seq, + bytes: sql`length(CAST(${EventTable.data} AS BLOB))`, + }) + .from(EventTable) + .leftJoin(FilePartArtifactBindingTable, eq(FilePartArtifactBindingTable.event_id, EventTable.id)) + .where( + and( + eq(EventTable.type, "message.part.updated.1"), + sql`length(CAST(${EventTable.data} AS BLOB)) > ${4 * 1024 * 1024}`, + sql`json_valid(${EventTable.data})`, + sql`json_type(${EventTable.data}, '$.part.url') = 'text'`, + sql`json_extract(${EventTable.data}, '$.part.url') LIKE 'data:%;base64,%'`, + isNull(FilePartArtifactBindingTable.event_id), + input?.afterID ? gt(EventTable.id, input.afterID) : undefined, + ), + ) + .orderBy(asc(EventTable.id)) + .limit(limit) + .all() + .pipe(Effect.orDie) + const oversized = rows.find((row) => row.bytes > MAX_BYTES) + if (oversized) + return yield* fail( + "pending", + `Legacy file-part event ${oversized.id} is ${oversized.bytes} bytes; limit is ${MAX_BYTES} bytes`, + ) + yield* Effect.forEach( + rows, + (row) => + Effect.gen(function* () { + const source = yield* db + .select({ data: EventTable.data }) + .from(EventTable) + .where(eq(EventTable.id, row.id)) + .get() + .pipe(Effect.orDie) + if (!source) + return yield* fail("pending", `Legacy file-part event ${row.id} disappeared during maintenance`) + const prepared = prepare("message.part.updated", source.data, row.bytes, 4 * 1024 * 1024) + if (prepared.artifacts.length !== 1) + return yield* fail("pending", `Legacy file-part event ${row.id} cannot be externalized canonically`) + yield* put(db, prepared.artifacts[0]!, input?.now) + yield* bind(db, { + eventID: row.id, + aggregateID: row.aggregateID, + seq: row.seq, + type: "message.part.updated", + data: prepared.data, + originalData: source.data, + now: input?.now, + }) + const originalBinding = descriptorFromData("message.part.updated", prepared.data) + const originalPart = source.data.part + const canonicalPart = prepared.data.part + if ( + !originalBinding || + !originalPart || + typeof originalPart !== "object" || + Array.isArray(originalPart) || + !canonicalPart || + typeof canonicalPart !== "object" || + Array.isArray(canonicalPart) + ) + return yield* fail(prepared.artifacts[0]!.descriptor.id, `Legacy file-part event ${row.id} has invalid part data`) + const projected = yield* db + .select() + .from(PartTable) + .where(eq(PartTable.id, originalBinding.partID as typeof PartTable.$inferSelect.id)) + .get() + .pipe(Effect.orDie) + if (!projected) + return yield* fail( + prepared.artifacts[0]!.descriptor.id, + `Legacy file-part event ${row.id} has no corresponding projected part`, + ) + const sourcePart = originalPart as Record + if (projected.session_id !== row.aggregateID || projected.message_id !== sourcePart.messageID) + return yield* fail(prepared.artifacts[0]!.descriptor.id, `Legacy file-part event ${row.id} has a conflicting projected part`) + const sourceData = Object.fromEntries( + Object.entries(sourcePart).filter(([key]) => key !== "id" && key !== "sessionID" && key !== "messageID"), + ) + if (!isDeepStrictEqual(projected.data, sourceData)) + return yield* fail( + prepared.artifacts[0]!.descriptor.id, + `Legacy file-part event ${row.id} no longer matches its projected part`, + ) + const canonicalData = Object.fromEntries( + Object.entries(canonicalPart as Record).filter( + ([key]) => key !== "id" && key !== "sessionID" && key !== "messageID", + ), + ) as typeof PartTable.$inferInsert.data + yield* db + .update(PartTable) + .set({ data: canonicalData }) + .where(eq(PartTable.id, originalBinding.partID as typeof PartTable.$inferSelect.id)) + .run() + .pipe(Effect.orDie) + }), + { discard: true }, + ) + return { processed: rows.length, next: rows.at(-1)?.id } + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie) +} + +function validate(descriptor: Descriptor, chunks: readonly Uint8Array[]) { + return Effect.gen(function* () { + if ( + descriptor.codec !== CODEC || + descriptor.id !== `fpart_${descriptor.hash}` || + descriptor.chunkBytes !== CHUNK_BYTES || + descriptor.bytes > MAX_BYTES || + descriptor.chunks !== Math.max(1, Math.ceil(descriptor.bytes / CHUNK_BYTES)) || + chunks.length !== descriptor.chunks + ) + return yield* fail(descriptor.id, "Artifact descriptor is not canonical") + const digest = createHash("sha256") + let bytes = 0 + for (const [index, chunk] of chunks.entries()) { + const expected = index === chunks.length - 1 ? descriptor.bytes - index * CHUNK_BYTES : CHUNK_BYTES + if (chunk.byteLength !== expected) return yield* fail(descriptor.id, `Artifact chunk ${index} has an invalid length`) + bytes += chunk.byteLength + digest.update(chunk) + } + if (bytes !== descriptor.bytes || digest.digest("hex") !== descriptor.hash) + return yield* fail(descriptor.id, "Artifact body hash does not match its descriptor") + }) +} + +function stage(db: Database.Interface["db"], metadata: Metadata) { + return Effect.gen(function* () { + yield* requireAvailable(db, metadata.descriptor) + const chunks = yield* db + .select({ index: FilePartArtifactChunkTable.chunk_index, hash: FilePartArtifactChunkTable.chunk_hash, data: FilePartArtifactChunkTable.data }) + .from(FilePartArtifactChunkTable) + .where(eq(FilePartArtifactChunkTable.artifact_id, metadata.descriptor.id)) + .orderBy(asc(FilePartArtifactChunkTable.chunk_index)) + .all() + .pipe(Effect.orDie) + if ( + metadata.chunkHashes.length !== metadata.descriptor.chunks || + chunks.length !== metadata.descriptor.chunks || + chunks.some( + (chunk, index) => + chunk.index !== index || + chunk.hash !== metadata.chunkHashes[index] || + Hash.sha256(chunk.data) !== chunk.hash, + ) + ) + return yield* fail(metadata.descriptor.id, "Artifact import metadata does not match durable chunks") + yield* validate(metadata.descriptor, chunks.map((chunk) => chunk.data)) + const existing = yield* db + .select() + .from(FilePartArtifactImportTable) + .where(eq(FilePartArtifactImportTable.event_id, metadata.eventID as EventV2.ID)) + .get() + .pipe(Effect.orDie) + if (existing) { + if ( + existing.aggregate_id !== metadata.aggregateID || + existing.seq !== metadata.seq || + existing.artifact_id !== metadata.descriptor.id || + existing.original_data_hash !== metadata.originalDataHash || + existing.canonical_data_hash !== metadata.canonicalDataHash || + !isDeepStrictEqual(existing.canonical_data, metadata.canonicalData) + ) + return yield* fail(metadata.descriptor.id, "Artifact import receipt conflicts with durable CAS") + return + } + const binding = yield* db + .select() + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, metadata.eventID as EventV2.ID)) + .get() + .pipe(Effect.orDie) + if (binding) { + if ( + binding.aggregate_id !== metadata.aggregateID || + binding.seq !== metadata.seq || + binding.artifact_id !== metadata.descriptor.id || + binding.original_data_hash !== metadata.originalDataHash || + binding.canonical_data_hash !== metadata.canonicalDataHash || + !isDeepStrictEqual(binding.canonical_data, metadata.canonicalData) + ) + return yield* fail(metadata.descriptor.id, "Artifact binding conflicts with import metadata") + return + } + yield* db + .insert(FilePartArtifactImportTable) + .values({ + event_id: metadata.eventID as EventV2.ID, + aggregate_id: metadata.aggregateID, + seq: metadata.seq, + artifact_id: metadata.descriptor.id, + original_data_hash: metadata.originalDataHash, + canonical_data_hash: metadata.canonicalDataHash, + canonical_data: metadata.canonicalData, + created_at: Date.now(), + }) + .run() + .pipe(Effect.orDie) + }) +} diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index 5b85e7e48..a349116b9 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -1,10 +1,13 @@ export * as SessionProjector from "./projector" -import { and, desc, eq, sql } from "drizzle-orm" +import { and, asc, desc, eq, gt, sql } from "drizzle-orm" +import { isDeepStrictEqual } from "node:util" import { DateTime, Effect, Layer, Schema } from "effect" import { Database } from "../database/database" import { makeGlobalNode } from "../effect/app-node" import { EventV2 } from "../event" +import { EventArtifactTable } from "../event/sql" +import { FilePartArtifact } from "../file-part-artifact" import { SessionEvent } from "./event" import { SessionV1 } from "../v1/session" import { WorkspaceTable } from "../control-plane/workspace.sql" @@ -13,7 +16,14 @@ import { SessionMessageUpdater } from "./message-updater" import { SessionInput } from "./input" import { WorkspaceV2 } from "../workspace" import { SessionContextEpoch } from "./context-epoch" -import { MessageTable, PartTable, SessionMessageTable, SessionTable } from "./sql" +import { + MessageTable, + PartTable, + SessionContextEpochTable, + SessionInputTable, + SessionMessageTable, + SessionTable, +} from "./sql" import type { DeepMutable } from "../schema" import { SessionSchema } from "./schema" @@ -104,6 +114,48 @@ function partData(part: (typeof SessionV1.Event.PartUpdated.Type)["data"]["part" return rest as DeepMutable } +const snapshotTableNames = [ + "session", + "message", + "part", + "file_part_artifact_binding", + "session_message", + "session_input", + "session_context_epoch", +] as const + +function stripDiffPatches(value: unknown): unknown { + if (!value || typeof value !== "object" || Array.isArray(value)) return value + const record = value as Record + if (!record.summary || typeof record.summary !== "object" || Array.isArray(record.summary)) return value + const summary = record.summary as Record + if (!Array.isArray(summary.diffs)) return value + return { + ...record, + summary: { + ...summary, + diffs: summary.diffs.map((diff) => { + if (!diff || typeof diff !== "object" || Array.isArray(diff)) return diff + const { patch: _, ...descriptor } = diff as Record + return descriptor + }), + }, + } +} + +function snapshotCursor(index: number, key: string) { + return `${index}:${key}` +} + +function parseSnapshotCursor(cursor: string | undefined) { + if (!cursor) return { index: 0, key: "" } + const separator = cursor.indexOf(":") + const index = Number(cursor.slice(0, separator)) + if (separator < 0 || !Number.isInteger(index) || index < 0 || index >= snapshotTableNames.length) + throw new EventV2.InvalidSyncEventError({ type: "snapshot", message: `Invalid Session snapshot cursor ${cursor}` }) + return { index, key: cursor.slice(separator + 1) } +} + function applyUsage( db: DatabaseService, sessionID: (typeof SessionV1.Event.MessageUpdated.Type)["data"]["sessionID"], @@ -229,6 +281,293 @@ export const layer = Layer.effectDiscard( Effect.gen(function* () { const events = yield* EventV2.Service const { db } = yield* Database.Service + if (events.registerSnapshotCodec) yield* events.registerSnapshotCodec({ + codec: "session-projection", + schemaVersion: 1, + rebuildEventTypes: new Set([ + SessionV1.Event.Created, + SessionV1.Event.Updated, + SessionV1.Event.MessageUpdated, + SessionV1.Event.MessageRemoved, + SessionV1.Event.PartUpdated, + SessionV1.Event.PartRemoved, + SessionEvent.AgentSwitched, + SessionEvent.ModelSwitched, + SessionEvent.ContextUpdated, + SessionEvent.Synthetic, + SessionEvent.Shell.Started, + SessionEvent.Shell.Ended, + SessionEvent.Step.Started, + SessionEvent.Step.Ended, + SessionEvent.Step.Failed, + SessionEvent.Text.Started, + SessionEvent.Text.Ended, + SessionEvent.Tool.Input.Started, + SessionEvent.Tool.Input.Ended, + SessionEvent.Tool.Called, + SessionEvent.Tool.Progress, + SessionEvent.Tool.Success, + SessionEvent.Tool.Failed, + SessionEvent.Reasoning.Started, + SessionEvent.Reasoning.Ended, + SessionEvent.Compaction.Ended, + ].map((definition) => EventV2.versionedType(definition.type, definition.sync!.version))), + revision: (aggregateID) => + Effect.gen(function* () { + const session = yield* db.select({ mutationEpoch: SessionTable.mutation_epoch }) + .from(SessionTable).where(eq(SessionTable.id, SessionSchema.ID.make(aggregateID))).get().pipe(Effect.orDie) + const context = yield* db.select({ revision: SessionContextEpochTable.revision }) + .from(SessionContextEpochTable).where(eq(SessionContextEpochTable.session_id, SessionSchema.ID.make(aggregateID))).get().pipe(Effect.orDie) + return `${session?.mutationEpoch ?? -1}:${context?.revision ?? -1}` + }), + next: (aggregateID, cursor) => + Effect.gen(function* () { + const position = parseSnapshotCursor(cursor) + for (const [index, tableName] of snapshotTableNames.entries()) { + if (index < position.index) continue + const after = index === position.index ? position.key : "" + if (tableName === "session") { + if (after) continue + const row = yield* db + .select() + .from(SessionTable) + .where(eq(SessionTable.id, SessionSchema.ID.make(aggregateID))) + .get() + .pipe(Effect.orDie) + if (!row) return + return { + cursor: snapshotCursor(index, row.id), + tableName, + rowKey: row.id, + value: { + id: row.id, + project_id: row.project_id, + workspace_id: row.workspace_id, + parent_id: row.parent_id, + slug: row.slug, + directory: row.directory, + path: row.path, + title: row.title, + version: row.version, + share_url: row.share_url, + summary_additions: row.summary_additions, + summary_deletions: row.summary_deletions, + summary_files: row.summary_files, + summary_diffs: row.summary_diffs?.map(({ patch: _, ...diff }) => diff), + summary_diff_manifest: row.summary_diff_manifest, + metadata: row.metadata, + cost: row.cost, + tokens_input: row.tokens_input, + tokens_output: row.tokens_output, + tokens_reasoning: row.tokens_reasoning, + tokens_cache_read: row.tokens_cache_read, + tokens_cache_write: row.tokens_cache_write, + revert: row.revert, + permission: row.permission, + agent: row.agent, + model: row.model, + time_created: row.time_created, + time_updated: row.time_updated, + time_compacting: row.time_compacting, + time_archived: row.time_archived, + preview: row.preview, + }, + } + } + if (tableName === "message") { + const row = yield* db + .select({ + id: MessageTable.id, + session_id: MessageTable.session_id, + time_created: MessageTable.time_created, + time_updated: MessageTable.time_updated, + data: MessageTable.data, + canonicalData: EventArtifactTable.canonical_data, + }) + .from(MessageTable) + .leftJoin( + EventArtifactTable, + and( + eq(EventArtifactTable.aggregate_id, aggregateID), + sql`json_extract(${EventArtifactTable.canonical_data}, '$.info.id') = ${MessageTable.id}`, + ), + ) + .where( + and( + eq(MessageTable.session_id, SessionSchema.ID.make(aggregateID)), + gt(sql`${MessageTable.id}`, after), + ), + ) + .orderBy(asc(MessageTable.id)) + .limit(1) + .get() + .pipe(Effect.orDie) + if (row) + return { + cursor: snapshotCursor(index, row.id), + tableName, + rowKey: row.id, + value: { + id: row.id, + session_id: row.session_id, + time_created: row.time_created, + time_updated: row.time_updated, + data: stripDiffPatches( + row.canonicalData && typeof row.canonicalData.info === "object" && row.canonicalData.info + ? messageData(row.canonicalData.info as (typeof SessionV1.Event.MessageUpdated.Type)["data"]["info"]) + : row.data, + ), + }, + } + continue + } + if (tableName === "part") { + const row = yield* db.select().from(PartTable) + .where(and(eq(PartTable.session_id, SessionSchema.ID.make(aggregateID)), gt(sql`${PartTable.id}`, after))) + .orderBy(asc(PartTable.id)).limit(1).get().pipe(Effect.orDie) + if (!row) continue + const { provenance: _, ...canonical } = row + return { cursor: snapshotCursor(index, row.id), tableName, rowKey: row.id, value: canonical } + } + if (tableName === "file_part_artifact_binding") { + let artifactAfter = after + while (true) { + const part = yield* db.select({ id: PartTable.id, messageID: PartTable.message_id, data: PartTable.data }).from(PartTable) + .where(and(eq(PartTable.session_id, SessionSchema.ID.make(aggregateID)), gt(sql`${PartTable.id}`, artifactAfter))) + .orderBy(asc(PartTable.id)).limit(1).get().pipe(Effect.orDie) + if (!part) break + const descriptor = FilePartArtifact.descriptor({ + sessionID: aggregateID, + part: { id: part.id, messageID: part.messageID, sessionID: aggregateID, ...part.data }, + }) + if (descriptor) { + const metadata = yield* FilePartArtifact.snapshotRef(db, { aggregateID, partID: part.id, descriptor }) + return { cursor: snapshotCursor(index, part.id), tableName, rowKey: part.id, value: { partID: part.id, metadata } } + } + artifactAfter = part.id + } + continue + } + if (tableName === "session_message") { + const row = yield* db.select().from(SessionMessageTable) + .where(and(eq(SessionMessageTable.session_id, SessionSchema.ID.make(aggregateID)), gt(sql`${SessionMessageTable.id}`, after))) + .orderBy(asc(SessionMessageTable.id)).limit(1).get().pipe(Effect.orDie) + if (!row) continue + return { cursor: snapshotCursor(index, row.id), tableName, rowKey: row.id, value: row } + } + if (tableName === "session_input") { + const row = yield* db.select().from(SessionInputTable) + .where(and( + eq(SessionInputTable.session_id, SessionSchema.ID.make(aggregateID)), + sql`${SessionInputTable.promoted_seq} is not null`, + gt(sql`${SessionInputTable.id}`, after), + )) + .orderBy(asc(SessionInputTable.id)).limit(1).get().pipe(Effect.orDie) + if (!row) continue + return { cursor: snapshotCursor(index, row.id), tableName, rowKey: row.id, value: row } + } + if (after) continue + const row = yield* db.select().from(SessionContextEpochTable) + .where(eq(SessionContextEpochTable.session_id, SessionSchema.ID.make(aggregateID))) + .get().pipe(Effect.orDie) + if (!row) continue + return { cursor: snapshotCursor(index, row.session_id), tableName, rowKey: row.session_id, value: row } + } + }), + clear: (aggregateID, snapshotID) => + Effect.gen(function* () { + const sessionID = SessionSchema.ID.make(aggregateID) + yield* db.delete(SessionContextEpochTable).where(eq(SessionContextEpochTable.session_id, sessionID)).run().pipe(Effect.orDie) + yield* db.delete(SessionMessageTable).where(eq(SessionMessageTable.session_id, sessionID)).run().pipe(Effect.orDie) + yield* db.run(sql` + DELETE FROM part + WHERE session_id = ${sessionID} + AND NOT EXISTS ( + SELECT 1 FROM event_snapshot_row row + WHERE row.snapshot_id = ${snapshotID} + AND row.table_name = 'part' + AND row.row_key = part.id + ) + `).pipe(Effect.orDie) + yield* db.run(sql` + DELETE FROM message + WHERE session_id = ${sessionID} + AND NOT EXISTS ( + SELECT 1 FROM event_snapshot_row row + WHERE row.snapshot_id = ${snapshotID} + AND row.table_name = 'message' + AND row.row_key = message.id + ) + `).pipe(Effect.orDie) + }), + import: (aggregateID, row, ownerID) => { + if (row.tableName === "session") { + const value = row.value as typeof SessionTable.$inferInsert + if ( + value.id !== aggregateID || + (ownerID !== undefined && value.workspace_id !== ownerID) + ) + return Effect.die( + new EventV2.InvalidSyncEventError({ + type: "snapshot", + message: `Session snapshot root does not match aggregate or owner ${aggregateID}`, + }), + ) + return Effect.gen(function* () { + const { id: _, ...update } = value + const updated = yield* db.update(SessionTable).set(update).where(eq(SessionTable.id, SessionSchema.ID.make(aggregateID))) + .returning({ id: SessionTable.id }).get().pipe(Effect.orDie) + if (!updated) yield* db.insert(SessionTable).values(value).run().pipe(Effect.orDie) + }) + } + if (row.tableName === "file_part_artifact_binding") { + const value = row.value as { readonly partID?: unknown; readonly metadata?: unknown } + if (value.partID !== row.rowKey || !Schema.is(FilePartArtifact.Metadata)(value.metadata) || value.metadata.aggregateID !== aggregateID) + return Effect.die(new EventV2.InvalidSyncEventError({ type: "snapshot", message: `Invalid file artifact snapshot row ${row.rowKey}` })) + return FilePartArtifact.bindSnapshotRef({ metadata: value.metadata, partID: value.partID }).pipe( + Effect.provideService(Database.Service, { db }), + ) + } + if (row.value.session_id !== aggregateID) + return Effect.die( + new EventV2.InvalidSyncEventError({ + type: "snapshot", + message: `Session snapshot row ${row.tableName}:${row.rowKey} crosses aggregate authority`, + }), + ) + if (row.tableName === "message") { + const value = row.value as typeof MessageTable.$inferInsert + return db.insert(MessageTable).values(value).onConflictDoUpdate({ target: MessageTable.id, set: value }).run().pipe(Effect.orDie) + } + if (row.tableName === "part") { + const value = row.value as typeof PartTable.$inferInsert + return db.insert(PartTable).values(value).onConflictDoUpdate({ target: PartTable.id, set: value }).run().pipe(Effect.orDie) + } + if (row.tableName === "session_message") + return db + .insert(SessionMessageTable) + .values(row.value as typeof SessionMessageTable.$inferInsert) + .run() + .pipe(Effect.orDie) + if (row.tableName === "session_input") + return Effect.gen(function* () { + const value = row.value as typeof SessionInputTable.$inferInsert + const existing = yield* db.select().from(SessionInputTable).where(eq(SessionInputTable.id, value.id)).get().pipe(Effect.orDie) + if (!existing) return yield* db.insert(SessionInputTable).values(value).run().pipe(Effect.orDie) + if (!isDeepStrictEqual(existing, { ...value, promoted_seq: value.promoted_seq ?? null })) + return yield* Effect.die(new EventV2.InvalidSyncEventError({ type: "snapshot", message: `Session input ${value.id} conflicts with local admission authority` })) + }) + if (row.tableName === "session_context_epoch") + return db + .insert(SessionContextEpochTable) + .values(row.value as typeof SessionContextEpochTable.$inferInsert) + .run() + .pipe(Effect.orDie) + return Effect.die( + new EventV2.InvalidSyncEventError({ type: "snapshot", message: `Unknown Session snapshot table ${row.tableName}` }), + ) + }, + }) yield* events.beforeCommit((event) => SessionInput.guardReservedID(db, event)) yield* events.beforeCommit((event) => Effect.gen(function* () { diff --git a/packages/core/src/session/transfer.sql.ts b/packages/core/src/session/transfer.sql.ts new file mode 100644 index 000000000..2f6fb1c0f --- /dev/null +++ b/packages/core/src/session/transfer.sql.ts @@ -0,0 +1,49 @@ +import { integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { sql } from "drizzle-orm" +import { SessionTable } from "../session/sql" + +export const SessionTransferOperationTable = sqliteTable( + "session_transfer_operation", + { + transfer_id: text().primaryKey(), + session_id: text() + .notNull() + .references(() => SessionTable.id, { onDelete: "cascade" }), + source_workspace_id: text(), + target_workspace_id: text(), + source_owner_id: text(), + target_owner_id: text(), + source_event_seq: integer().notNull(), + source_mutation_epoch: integer().notNull(), + snapshot_id: text(), + snapshot_hash: text(), + state: text() + .$type<"admitted" | "source_frozen" | "target_staged" | "owner_committed" | "target_activated" | "aborted">() + .notNull(), + request_hash: text().notNull(), + error_code: text(), + created_at: integer().notNull(), + updated_at: integer().notNull(), + completed_at: integer(), + }, + (table) => [ + uniqueIndex("session_transfer_operation_session_request_idx").on(table.session_id, table.request_hash), + uniqueIndex("session_transfer_operation_active_idx") + .on(table.session_id) + .where(sql`${table.state} NOT IN ('target_activated', 'aborted')`), + ], +) + +export const SessionTransferTargetReceiptTable = sqliteTable("session_transfer_target_receipt", { + transfer_id: text().primaryKey(), + session_id: text().notNull(), + source_snapshot_id: text().notNull(), + source_snapshot_hash: text().notNull(), + source_event_seq: integer().notNull(), + target_workspace_id: text(), + target_owner_id: text(), + state: text().$type<"staged" | "activated">().notNull(), + activated_snapshot_id: text(), + activated_at: integer(), + created_at: integer().notNull(), +}) diff --git a/packages/core/src/v1/session.ts b/packages/core/src/v1/session.ts index 9ef607de2..4a39ab89f 100644 --- a/packages/core/src/v1/session.ts +++ b/packages/core/src/v1/session.ts @@ -189,6 +189,16 @@ export const FilePart = Schema.Struct({ mime: Schema.String, filename: Schema.optional(Schema.String), url: Schema.String, + artifact: Schema.optional( + Schema.Struct({ + codec: Schema.Literal("file-part.v1"), + id: Schema.String.check(Schema.isPattern(/^fpart_[a-f0-9]{64}$/)), + hash: Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)), + bytes: NonNegativeInt.check(Schema.isLessThanOrEqualTo(32 * 1024 * 1024)), + chunkBytes: Schema.Literal(262_144 as const), + chunks: NonNegativeInt.check(Schema.isGreaterThanOrEqualTo(1), Schema.isLessThanOrEqualTo(128)), + }), + ), source: Schema.optional(FilePartSource), }).annotate({ identifier: "FilePart" }) export type FilePart = Types.DeepMutable> diff --git a/packages/core/test/database-capability.test.ts b/packages/core/test/database-capability.test.ts new file mode 100644 index 000000000..3114de6e3 --- /dev/null +++ b/packages/core/test/database-capability.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from "bun:test" +import { sql } from "drizzle-orm" +import { Effect } from "effect" +import { Database } from "@deepagent-code/core/database/database" +import { testEffect } from "./lib/effect" +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" + +const it = testEffect(Database.layerFromPath(":memory:")) + +describe("database capability", () => { + it.effect("persists an immutable reader and writer compatibility boundary", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + expect(yield* db.get(sql`SELECT capability, minimum_reader_protocol, minimum_writer_protocol + FROM database_capability WHERE capability = 'bounded_event_snapshot_v1'`)).toEqual({ + capability: "bounded_event_snapshot_v1", + minimum_reader_protocol: 2, + minimum_writer_protocol: 2, + }) + expect( + (yield* db.run(sql`UPDATE database_capability SET minimum_writer_protocol = 3 + WHERE capability = 'bounded_event_snapshot_v1'`).pipe(Effect.exit))._tag, + ).toBe("Failure") + expect( + (yield* db.run(sql`DELETE FROM database_capability + WHERE capability = 'bounded_event_snapshot_v1'`).pipe(Effect.exit))._tag, + ).toBe("Failure") + }), + ) + + test("fails startup before exposing a database that requires a newer protocol", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "deepagent-code-db-capability-")) + const filename = path.join(directory, "database.db") + try { + await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const { db } = yield* Database.Service + yield* db.run(sql`DROP TRIGGER database_capability_immutable_update`) + yield* db.run(sql`UPDATE database_capability SET minimum_reader_protocol = 3, minimum_writer_protocol = 3 + WHERE capability = 'bounded_event_snapshot_v1'`) + }).pipe(Effect.provide(Database.layerFromPath(filename))), + ), + ) + const exit = await Effect.runPromise( + Effect.scoped(Database.Service.pipe(Effect.provide(Database.layerFromPath(filename)))).pipe(Effect.exit), + ) + expect(exit._tag).toBe("Failure") + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/packages/core/test/database-migration.test.ts b/packages/core/test/database-migration.test.ts index 4f6012d51..a67348b8f 100644 --- a/packages/core/test/database-migration.test.ts +++ b/packages/core/test/database-migration.test.ts @@ -59,7 +59,7 @@ describe("DatabaseMigration", () => { ) }) - test("upgrades EventV2 history and allocates sync_seq for old and new writers", async () => { + test("stages EventV2 history without rewriting legacy rows and fences old writers", async () => { await run( Effect.gen(function* () { const db = yield* makeDb @@ -70,19 +70,22 @@ describe("DatabaseMigration", () => { yield* db.run(sql`INSERT INTO event VALUES ('event-b', 'session-a', 1, 'test.1', '{}')`) yield* DatabaseMigration.applyOnly(db, [eventSnapshotAuthorityMigration]) - yield* db.run(sql`INSERT INTO event(id, aggregate_id, seq, type, data) VALUES ('event-old-writer', 'session-a', 2, 'test.1', '{}')`) + const oldWriter = yield* db + .run(sql`INSERT INTO event(id, aggregate_id, seq, type, data) VALUES ('event-old-writer', 'session-a', 2, 'test.1', '{}')`) + .pipe(Effect.exit) yield* db.run(sql`UPDATE event_sync_sequence SET seq = seq + 1 WHERE id = 1`) - yield* db.run(sql`INSERT INTO event(id, aggregate_id, seq, type, data, sync_seq) VALUES ('event-new-writer', 'session-a', 3, 'test.1', '{}', (SELECT seq FROM event_sync_sequence WHERE id = 1))`) - yield* db.run(sql`INSERT INTO event(id, aggregate_id, seq, type, data) VALUES ('event-old-writer-again', 'session-a', 4, 'test.1', '{}')`) - - expect(yield* db.all(sql`SELECT id, sync_seq FROM event ORDER BY sync_seq`)).toEqual([ - { id: "event-a", sync_seq: 1 }, - { id: "event-b", sync_seq: 2 }, - { id: "event-old-writer", sync_seq: 3 }, - { id: "event-new-writer", sync_seq: 4 }, - { id: "event-old-writer-again", sync_seq: 5 }, + yield* db.run(sql`INSERT INTO event(id, aggregate_id, seq, type, data, sync_seq) VALUES ('event-new-writer', 'session-a', 2, 'test.1', '{}', (SELECT seq FROM event_sync_sequence WHERE id = 1))`) + + expect(oldWriter._tag).toBe("Failure") + expect(yield* db.all(sql`SELECT id, sync_seq FROM event ORDER BY rowid`)).toEqual([ + { id: "event-a", sync_seq: null }, + { id: "event-b", sync_seq: null }, + { id: "event-new-writer", sync_seq: 3 }, + ]) + expect(yield* db.all(sql`SELECT sync_seq, event_id FROM event_sync_index`)).toEqual([ + { sync_seq: 3, event_id: "event-new-writer" }, ]) - expect(yield* db.get(sql`SELECT seq, length(generation) AS generation_length, length(cursor_secret) AS secret_length FROM event_sync_sequence WHERE id = 1`)).toEqual({ seq: 5, generation_length: 32, secret_length: 64 }) + expect(yield* db.get(sql`SELECT seq, backfill_complete, length(generation) AS generation_length, length(cursor_secret) AS secret_length FROM event_sync_sequence WHERE id = 1`)).toEqual({ seq: 3, backfill_complete: 0, generation_length: 32, secret_length: 64 }) }), ) }) diff --git a/packages/core/test/event-snapshot.test.ts b/packages/core/test/event-snapshot.test.ts new file mode 100644 index 000000000..671ca8b59 --- /dev/null +++ b/packages/core/test/event-snapshot.test.ts @@ -0,0 +1,195 @@ +import { describe, expect } from "bun:test" +import path from "node:path" +import { Effect, Layer } from "effect" +import { asc, eq } from "drizzle-orm" +import { Database } from "@deepagent-code/core/database/database" +import { EventV2 } from "@deepagent-code/core/event" +import { + EventCompactionReceiptTable, + EventDedupeTable, + EventSequenceTable, + EventSnapshotTable, + EventTable, +} from "@deepagent-code/core/event/sql" +import { ProjectV2 } from "@deepagent-code/core/project" +import { ProjectTable } from "@deepagent-code/core/project/sql" +import { ProviderV2 } from "@deepagent-code/core/provider" +import { ModelV2 } from "@deepagent-code/core/model" +import { AbsolutePath } from "@deepagent-code/core/schema" +import { SessionV2 } from "@deepagent-code/core/session" +import { MessageTable, PartTable, SessionInputTable, SessionHistoryStateTable, SessionTable } from "@deepagent-code/core/session/sql" +import { SessionMessage } from "@deepagent-code/core/session/message" +import { SessionProjector } from "@deepagent-code/core/session/projector" +import { SessionV1 } from "@deepagent-code/core/v1/session" +import { testEffect } from "./lib/effect" +import { tmpdir } from "./fixture/tmpdir" + +const sourceDatabase = Database.layerFromPath(":memory:") +const sourceEvents = EventV2.layer.pipe(Layer.provide(sourceDatabase)) +const sourceProjector = SessionProjector.layer.pipe(Layer.provide(sourceEvents), Layer.provide(sourceDatabase)) +const it = testEffect(Layer.mergeAll(sourceDatabase, sourceEvents, sourceProjector)) +const sessionID = SessionV2.ID.make("ses_snapshot_two_database") +const projectID = ProjectV2.ID.global + +const message = (id: string, text: string) => ({ + sessionID, + info: { + id: SessionV1.MessageID.make(id), + sessionID, + role: "user" as const, + time: { created: 1 }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("test") }, + system: text, + }, +}) + +describe("EventV2 canonical projection snapshots", () => { + it.effect("stages bounded checkpoints, imports atomically into a fresh database, and continues at the next event", () => + Effect.gen(function* () { + const source = yield* EventV2.Service + const sourceDb = (yield* Database.Service).db + yield* sourceDb.insert(ProjectTable).values({ id: projectID, worktree: AbsolutePath.make("/project"), sandboxes: [] }).run().pipe(Effect.orDie) + const info = SessionV1.SessionInfo.make({ + id: sessionID, + slug: "snapshot", + version: "test", + projectID, + directory: "/project", + title: "snapshot", + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 1, updated: 1 }, + }) + yield* source.publish(SessionV1.Event.Created, { sessionID, info }) + yield* source.publish(SessionV1.Event.MessageUpdated, message("msg_snapshot_one", "one")) + yield* sourceDb.insert(SessionInputTable).values({ + id: SessionMessage.ID.make("msg_snapshot_pending"), + session_id: sessionID, + prompt: { text: "must not transfer" }, + delivery: "queue", + admitted_seq: 100, + time_created: 1, + }).run().pipe(Effect.orDie) + + const stale = yield* source.prepareCheckpoint!({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(1), + expectedLatest: EventV2.Cursor.make(1), + codec: "session-projection", + schemaVersion: 1, + }) + let staged = stale + while (staged.hasMore) staged = yield* source.stageCheckpoint!({ snapshotID: staged.snapshotID, limit: 1 }) + expect((yield* sourceDb.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, sessionID)).get().pipe(Effect.orDie))?.retention_floor_seq).toBeNull() + + yield* source.publish(SessionV1.Event.MessageUpdated, message("msg_snapshot_two", "two")) + const staleFinalize = yield* source.finalizeCheckpoint!({ snapshotID: staged.snapshotID }).pipe(Effect.catchDefect(Effect.succeed)) + expect(staleFinalize).toBeInstanceOf(EventV2.InvalidSyncEventError) + expect(yield* source.snapshot(sessionID)).toBeUndefined() + + const active = yield* source.checkpoint({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(2), + expectedLatest: EventV2.Cursor.make(2), + codec: "session-projection", + schemaVersion: 1, + }) + expect((yield* sourceDb.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, sessionID)).get().pipe(Effect.orDie))?.retention_floor_seq).toBe(2) + expect(active.body).toMatchObject({ format: "chunked-rows.v1", tables: { session: 1, message: 2 } }) + expect((active.body.tables as Record).session_input).toBeUndefined() + + yield* source.publish(SessionV1.Event.MessageUpdated, message("msg_snapshot_tail", "tail")) + const tail = yield* sourceDb.select().from(EventTable) + .where(eq(EventTable.aggregate_id, sessionID)).orderBy(asc(EventTable.seq)).all().pipe(Effect.orDie) + const rows = [] as EventV2.SerializedSnapshotRow[] + let rowAfter = -1 + while (true) { + const page = yield* source.snapshotRows!({ snapshotID: active.snapshotID, after: rowAfter, limit: 2 }) + if (page.length === 0) break + rows.push(...page) + rowAfter = page.at(-1)!.rowIndex + } + const chunks = new Map() + for (const row of rows) { + const output = [] as EventV2.SerializedSnapshotChunk[] + let chunkAfter = -1 + while (true) { + const page = yield* source.snapshotChunks!({ rowHash: row.rowHash, after: chunkAfter }) + if (page.length === 0) break + output.push(...page) + chunkAfter = page.at(-1)!.chunkIndex + } + chunks.set(row.rowHash, output) + } + + const tmp = yield* Effect.acquireRelease(Effect.promise(() => tmpdir()), (value) => Effect.promise(() => value[Symbol.asyncDispose]())) + const targetDatabase = Database.layerFromPath(path.join(tmp.path, "target.sqlite")) + const targetEvents = EventV2.layer.pipe(Layer.provide(targetDatabase)) + const targetProjector = SessionProjector.layer.pipe(Layer.provide(targetEvents), Layer.provide(targetDatabase)) + yield* Effect.gen(function* () { + const target = yield* EventV2.Service + const targetDb = (yield* Database.Service).db + yield* targetDb.insert(ProjectTable).values({ id: projectID, worktree: AbsolutePath.make("/project"), sandboxes: [] }).run().pipe(Effect.orDie) + yield* targetDb.insert(SessionTable).values({ + id: sessionID, + project_id: projectID, + slug: "target", + directory: "/project", + title: "target", + version: "test", + }).run().pipe(Effect.orDie) + yield* targetDb.insert(SessionInputTable).values({ + id: SessionMessage.ID.make("msg_target_pending"), + session_id: sessionID, + prompt: { text: "local pending" }, + delivery: "queue", + admitted_seq: 200, + time_created: 1, + }).run().pipe(Effect.orDie) + yield* targetDb.insert(SessionHistoryStateTable).values({ + session_id: sessionID, + state: "provisioning", + time_created: 1, + time_updated: 1, + }).run().pipe(Effect.orDie) + yield* targetDb.insert(MessageTable).values({ id: SessionV1.MessageID.make("msg_target_stale"), session_id: sessionID, + time_created: 1, data: { role: "user", time: { created: 1 }, agent: "build", model: { providerID: "test", modelID: "test" } } as typeof MessageTable.$inferInsert["data"] }).run().pipe(Effect.orDie) + yield* targetDb.insert(PartTable).values({ id: SessionV1.PartID.make("prt_target_stale"), message_id: SessionV1.MessageID.make("msg_target_stale"), + session_id: sessionID, time_created: 1, data: { type: "text", text: "stale" } as typeof PartTable.$inferInsert["data"] }).run().pipe(Effect.orDie) + + yield* target.stageSnapshotRows!(active, rows) + yield* target.stageSnapshotRows!(active, rows) + for (const row of rows) { + yield* target.stageSnapshotChunks!(active, row, chunks.get(row.rowHash)!) + yield* target.stageSnapshotChunks!(active, row, chunks.get(row.rowHash)!) + } + yield* target.importSnapshot(active) + yield* target.importSnapshot(active) + expect(yield* targetDb.select().from(SessionInputTable).where(eq(SessionInputTable.id, SessionMessage.ID.make("msg_target_pending"))).get().pipe(Effect.orDie)).toBeDefined() + expect(yield* targetDb.select().from(SessionInputTable).where(eq(SessionInputTable.id, SessionMessage.ID.make("msg_snapshot_pending"))).get().pipe(Effect.orDie)).toBeUndefined() + expect(yield* targetDb.select().from(SessionHistoryStateTable).where(eq(SessionHistoryStateTable.session_id, sessionID)).get().pipe(Effect.orDie)).toMatchObject({ state: "provisioning" }) + expect(yield* targetDb.select().from(MessageTable).where(eq(MessageTable.id, SessionV1.MessageID.make("msg_target_stale"))).get().pipe(Effect.orDie)).toBeUndefined() + expect(yield* targetDb.select().from(PartTable).where(eq(PartTable.id, SessionV1.PartID.make("prt_target_stale"))).get().pipe(Effect.orDie)).toBeUndefined() + expect(yield* targetDb.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, sessionID)).get().pipe(Effect.orDie)).toMatchObject({ seq: 2, retention_floor_seq: 2, snapshot_id: active.snapshotID }) + expect((yield* target.snapshot(sessionID))?.syncSeq).not.toBe(active.syncSeq) + + const next = tail.find((event) => event.seq === 3)! + yield* target.replay({ id: next.id, aggregateID: next.aggregate_id, seq: next.seq, type: next.type, data: next.data }) + expect((yield* targetDb.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, sessionID)).get().pipe(Effect.orDie))?.seq).toBe(3) + }).pipe(Effect.provide(Layer.fresh(Layer.mergeAll(targetDatabase, targetEvents, targetProjector)))) + + let compacted = { deleted: 0, complete: false } + let total = 0 + while (!compacted.complete) { + compacted = yield* source.compact({ aggregateID: sessionID, throughSeq: EventV2.Cursor.make(2), limit: 1 }) + total += compacted.deleted + } + expect(total).toBe(3) + const dedupe = yield* sourceDb.select().from(EventDedupeTable).where(eq(EventDedupeTable.aggregate_id, sessionID)).all().pipe(Effect.orDie) + expect(dedupe).toHaveLength(3) + expect(dedupe.every((row) => row.source_data === null)).toBe(true) + expect(yield* sourceDb.select().from(EventCompactionReceiptTable).where(eq(EventCompactionReceiptTable.aggregate_id, sessionID)).get().pipe(Effect.orDie)).toMatchObject({ state: "complete", cursor_seq: 2 }) + }), + ) +}) diff --git a/packages/core/test/event-sync-backfill-migration.test.ts b/packages/core/test/event-sync-backfill-migration.test.ts new file mode 100644 index 000000000..fb20b744c --- /dev/null +++ b/packages/core/test/event-sync-backfill-migration.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from "bun:test" +import { SqliteClient } from "@effect/sql-sqlite-bun" +import { EffectDrizzleSqlite } from "@deepagent-code/effect-drizzle-sqlite" +import eventSnapshotAuthorityMigration from "@deepagent-code/core/database/migration/20260813100000_event_snapshot_authority" +import eventSyncBackfillAuthorityMigration from "@deepagent-code/core/database/migration/20260813125000_event_sync_backfill_authority" +import { DatabaseMigration } from "@deepagent-code/core/database/migration" +import { Effect } from "effect" +import { sql } from "drizzle-orm" +import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" + +const run = (effect: Effect.Effect) => + Effect.runPromise( + effect.pipe( + Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), + Effect.scoped, + ), + ) + +describe("Event sync backfill migration", () => { + test("installs staged authority without rewriting legacy event rows", async () => { + await run( + Effect.gen(function* () { + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + yield* db.run(sql`CREATE TABLE event_sequence (aggregate_id TEXT PRIMARY KEY, seq INTEGER NOT NULL, owner_id TEXT)`) + yield* db.run(sql`CREATE TABLE event ( + id TEXT PRIMARY KEY, + aggregate_id TEXT NOT NULL, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + data TEXT NOT NULL + )`) + yield* db.run(sql`INSERT INTO event_sequence VALUES ('session-a', 2, NULL)`) + yield* db.run(sql`INSERT INTO event VALUES + ('event-a', 'session-a', 0, 'test.1', '{"payload":"a"}'), + ('event-b', 'session-a', 1, 'test.1', '{"payload":"b"}'), + ('event-c', 'session-a', 2, 'test.1', '{"payload":"c"}')`) + + yield* DatabaseMigration.applyOnly(db, [eventSnapshotAuthorityMigration]) + + expect(yield* db.all(sql`SELECT id, sync_seq FROM event ORDER BY rowid`)).toEqual([ + { id: "event-a", sync_seq: null }, + { id: "event-b", sync_seq: null }, + { id: "event-c", sync_seq: null }, + ]) + expect( + yield* db.get(sql`SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'event_sync_seq_idx'`), + ).toBeUndefined() + expect(yield* db.get(sql`SELECT state, cursor_rowid, high_water_rowid, processed_count FROM event_sync_backfill`)).toEqual({ + state: "pending", + cursor_rowid: 0, + high_water_rowid: 3, + processed_count: 0, + }) + expect(yield* db.get(sql`SELECT seq, backfill_complete FROM event_sync_sequence`)).toEqual({ + seq: 3, + backfill_complete: 0, + }) + + yield* db.run(sql`UPDATE event_sync_sequence SET seq = seq + 1 WHERE id = 1`) + yield* db.run(sql`INSERT INTO event(id, aggregate_id, seq, type, data, sync_seq) + VALUES ('event-new', 'session-a', 3, 'test.1', '{}', (SELECT seq FROM event_sync_sequence WHERE id = 1))`) + const legacyWriter = yield* db.run(sql`INSERT INTO event(id, aggregate_id, seq, type, data) + VALUES ('event-legacy-writer', 'session-a', 4, 'test.1', '{}')`).pipe(Effect.exit) + + expect(yield* db.all(sql`SELECT sync_seq, event_id FROM event_sync_index ORDER BY sync_seq`)).toEqual([ + { sync_seq: 4, event_id: "event-new" }, + ]) + expect(legacyWriter._tag).toBe("Failure") + expect(yield* db.get(sql`SELECT id FROM event WHERE id = 'event-legacy-writer'`)).toBeUndefined() + }), + ) + }) + + test("repairs an already migrated candidate database into pending staged authority", async () => { + await run( + Effect.gen(function* () { + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + yield* db.run(sql`PRAGMA foreign_keys = ON`) + yield* db.run(sql`CREATE TABLE event_sequence ( + aggregate_id TEXT PRIMARY KEY, + seq INTEGER NOT NULL, + owner_id TEXT, + retention_floor_seq INTEGER, + snapshot_id TEXT + )`) + yield* db.run(sql`CREATE TABLE event ( + id TEXT PRIMARY KEY, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + data TEXT NOT NULL, + sync_seq INTEGER + )`) + yield* db.run(sql`CREATE TABLE event_sync_sequence ( + id INTEGER PRIMARY KEY, + seq INTEGER NOT NULL, + generation TEXT NOT NULL, + cursor_secret TEXT NOT NULL + )`) + yield* db.run(sql`INSERT INTO event_sequence VALUES ('session-a', 0, NULL, NULL, NULL)`) + yield* db.run(sql`INSERT INTO event VALUES ('event-a', 'session-a', 0, 'test.1', '{}', 1)`) + yield* db.run(sql`INSERT INTO event_sync_sequence VALUES (1, 1, lower(hex(randomblob(16))), lower(hex(randomblob(32))))`) + + yield* DatabaseMigration.applyOnly(db, [eventSyncBackfillAuthorityMigration]) + + expect(yield* db.get(sql`SELECT backfill_complete FROM event_sync_sequence`)).toEqual({ + backfill_complete: 0, + }) + expect(yield* db.get(sql`SELECT state, high_water_rowid FROM event_sync_backfill`)).toEqual({ + state: "pending", + high_water_rowid: 1, + }) + expect(yield* db.all(sql`SELECT * FROM event_sync_index`)).toEqual([]) + }), + ) + }) +}) diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index ea0f0982d..5624178fd 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -7,6 +7,8 @@ import { EventArtifactTable, EventDedupeTable, EventSequenceTable, + EventSyncIndexTable, + EventSyncSequenceTable, EventSnapshotTable, EventTable, } from "@deepagent-code/core/event/sql" @@ -904,7 +906,6 @@ describe("EventV2", () => { expectedLatest: EventV2.Cursor.make(0), codec: "test.v1", schemaVersion: 1, - body: { value: "zero" }, }).pipe(Effect.catchDefect(Effect.succeed)) expect(checkpoint).toBeInstanceOf(EventV2.InvalidSyncEventError) const compact = yield* events @@ -929,14 +930,22 @@ describe("EventV2", () => { sessionID: aggregateID, info: { summary: { diffs: [{ file: "large.patch", patch, additions: 1, deletions: 0 }] } }, } + const syncSeq = yield* db.update(EventSyncSequenceTable).set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)).returning({ seq: EventSyncSequenceTable.seq }).get().pipe(Effect.orDie) yield* db.insert(EventTable).values({ id, aggregate_id: aggregateID, seq: 1, type: EventV2.versionedType("message.updated", 1), data, + sync_seq: syncSeq!.seq, }).run().pipe(Effect.orDie) yield* db.update(EventSequenceTable).set({ seq: 1 }).where(eq(EventSequenceTable.aggregate_id, aggregateID)).run().pipe(Effect.orDie) + expect(yield* db.select().from(EventSyncIndexTable).where(eq(EventSyncIndexTable.event_id, id)).get().pipe(Effect.orDie)).toMatchObject({ + sync_seq: syncSeq!.seq, + aggregate_id: aggregateID, + seq: 1, + }) const first = yield* events.canonicalizeLegacyArtifacts({ limit: 1, now: 10 }) expect(first).toEqual({ processed: 1, next: id }) diff --git a/packages/core/test/file-part-artifact.test.ts b/packages/core/test/file-part-artifact.test.ts new file mode 100644 index 000000000..3cfb93a73 --- /dev/null +++ b/packages/core/test/file-part-artifact.test.ts @@ -0,0 +1,334 @@ +import { describe, expect } from "bun:test" +import { Context, Effect, Layer } from "effect" +import { eq, sql } from "drizzle-orm" +import { Database } from "@deepagent-code/core/database/database" +import { EventV2 } from "@deepagent-code/core/event" +import { EventSequenceTable, EventSyncSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" +import { + FilePartArtifactBindingTable, + FilePartArtifactChunkTable, + FilePartArtifactImportTable, + FilePartArtifactTable, +} from "@deepagent-code/core/file-part-artifact.sql" +import { SessionSchema } from "@deepagent-code/core/session/schema" +import { SessionV1 } from "@deepagent-code/core/v1/session" +import { Hash } from "@deepagent-code/core/util/hash" +import { AbsolutePath } from "@deepagent-code/core/schema" +import { Project } from "@deepagent-code/core/project" +import { ProjectTable } from "@deepagent-code/core/project/sql" +import { MessageTable, PartTable, SessionTable } from "@deepagent-code/core/session/sql" +import { testEffect } from "./lib/effect" + +const Prelude = EventV2.define({ + type: "test.file-part-prelude", + sync: { version: 1, aggregate: "sessionID" }, + schema: { sessionID: SessionSchema.ID }, +}) + +const runtime = () => { + const database = Database.layerFromPath(":memory:") + return Layer.mergeAll(database, EventV2.layer.pipe(Layer.provide(database))) +} + +const it = testEffect(Layer.empty) + +describe("FilePartArtifact", () => { + it.effect("externalizes one bounded legacy inline file-part batch idempotently", () => + Effect.gen(function* () { + const source = yield* Layer.buildWithScope(Layer.fresh(runtime()), yield* Effect.scope) + const events = Context.get(source, EventV2.Service) + const db = Context.get(source, Database.Service).db + const sessionID = SessionSchema.ID.make("ses_legacy_file_part") + yield* events.publish(Prelude, { sessionID }) + const eventID = EventV2.ID.make("evt_legacy_file_part") + const messageID = SessionV1.MessageID.make("msg_legacy_file_part") + const partID = SessionV1.PartID.make("prt_legacy_file_part") + const body = Buffer.alloc(3 * 1024 * 1024 + 1, 0x4c) + const data = { + sessionID, + time: 1, + part: { + id: partID, + sessionID, + messageID, + type: "file" as const, + mime: "application/octet-stream", + url: `data:application/octet-stream;base64,${body.toString("base64")}`, + }, + } + yield* db + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "legacy-file", + directory: AbsolutePath.make("/project"), + title: "legacy-file", + version: "test", + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(MessageTable) + .values({ + id: messageID, + session_id: sessionID, + time_created: 1, + data: { role: "user", time: { created: 1 } }, + } as typeof MessageTable.$inferInsert) + .run() + .pipe(Effect.orDie) + yield* db + .insert(PartTable) + .values({ + id: partID, + message_id: messageID, + session_id: sessionID, + time_created: 1, + data: { type: "file", mime: "application/octet-stream", url: data.part.url }, + } as typeof PartTable.$inferInsert) + .run() + .pipe(Effect.orDie) + expect(Buffer.byteLength(JSON.stringify(data))).toBeGreaterThan(EventV2.MAX_ENCODED_PAYLOAD_BYTES) + const sync = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) + yield* db + .insert(EventTable) + .values({ + id: eventID, + aggregate_id: sessionID, + seq: 1, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + data, + sync_seq: sync!.seq, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ seq: 1 }) + .where(eq(EventSequenceTable.aggregate_id, sessionID)) + .run() + .pipe(Effect.orDie) + + const first = yield* FilePartArtifact.canonicalizeLegacy(db, { limit: 1, now: 7 }) + expect(first).toEqual({ processed: 1, next: eventID }) + expect(yield* FilePartArtifact.canonicalizeLegacy(db, { afterID: eventID, limit: 1, now: 8 })).toEqual({ + processed: 0, + next: undefined, + }) + const immutable = yield* db.select().from(EventTable).where(eq(EventTable.id, eventID)).get().pipe(Effect.orDie) + expect(immutable!.data).toEqual(data) + const binding = yield* db + .select() + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, eventID)) + .get() + .pipe(Effect.orDie) + expect(binding!.original_data_hash).toBe(Hash.sha256(JSON.stringify(data))) + const descriptor = FilePartArtifact.descriptor(binding!.canonical_data) + expect(descriptor).toBeDefined() + if (!descriptor) return + expect(Buffer.byteLength(JSON.stringify(binding!.canonical_data))).toBeLessThan(2_000) + const projected = yield* db.select().from(PartTable).where(eq(PartTable.id, partID)).get().pipe(Effect.orDie) + expect(Buffer.byteLength(JSON.stringify(projected!.data))).toBeLessThan(1_000) + expect(projected!.data).toMatchObject({ url: `artifact:${descriptor.id}`, artifact: descriptor }) + expect( + yield* FilePartArtifact.read({ aggregateID: sessionID, descriptor }).pipe(Effect.provide(source)), + ).toEqual(body) + expect(yield* db.select().from(FilePartArtifactBindingTable).all()).toHaveLength(1) + expect(yield* db.select().from(FilePartArtifactChunkTable).all()).toHaveLength(descriptor.chunks) + + yield* events.replay({ + id: eventID, + aggregateID: sessionID, + seq: 1, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + data, + }) + expect(yield* db.select().from(EventTable).where(eq(EventTable.id, eventID)).all()).toHaveLength(1) + expect((yield* db.select().from(EventTable).where(eq(EventTable.id, eventID)).get())!.data).toEqual(data) + }), + ) + + it.effect("transfers canonical chunks between databases and fences replay until the exact body is staged", () => + Effect.gen(function* () { + const source = yield* Layer.buildWithScope(Layer.fresh(runtime()), yield* Effect.scope) + const target = yield* Layer.buildWithScope(Layer.fresh(runtime()), yield* Effect.scope) + const snapshotTarget = yield* Layer.buildWithScope(Layer.fresh(runtime()), yield* Effect.scope) + const sourceEvents = Context.get(source, EventV2.Service) + const targetEvents = Context.get(target, EventV2.Service) + const sourceDb = Context.get(source, Database.Service).db + const targetDb = Context.get(target, Database.Service).db + const snapshotDb = Context.get(snapshotTarget, Database.Service).db + const sessionID = SessionSchema.ID.make("ses_file_part_two_db") + const body = Buffer.alloc(FilePartArtifact.CHUNK_BYTES * 2 + 17, 0x5a) + const prelude = yield* sourceEvents.publish(Prelude, { sessionID }) + const published = yield* sourceEvents.publish(SessionV1.Event.PartUpdated, { + sessionID, + time: 1, + part: { + id: SessionV1.PartID.make("prt_file_part_two_db"), + sessionID, + messageID: SessionV1.MessageID.make("msg_file_part_two_db"), + type: "file", + mime: "application/octet-stream", + filename: "large.bin", + url: `data:application/octet-stream;base64,${body.toString("base64")}`, + }, + }) + const descriptor = FilePartArtifact.descriptor(published.data) + expect(descriptor).toBeDefined() + if (!descriptor) return + expect(descriptor).toMatchObject({ + id: `fpart_${Hash.sha256(body)}`, + hash: Hash.sha256(body), + bytes: body.byteLength, + chunkBytes: FilePartArtifact.CHUNK_BYTES, + chunks: 3, + }) + expect(Buffer.byteLength(JSON.stringify(published.data))).toBeLessThan(2_000) + + const binding = yield* sourceDb + .select({ seq: FilePartArtifactBindingTable.seq }) + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, published.id)) + .get() + expect(binding).toEqual({ seq: 1 }) + const metadata = yield* FilePartArtifact.metadata(sourceDb, { + eventID: published.id, + aggregateID: sessionID, + seq: 1, + artifactID: descriptor.id, + }) + const chunks = yield* Effect.forEach(metadata.chunkHashes, (hash, index) => + FilePartArtifact.chunk(sourceDb, { artifactID: descriptor.id, index, expectedHash: hash }), + ) + + yield* targetEvents.replay({ + id: prelude.id, + type: EventV2.versionedType(Prelude.type, 1), + aggregateID: sessionID, + seq: 0, + data: prelude.data, + }) + const corrupt = yield* FilePartArtifact.importChunk({ + metadata, + index: 0, + hash: metadata.chunkHashes[0]!, + data: Buffer.from("corrupt"), + }).pipe(Effect.provide(target), Effect.catchDefect(Effect.succeed)) + expect(corrupt).toBeInstanceOf(FilePartArtifact.IntegrityError) + expect(yield* targetDb.select().from(FilePartArtifactTable).all()).toEqual([]) + expect(yield* targetDb.select().from(FilePartArtifactChunkTable).all()).toEqual([]) + + expect( + yield* FilePartArtifact.importChunk({ + metadata, + index: 0, + hash: metadata.chunkHashes[0]!, + data: chunks[0]!, + }).pipe(Effect.provide(target)), + ).toBe(false) + const premature = yield* targetEvents + .replay({ + id: published.id, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + aggregateID: sessionID, + seq: 1, + data: published.data, + }) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(premature).toBeInstanceOf(FilePartArtifact.IntegrityError) + expect( + yield* targetDb.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, sessionID)).get(), + ).toMatchObject({ seq: 0 }) + expect(yield* targetDb.select().from(EventTable).where(eq(EventTable.id, published.id)).get()).toBeUndefined() + expect(yield* targetDb.select().from(FilePartArtifactBindingTable).all()).toEqual([]) + expect(yield* targetDb.select().from(FilePartArtifactImportTable).all()).toEqual([]) + + yield* Effect.forEach( + chunks.slice(1), + (chunk, offset) => + FilePartArtifact.importChunk({ + metadata, + index: offset + 1, + hash: metadata.chunkHashes[offset + 1]!, + data: chunk, + }).pipe(Effect.provide(target)), + { discard: true }, + ) + yield* FilePartArtifact.stageImport(metadata).pipe(Effect.provide(target)) + yield* targetEvents.replay({ + id: published.id, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + aggregateID: sessionID, + seq: 1, + data: published.data, + }) + yield* targetEvents.replay({ + id: published.id, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + aggregateID: sessionID, + seq: 1, + data: published.data, + }) + + expect( + yield* FilePartArtifact.read({ aggregateID: sessionID, descriptor }).pipe(Effect.provide(target)), + ).toEqual(body) + expect(yield* targetDb.select().from(FilePartArtifactBindingTable).all()).toHaveLength(1) + expect(yield* targetDb.select().from(FilePartArtifactImportTable).all()).toEqual([]) + expect(yield* targetDb.select().from(EventTable).where(eq(EventTable.id, published.id)).all()).toHaveLength(1) + + const snapshotRef = yield* FilePartArtifact.snapshotRef(sourceDb, { + aggregateID: sessionID, + partID: SessionV1.PartID.make("prt_file_part_two_db"), + descriptor, + }) + yield* Effect.forEach( + chunks, + (chunk, index) => + FilePartArtifact.importChunk({ + metadata: snapshotRef, + index, + hash: snapshotRef.chunkHashes[index]!, + data: chunk, + }).pipe(Effect.provide(snapshotTarget)), + { discard: true }, + ) + const beforeSequence = yield* FilePartArtifact.bindSnapshotRef({ + metadata: snapshotRef, + partID: SessionV1.PartID.make("prt_file_part_two_db"), + }).pipe(Effect.provide(snapshotTarget), Effect.exit) + expect(beforeSequence._tag).toBe("Failure") + expect(yield* snapshotDb.select().from(FilePartArtifactBindingTable).all()).toEqual([]) + expect(yield* snapshotDb.select().from(FilePartArtifactImportTable).all()).toHaveLength(1) + + yield* snapshotDb + .insert(EventSequenceTable) + .values({ aggregate_id: sessionID, seq: 1 }) + .run() + .pipe(Effect.orDie) + yield* FilePartArtifact.bindSnapshotRef({ + metadata: snapshotRef, + partID: SessionV1.PartID.make("prt_file_part_two_db"), + }).pipe(Effect.provide(snapshotTarget)) + expect( + yield* FilePartArtifact.read({ aggregateID: sessionID, descriptor }).pipe(Effect.provide(snapshotTarget)), + ).toEqual(body) + expect(yield* snapshotDb.select().from(FilePartArtifactImportTable).all()).toEqual([]) + }), + ) +}) diff --git a/packages/core/test/session-transfer-authority.test.ts b/packages/core/test/session-transfer-authority.test.ts new file mode 100644 index 000000000..c9f618381 --- /dev/null +++ b/packages/core/test/session-transfer-authority.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from "bun:test" +import { SqliteClient } from "@effect/sql-sqlite-bun" +import { EffectDrizzleSqlite } from "@deepagent-code/effect-drizzle-sqlite" +import sessionTransferAuthorityMigration from "@deepagent-code/core/database/migration/20260813133000_session_transfer_authority" +import { DatabaseMigration } from "@deepagent-code/core/database/migration" +import { Effect, Exit } from "effect" +import { sql } from "drizzle-orm" +import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" + +const run = (effect: Effect.Effect) => + Effect.runPromise( + effect.pipe( + Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), + Effect.scoped, + ), + ) + +describe("Session transfer authority", () => { + test("fences source input and enforces monotonic cross-database phases", async () => { + await run( + Effect.gen(function* () { + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + yield* db.run(sql`PRAGMA foreign_keys = ON`) + yield* db.run(sql`CREATE TABLE session (id TEXT PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE event_sequence ( + aggregate_id TEXT PRIMARY KEY, + seq INTEGER NOT NULL, + owner_id TEXT + )`) + yield* db.run(sql`CREATE TABLE session_input ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL + )`) + yield* db.run(sql`INSERT INTO session VALUES ('session-a')`) + yield* db.run(sql`INSERT INTO event_sequence VALUES ('session-a', 7, 'source-owner')`) + yield* DatabaseMigration.applyOnly(db, [sessionTransferAuthorityMigration]) + + yield* db.run(sql`INSERT INTO session_transfer_operation( + transfer_id, session_id, source_owner_id, target_owner_id, + source_event_seq, source_mutation_epoch, state, request_hash, created_at, updated_at + ) VALUES ( + 'transfer-a', 'session-a', 'source-owner', 'target-owner', + 7, 3, 'admitted', ${"a".repeat(64)}, 1, 1 + )`) + yield* db.run(sql`UPDATE event_sequence SET write_fence_transfer_id = 'transfer-a' + WHERE aggregate_id = 'session-a'`) + const input = yield* db + .run(sql`INSERT INTO session_input VALUES ('input-a', 'session-a')`) + .pipe(Effect.exit) + expect(Exit.isFailure(input)).toBe(true) + + yield* db.run(sql`UPDATE session_transfer_operation + SET state = 'source_frozen', snapshot_id = 'snapshot-a', snapshot_hash = ${"b".repeat(64)} + WHERE transfer_id = 'transfer-a'`) + yield* db.run(sql`UPDATE session_transfer_operation SET state = 'target_staged' + WHERE transfer_id = 'transfer-a'`) + const skipped = yield* db + .run(sql`UPDATE session_transfer_operation SET state = 'target_activated' + WHERE transfer_id = 'transfer-a'`) + .pipe(Effect.exit) + expect(Exit.isFailure(skipped)).toBe(true) + yield* db.run(sql`UPDATE session_transfer_operation SET state = 'owner_committed' + WHERE transfer_id = 'transfer-a'`) + yield* db.run(sql`UPDATE session_transfer_operation SET state = 'target_activated', completed_at = 5 + WHERE transfer_id = 'transfer-a'`) + expect(yield* db.get(sql`SELECT state, snapshot_id FROM session_transfer_operation`)).toEqual({ + state: "target_activated", + snapshot_id: "snapshot-a", + }) + }), + ) + }) +}) diff --git a/packages/deepagent-code/script/bug-407-010-release-gate.ts b/packages/deepagent-code/script/bug-407-010-release-gate.ts new file mode 100644 index 000000000..f30e51725 --- /dev/null +++ b/packages/deepagent-code/script/bug-407-010-release-gate.ts @@ -0,0 +1,241 @@ +#!/usr/bin/env bun + +import { Database } from "@deepagent-code/core/database/database" +import { EventV2 } from "@deepagent-code/core/event" +import { + EventArtifactTable, + EventCompactionReceiptTable, + EventSequenceTable, + EventSnapshotTable, + EventSyncBackfillTable, + EventSyncIndexTable, + EventTable, +} from "@deepagent-code/core/event/sql" +import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" +import { FilePartArtifactBindingTable } from "@deepagent-code/core/file-part-artifact.sql" +import { SessionProjector } from "@deepagent-code/core/session/projector" +import { SessionTable } from "@deepagent-code/core/session/sql" +import { Cause, Effect, Exit, Layer } from "effect" +import { eq, sql } from "drizzle-orm" +import path from "node:path" +import { parseArgs } from "node:util" +import { SessionDiffArtifact } from "@/session/diff-artifact" +import { SessionDiffMigrationReceiptTable } from "@/session/diff-artifact.sql" +import { SessionID } from "@/session/schema" + +const args = parseArgs({ + args: process.argv.slice(2), + options: { + database: { type: "string" }, + session: { type: "string", multiple: true }, + }, + strict: true, +}) +const filename = args.values.database ? path.resolve(args.values.database) : undefined +const sessions = (args.values.session ?? []).map((sessionID) => SessionID.make(sessionID)) + +if (!filename || !path.basename(filename).includes("release-gate")) + throw new Error("--database must name an explicit release-gate copy") +if (!(await Bun.file(filename).exists())) throw new Error(`Release-gate database does not exist: ${filename}`) +if (sessions.length === 0) throw new Error("At least one --session is required") + +const databaseLayer = Database.layerFromPath(filename) +const eventLayer = EventV2.layer.pipe(Layer.provide(databaseLayer)) +const projectorLayer = SessionProjector.layer.pipe(Layer.provide(eventLayer), Layer.provide(databaseLayer)) +const runtime = Layer.mergeAll(databaseLayer, eventLayer, projectorLayer) + +const stage = Effect.fn("Bug407010ReleaseGate.stage")(function* ( + name: string, + effect: Effect.Effect, +) { + const started = Bun.nanoseconds() + const result = yield* effect + console.log(JSON.stringify({ + stage: name, + elapsedMs: Math.round((Bun.nanoseconds() - started) / 1_000_000), + rssBytes: process.memoryUsage.rss(), + result, + })) + return result +}) + +const program = Effect.gen(function* () { + const { db } = yield* Database.Service + const events = yield* EventV2.Service + + const before = yield* db.all<{ + events: number + eventBytes: number + maximumEventBytes: number + sessions: number + }>(` + SELECT + (SELECT count(*) FROM event) AS events, + (SELECT coalesce(sum(length(data)), 0) FROM event) AS eventBytes, + (SELECT coalesce(max(length(data)), 0) FROM event) AS maximumEventBytes, + (SELECT count(*) FROM session) AS sessions + `).pipe(Effect.orDie) + console.log(JSON.stringify({ stage: "before", database: filename, ...before[0], rssBytes: process.memoryUsage.rss() })) + + yield* stage("event-index", Effect.gen(function* () { + let batches = 0 + let processed = 0 + while (true) { + const result = yield* events.backfillSyncIndex!({ limit: 5000 }) + batches += 1 + processed += result.processed + if (result.complete) return { batches, processed } + } + })) + + yield* stage("legacy-message-artifacts", Effect.gen(function* () { + let batches = 0 + let processed = 0 + let cursor: EventV2.ID | undefined + while (true) { + const result = yield* events.canonicalizeLegacyArtifacts({ ...(cursor ? { afterID: cursor } : {}), limit: 1 }) + batches += 1 + processed += result.processed + if (!result.next) return { batches, processed } + cursor = result.next + } + })) + + yield* stage("legacy-file-artifacts", Effect.gen(function* () { + let batches = 0 + let processed = 0 + let cursor: EventV2.ID | undefined + while (true) { + const result = yield* FilePartArtifact.canonicalizeLegacy(db, { ...(cursor ? { afterID: cursor } : {}), limit: 1 }) + batches += 1 + processed += result.processed + if (!result.next) return { batches, processed } + cursor = result.next + } + })) + + const artifactSessions = yield* db + .selectDistinct({ id: EventArtifactTable.aggregate_id }) + .from(EventArtifactTable) + .where(eq(EventArtifactTable.kind, "legacy_message_diff")) + .all() + .pipe(Effect.orDie) + yield* stage("legacy-session-diffs", Effect.gen(function* () { + let processed = 0 + let committed = 0 + let failed = 0 + for (const row of artifactSessions) { + while (true) { + const result = yield* SessionDiffArtifact.migrate({ sessionID: SessionID.make(row.id), limit: 1 }) + processed += result.processed + committed += result.committed + failed += result.failed + if (result.processed === 0) break + } + } + return { sessions: artifactSessions.length, processed, committed, failed } + })) + + yield* stage("checkpoint-and-compact", Effect.forEach(sessions, (sessionID) => Effect.gen(function* () { + const session = yield* db + .select({ id: SessionTable.id }) + .from(SessionTable) + .where(eq(SessionTable.id, sessionID)) + .get() + .pipe(Effect.orDie) + if (!session) return yield* Effect.die(new Error(`Session not found: ${sessionID}`)) + const sequence = yield* db + .select({ seq: EventSequenceTable.seq, ownerID: EventSequenceTable.owner_id }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, sessionID)) + .get() + .pipe(Effect.orDie) + if (!sequence) return yield* Effect.die(new Error(`Event sequence not found: ${sessionID}`)) + const current = yield* events.snapshot(sessionID) + const snapshot = current?.throughSeq === sequence.seq + ? current + : yield* Effect.gen(function* () { + let attempt = yield* events.prepareCheckpoint!({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(sequence.seq), + expectedLatest: EventV2.Cursor.make(sequence.seq), + codec: "session-projection", + schemaVersion: 1, + ...(sequence.ownerID ? { ownerID: sequence.ownerID } : {}), + }) + while (attempt.state === "prepared") + attempt = yield* events.stageCheckpoint!({ snapshotID: attempt.snapshotID }) + if (attempt.state === "complete") { + const active = yield* events.snapshot(sessionID) + if (active?.snapshotID === attempt.snapshotID) return active + return yield* Effect.die(new Error(`Completed snapshot is not active: ${attempt.snapshotID}`)) + } + return yield* events.finalizeCheckpoint!({ snapshotID: attempt.snapshotID }) + }) + let compacted = { deleted: 0, complete: false } + let deleted = 0 + let batches = 0 + while (!compacted.complete) { + compacted = yield* events.compact({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(snapshot.throughSeq), + limit: 100, + }) + deleted += compacted.deleted + batches += 1 + } + return { sessionID, snapshotID: snapshot.snapshotID, throughSeq: snapshot.throughSeq, deleted, batches } + }), { concurrency: 1 })) + + const after = yield* db.all<{ + events: number + eventBytes: number + maximumEventBytes: number + syncIndex: number + snapshots: number + eventArtifacts: number + fileArtifacts: number + diffReceipts: number + compactions: number + }>(` + SELECT + (SELECT count(*) FROM event) AS events, + (SELECT coalesce(sum(length(data)), 0) FROM event) AS eventBytes, + (SELECT coalesce(max(length(data)), 0) FROM event) AS maximumEventBytes, + (SELECT count(*) FROM event_sync_index) AS syncIndex, + (SELECT count(*) FROM event_snapshot) AS snapshots, + (SELECT count(*) FROM event_artifact) AS eventArtifacts, + (SELECT count(*) FROM file_part_artifact_binding) AS fileArtifacts, + (SELECT count(*) FROM session_diff_migration_receipt WHERE state = 'committed') AS diffReceipts, + (SELECT count(*) FROM event_compaction_receipt WHERE state = 'complete') AS compactions + `).pipe(Effect.orDie) + const backfill = yield* db.select().from(EventSyncBackfillTable).where(eq(EventSyncBackfillTable.id, 1)).get().pipe(Effect.orDie) + const quickCheck = yield* db.all<{ quick_check: string }>("PRAGMA quick_check").pipe(Effect.orDie) + const foreignKeys = yield* db.all>("PRAGMA foreign_key_check").pipe(Effect.orDie) + const indexed = yield* db.select({ count: sql`count(*)` }).from(EventSyncIndexTable).get().pipe(Effect.orDie) + const activeSnapshots = yield* db.select({ count: sql`count(*)` }).from(EventSnapshotTable).get().pipe(Effect.orDie) + const compactions = yield* db.select({ count: sql`count(*)` }).from(EventCompactionReceiptTable).get().pipe(Effect.orDie) + const diffReceipts = yield* db.select({ count: sql`count(*)` }).from(SessionDiffMigrationReceiptTable).get().pipe(Effect.orDie) + const fileArtifacts = yield* db.select({ count: sql`count(*)` }).from(FilePartArtifactBindingTable).get().pipe(Effect.orDie) + const result = { + ...after[0], + indexed: indexed?.count ?? 0, + activeSnapshots: activeSnapshots?.count ?? 0, + compactionReceipts: compactions?.count ?? 0, + diffMigrationReceipts: diffReceipts?.count ?? 0, + fileArtifactBindings: fileArtifacts?.count ?? 0, + backfill, + quickCheck: quickCheck.map((row) => row.quick_check), + foreignKeyViolations: foreignKeys.length, + rssBytes: process.memoryUsage.rss(), + } + if (backfill?.state !== "complete" || quickCheck.some((row) => row.quick_check !== "ok") || foreignKeys.length > 0) + return yield* Effect.die(new Error(`Release gate failed: ${JSON.stringify(result)}`)) + console.log(JSON.stringify({ stage: "after", ...result })) +}) + +const exit = await Effect.runPromiseExit(program.pipe(Effect.provide(runtime), Effect.scoped)) +if (Exit.isFailure(exit)) { + for (const error of Cause.prettyErrors(exit.cause)) console.error(error) + process.exit(1) +} diff --git a/packages/deepagent-code/src/control-plane/workspace.ts b/packages/deepagent-code/src/control-plane/workspace.ts index 152d6df68..9701fb481 100644 --- a/packages/deepagent-code/src/control-plane/workspace.ts +++ b/packages/deepagent-code/src/control-plane/workspace.ts @@ -32,6 +32,7 @@ import { Vcs } from "@/project/vcs" import { WorkspaceAdapterRuntime } from "./workspace-adapter-runtime" import { SessionRecoveryTransferGuard } from "@/session/recovery-transfer-guard" import { Hash } from "@deepagent-code/core/util/hash" +import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" export const Info = Schema.Struct({ ...WorkspaceInfoSchema.fields, @@ -327,6 +328,191 @@ export const layer = Layer.effect( let total = 0 let cursor: string | undefined = receipt?.cursor let reset = Boolean(cursor) + const requestJson = Effect.fn("Workspace.syncArtifactRequest")(function* ( + path: string, + body: unknown, + maxBytes: number, + ) { + const response = yield* http.execute( + HttpClientRequest.post(route(url, path), { + headers: new Headers(headers), + body: HttpBody.jsonUnsafe(body), + }), + ) + const declaredBytes = Number(response.headers["content-length"]) + if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) + return yield* new SyncHttpError({ + message: `Workspace artifact response exceeds ${maxBytes} bytes`, + status: 502, + }) + const chunks: Uint8Array[] = [] + let bytes = 0 + yield* Stream.runForEach(response.stream, (chunk) => { + bytes += chunk.byteLength + if (bytes > maxBytes) + return Effect.fail( + new SyncHttpError({ message: `Workspace artifact response exceeds ${maxBytes} bytes`, status: 502 }), + ) + chunks.push(chunk) + return Effect.void + }) + const text = Buffer.concat(chunks, bytes).toString() + if (response.status < 200 || response.status >= 300) { + return yield* new SyncHttpError({ + message: `Workspace artifact HTTP failure: ${response.status} ${text}`, + status: response.status, + body: text, + }) + } + const parsed = Option.getOrUndefined(Schema.decodeUnknownOption(Schema.UnknownFromJsonString)(text)) + if (parsed === undefined) + return yield* new SyncHttpError({ message: "Workspace artifact response is not valid JSON", status: 502 }) + return parsed + }) + const importArtifactMetadata = Effect.fn("Workspace.syncFilePartArtifactMetadata")(function* ( + metadata: FilePartArtifact.Metadata, + verified = false, + ) { + const descriptor = metadata.descriptor + const remote = verified + ? metadata + : Schema.decodeUnknownSync(FilePartArtifact.Metadata)( + yield* requestJson( + "/sync/artifact/file/metadata", + { + eventID: metadata.eventID, + aggregateID: metadata.aggregateID, + seq: metadata.seq, + artifactID: descriptor.id, + }, + 64 * 1024, + ), + ) + if (JSON.stringify(remote) !== JSON.stringify(metadata)) + return yield* new SyncHttpError({ message: "Workspace artifact metadata diverged from history", status: 409 }) + if (!(yield* FilePartArtifact.has(descriptor).pipe(Effect.provideService(Database.Service, { db })))) + for (const [index, hash] of remote.chunkHashes.entries()) { + const chunk = yield* requestJson( + "/sync/artifact/file/chunk", + { + eventID: remote.eventID, + aggregateID: remote.aggregateID, + seq: remote.seq, + artifactID: descriptor.id, + index, + hash, + }, + 512 * 1024, + ) + if (!chunk || typeof chunk !== "object" || Array.isArray(chunk)) + return yield* new SyncHttpError({ message: `Workspace artifact chunk ${index} is invalid`, status: 409 }) + const value = chunk as Record + if ( + value.artifactID !== descriptor.id || + value.index !== index || + value.hash !== hash || + typeof value.data !== "string" || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value.data) + ) + return yield* new SyncHttpError({ message: `Workspace artifact chunk ${index} is invalid`, status: 409 }) + yield* FilePartArtifact.importChunk({ + metadata: remote, + index, + hash, + data: Buffer.from(value.data, "base64"), + }).pipe(Effect.provideService(Database.Service, { db })) + } + yield* FilePartArtifact.stageImport(remote).pipe(Effect.provideService(Database.Service, { db })) + }) + const importArtifact = Effect.fn("Workspace.syncFilePartArtifact")(function* (item: HistoryEvent) { + const descriptor = FilePartArtifact.descriptor(item.data) + if (!descriptor) return + const metadata = Schema.decodeUnknownSync(FilePartArtifact.Metadata)( + yield* requestJson( + "/sync/artifact/file/metadata", + { eventID: item.id, aggregateID: item.aggregate_id, seq: item.seq, artifactID: descriptor.id }, + 64 * 1024, + ), + ) + if ( + metadata.eventID !== item.id || + metadata.aggregateID !== item.aggregate_id || + metadata.seq !== item.seq || + metadata.canonicalDataHash !== Hash.sha256(JSON.stringify(item.data)) || + JSON.stringify(metadata.descriptor) !== JSON.stringify(descriptor) + ) + return yield* new SyncHttpError({ message: "Workspace artifact metadata diverged from history", status: 409 }) + yield* importArtifactMetadata(metadata, true) + }) + const importSnapshot = Effect.fn("Workspace.syncSnapshot")(function* (item: HistoryResync) { + if (!events.snapshotRows || !events.snapshotChunks || !events.stageSnapshotRows || !events.stageSnapshotChunks) + return yield* new SyncHttpError({ message: "Workspace snapshot transfer is unavailable", status: 503 }) + const stageRows = events.stageSnapshotRows + const stageChunks = events.stageSnapshotChunks + let after = -1 + while (true) { + const page = Schema.decodeUnknownSync(SnapshotRowsPage)( + yield* requestJson( + "/sync/snapshot/rows", + { + aggregateID: item.snapshot.aggregateID, + snapshotID: item.snapshot.snapshotID, + snapshotHash: item.snapshot.snapshotHash, + after, + limit: EventV2.SNAPSHOT_TRANSFER_ROWS, + }, + 256 * 1024, + ), + ) + yield* stageRows(item.snapshot, page.rows) + yield* Effect.forEach( + page.rows, + (row) => + Effect.gen(function* () { + let chunkAfter = -1 + const body: Buffer[] = [] + while (true) { + const chunks = Schema.decodeUnknownSync(SnapshotChunksPage)( + yield* requestJson( + "/sync/snapshot/chunks", + { + aggregateID: item.snapshot.aggregateID, + snapshotID: item.snapshot.snapshotID, + snapshotHash: item.snapshot.snapshotHash, + rowHash: row.rowHash, + after: chunkAfter, + limit: 8, + }, + 4 * 1024 * 1024, + ), + ) + const decoded = chunks.chunks.map((chunk) => ({ ...chunk, data: Buffer.from(chunk.data, "base64") })) + yield* stageChunks(item.snapshot, row, decoded) + body.push(...decoded.map((chunk) => chunk.data)) + if (chunks.complete) break + const next = chunks.chunks.at(-1)?.chunkIndex + if (next === undefined || next <= chunkAfter) + return yield* new SyncHttpError({ message: "Workspace snapshot chunk cursor did not advance", status: 409 }) + chunkAfter = next + } + if (row.tableName !== "file_part_artifact_binding") return + const value = Option.getOrUndefined( + Schema.decodeUnknownOption(Schema.UnknownFromJsonString)(Buffer.concat(body, row.rowBytes).toString()), + ) + if (!value || typeof value !== "object" || Array.isArray(value) || !("metadata" in value)) + return yield* new SyncHttpError({ message: "Workspace snapshot artifact row is invalid", status: 409 }) + yield* importArtifactMetadata(Schema.decodeUnknownSync(FilePartArtifact.Metadata)(value.metadata)) + }), + { discard: true }, + ) + if (page.complete) break + const next = page.rows.at(-1)?.rowIndex + if (next === undefined || next <= after) + return yield* new SyncHttpError({ message: "Workspace snapshot row cursor did not advance", status: 409 }) + after = next + } + yield* events.importSnapshot(item.snapshot) + }) while (true) { const expectedCursor = cursor const response = yield* http.execute( @@ -375,20 +561,25 @@ export const layer = Layer.effect( yield* Effect.forEach( history, (item) => { - return events - .replay( - { - id: EventV2.ID.make(item.id), - aggregateID: item.aggregate_id, - seq: item.seq, - type: item.type, - data: item.data, - }, - { publish: true, ownerID: space.id }, - ) - .pipe( - Effect.provideService(WorkspaceRef, space.id), - ) + return Effect.gen(function* () { + if (item.kind === "resync_required") { + yield* importSnapshot(item) + return + } + yield* importArtifact(item) + yield* events + .replay( + { + id: EventV2.ID.make(item.id), + aggregateID: item.aggregate_id, + seq: item.seq, + type: item.type, + data: item.data, + }, + { publish: true, ownerID: space.id }, + ) + .pipe(Effect.provideService(WorkspaceRef, space.id)) + }) }, { discard: true }, ) @@ -872,13 +1063,39 @@ type HistoryEvent = { data: Record } +type HistoryResync = { + kind: "resync_required" + snapshot: EventV2.SerializedSnapshot +} + type HistoryEnvelope = { version: 1 - items: HistoryEvent[] + items: (HistoryEvent | HistoryResync)[] nextCursor: string complete: boolean } +const SnapshotRow = Schema.Struct({ + snapshotID: Schema.String, + rowIndex: Schema.Number, + tableName: Schema.String, + rowKey: Schema.String, + rowHash: Schema.String, + rowBytes: Schema.Number, + chunkCount: Schema.Number, + chainHash: Schema.String, +}) +const SnapshotRowsPage = Schema.Struct({ rows: Schema.Array(SnapshotRow), complete: Schema.Boolean }) +const SnapshotChunksPage = Schema.Struct({ + chunks: Schema.Array(Schema.Struct({ + rowHash: Schema.String, + chunkIndex: Schema.Number, + data: Schema.String, + chunkHash: Schema.String, + })), + complete: Schema.Boolean, +}) + function waitUntilSynced(input: { db: Database.Interface["db"] workspaceID: WorkspaceV2.ID diff --git a/packages/deepagent-code/src/effect/app-runtime.ts b/packages/deepagent-code/src/effect/app-runtime.ts index e3e91ff6a..b585524ee 100644 --- a/packages/deepagent-code/src/effect/app-runtime.ts +++ b/packages/deepagent-code/src/effect/app-runtime.ts @@ -29,6 +29,7 @@ import { SessionProcessor } from "@/session/processor" import { SessionCompaction } from "@/session/compaction" import { SessionRevert } from "@/session/revert" import { SessionSummary } from "@/session/summary" +import { SessionProjection } from "@/session/session-projector" import { SessionPrompt } from "@/session/prompt" import { GoalManager } from "@/session/goal-manager" import { Instruction } from "@/session/instruction" @@ -82,6 +83,7 @@ export const AppLayer = Layer.mergeAll( BackgroundJob.defaultLayer, RuntimeFlags.defaultLayer, EventV2Bridge.defaultLayer, + SessionProjection.defaultLayer, SessionRunState.defaultLayer, SessionProcessor.defaultLayer, SessionCompaction.defaultLayer, diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts index 387aa60ee..b39b33b93 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts @@ -36,6 +36,7 @@ import { GraphKind } from "@deepagent-code/core/context-federation/contract" import { GraphQueryStatus } from "@deepagent-code/core/context-federation/federation" import { Sensitivity } from "@deepagent-code/core/context-federation/authorization" import { SessionLegacyProviderResolution } from "@/session/legacy-provider-resolution" +import { File as DiffArtifactFile, Limits as DiffArtifactLimits, Manifest as DiffArtifactManifest } from "@/session/diff-artifact-schema" const root = "/session" export const ListQuery = Schema.Struct({ @@ -62,6 +63,41 @@ export const MessagesQuery = Schema.Struct({ ), before: Schema.optional(Schema.String), }) +export const DiffArtifactMaintenancePayload = Schema.Struct({ + limit: Schema.optional( + Schema.Number.check( + Schema.isInt(), + Schema.isGreaterThanOrEqualTo(1), + Schema.isLessThanOrEqualTo(DiffArtifactLimits.batch), + ), + ), +}) +export const DiffArtifactManifestQuery = Schema.Struct({ + ...WorkspaceRoutingQueryFields, + messageID: MessageID, + artifactID: Schema.String, + cursor: Schema.optional(Schema.String.check(Schema.isMaxLength(512))), + limit: Schema.optional( + Schema.NumberFromString.check( + Schema.isInt(), + Schema.isGreaterThanOrEqualTo(1), + Schema.isLessThanOrEqualTo(DiffArtifactLimits.manifestFiles), + ), + ), +}) +export const DiffArtifactFileQuery = Schema.Struct({ + ...WorkspaceRoutingQueryFields, + messageID: MessageID, + artifactID: Schema.String, + path: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(4096)), + maxBytes: Schema.optional( + Schema.NumberFromString.check( + Schema.isInt(), + Schema.isGreaterThanOrEqualTo(1), + Schema.isLessThanOrEqualTo(DiffArtifactLimits.patchBytes), + ), + ), +}) export const StatusMap = Schema.Record(Schema.String, SessionStatus.Info) export const UpdatePayload = Schema.Struct({ title: Schema.optional(Schema.String), @@ -293,6 +329,9 @@ export const SessionPaths = { todo: `${root}/:sessionID/todo`, plan: `${root}/:sessionID/plan`, diff: `${root}/:sessionID/diff`, + diffArtifactMaintenance: `${root}/:sessionID/diff-artifact/maintenance`, + diffArtifactManifest: `${root}/:sessionID/diff-artifact/manifest`, + diffArtifactFile: `${root}/:sessionID/diff-artifact/file`, messages: `${root}/:sessionID/message`, message: `${root}/:sessionID/message/:messageID`, create: root, @@ -405,6 +444,53 @@ export const SessionApi = HttpApi.make("session") description: "Get the file changes (diff) that resulted from a specific user message in the session.", }), ), + HttpApiEndpoint.post("diffArtifactMaintenance", SessionPaths.diffArtifactMaintenance, { + params: { sessionID: SessionID }, + query: WorkspaceRoutingQuery, + payload: DiffArtifactMaintenancePayload, + success: described( + Schema.Struct({ + processed: Schema.Number, + committed: Schema.Number, + failed: Schema.Number, + }), + "Legacy Session diff migration batch result", + ), + error: [HttpApiError.BadRequest, ApiNotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "session.diffArtifactMaintenance", + summary: "Migrate one bounded legacy Session diff batch", + description: + "Build authorized per-file artifact indexes, verify PromptEpoch history hashes, and CAS-rewrite user messages without publishing giant events.", + exclude: true, + }), + ), + HttpApiEndpoint.get("diffArtifactManifest", SessionPaths.diffArtifactManifest, { + params: { sessionID: SessionID }, + query: DiffArtifactManifestQuery, + success: described(DiffArtifactManifest, "Bounded Session diff artifact manifest page"), + error: [HttpApiError.BadRequest, ApiNotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "session.diffArtifactManifest", + summary: "Get a Session diff artifact manifest page", + description: "Read only metadata for an artifact committed to the addressed Session message.", + }), + ), + HttpApiEndpoint.get("diffArtifactFile", SessionPaths.diffArtifactFile, { + params: { sessionID: SessionID }, + query: DiffArtifactFileQuery, + success: described(DiffArtifactFile, "Bounded Session diff artifact file patch"), + error: [HttpApiError.BadRequest, ApiNotFoundError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "session.diffArtifactFile", + summary: "Get one bounded Session diff artifact file patch", + description: + "Verify content-addressed chunks and return at most the requested UTF-8 byte budget for one authorized path.", + }), + ), HttpApiEndpoint.get("messages", SessionPaths.messages, { params: { sessionID: SessionID }, query: MessagesQuery, diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts index 1ab71a58e..65edcfc89 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/sync.ts @@ -9,6 +9,7 @@ import { WorkspaceRoutingMiddleware, WorkspaceRoutingQuery } from "../middleware import { described } from "./metadata" import { ConflictError, ServiceUnavailableError } from "../errors" import { SyncReplayLimits } from "@/sync/replay-protocol" +import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" const root = "/sync" export { SyncReplayLimits } from "@/sync/replay-protocol" @@ -49,9 +50,25 @@ export const HistoryEvent = Schema.Struct({ type: Schema.String, data: Schema.Record(Schema.String, Schema.Unknown), }) +export const SnapshotDescriptor = Schema.Struct({ + snapshotID: Schema.String, + aggregateID: Schema.String, + throughSeq: NonNegativeInt, + syncSeq: NonNegativeInt, + codec: Schema.String, + schemaVersion: NonNegativeInt, + snapshotHash: Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)), + body: Schema.Record(Schema.String, Schema.Unknown), + ownerID: Schema.optional(Schema.String), + createdAt: NonNegativeInt, +}) +export const HistoryResync = Schema.Struct({ + kind: Schema.Literal("resync_required"), + snapshot: SnapshotDescriptor, +}) export const HistoryEnvelope = Schema.Struct({ version: Schema.Literal(1), - items: Schema.Array(HistoryEvent), + items: Schema.Array(Schema.Union([HistoryEvent, HistoryResync])), nextCursor: Schema.String, complete: Schema.Boolean, }) @@ -63,6 +80,96 @@ export const ArtifactMaintenanceResponse = Schema.Struct({ processed: NonNegativeInt, nextCursor: Schema.optional(EventV2.ID), }) +export const EventIndexMaintenancePayload = Schema.Struct({ + limit: Schema.optional(NonNegativeInt), +}) +export const EventIndexMaintenanceResponse = Schema.Struct({ + processed: NonNegativeInt, + complete: Schema.Boolean, +}) +export const FileArtifactMetadataPayload = Schema.Struct({ + eventID: EventV2.ID, + aggregateID: Schema.String, + seq: NonNegativeInt, + artifactID: FilePartArtifact.ID, +}) +export const FileArtifactChunkPayload = Schema.Struct({ + eventID: EventV2.ID, + aggregateID: Schema.String, + seq: NonNegativeInt, + artifactID: FilePartArtifact.ID, + index: NonNegativeInt, + hash: Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)), +}) +export const FileArtifactChunkResponse = Schema.Struct({ + artifactID: FilePartArtifact.ID, + index: NonNegativeInt, + hash: Schema.String, + data: Schema.String, +}) +export const SnapshotRowsPayload = Schema.Struct({ + aggregateID: Schema.String, + snapshotID: Schema.String, + snapshotHash: Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)), + after: Schema.optional(Schema.Number), + limit: Schema.optional(NonNegativeInt), +}) +export const SnapshotRow = Schema.Struct({ + snapshotID: Schema.String, + rowIndex: NonNegativeInt, + tableName: Schema.String, + rowKey: Schema.String, + rowHash: Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)), + rowBytes: NonNegativeInt, + chunkCount: NonNegativeInt, + chainHash: Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)), +}) +export const SnapshotRowsResponse = Schema.Struct({ rows: Schema.Array(SnapshotRow), complete: Schema.Boolean }) +export const SnapshotChunksPayload = Schema.Struct({ + aggregateID: Schema.String, + snapshotID: Schema.String, + snapshotHash: Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)), + rowHash: Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)), + after: Schema.optional(Schema.Number), + limit: Schema.optional(NonNegativeInt), +}) +export const SnapshotChunk = Schema.Struct({ + rowHash: Schema.String, + chunkIndex: NonNegativeInt, + data: Schema.String, + chunkHash: Schema.String, +}) +export const SnapshotChunksResponse = Schema.Struct({ chunks: Schema.Array(SnapshotChunk), complete: Schema.Boolean }) +export const CheckpointPreparePayload = Schema.Struct({ aggregateID: SessionID }) +export const CheckpointStagePayload = Schema.Struct({ + snapshotID: Schema.String, + limit: Schema.optional(NonNegativeInt), +}) +export const CheckpointFinalizePayload = Schema.Struct({ snapshotID: Schema.String }) +export const CheckpointDiscardPayload = Schema.Struct({ + snapshotID: Schema.String, + limit: Schema.optional(NonNegativeInt), +}) +export const CheckpointCompactPayload = Schema.Struct({ + aggregateID: SessionID, + limit: Schema.optional(NonNegativeInt), +}) +export const CheckpointAttempt = Schema.Struct({ + snapshotID: Schema.String, + aggregateID: Schema.String, + throughSeq: NonNegativeInt, + expectedLatest: NonNegativeInt, + ownerID: Schema.optional(Schema.String), + codec: Schema.String, + schemaVersion: NonNegativeInt, + cursor: Schema.optional(Schema.String), + rowCount: NonNegativeInt, + encodedBytes: NonNegativeInt, + state: Schema.Literals(["prepared", "staged", "complete"]), + hasMore: Schema.Boolean, +}) +export const CheckpointDiscardResponse = Schema.Struct({ deletedRows: NonNegativeInt, complete: Schema.Boolean }) +export const CheckpointCompactResponse = Schema.Struct({ deleted: NonNegativeInt, complete: Schema.Boolean }) export const SyncPaths = { start: `${root}/start`, @@ -70,6 +177,17 @@ export const SyncPaths = { steal: `${root}/steal`, history: `${root}/history`, artifacts: `${root}/maintenance/artifacts`, + fileArtifacts: `${root}/maintenance/file-artifacts`, + eventIndex: `${root}/maintenance/event-index`, + fileArtifactMetadata: `${root}/artifact/file/metadata`, + fileArtifactChunk: `${root}/artifact/file/chunk`, + snapshotRows: `${root}/snapshot/rows`, + snapshotChunks: `${root}/snapshot/chunks`, + checkpointPrepare: `${root}/maintenance/checkpoint/prepare`, + checkpointStage: `${root}/maintenance/checkpoint/stage`, + checkpointFinalize: `${root}/maintenance/checkpoint/finalize`, + checkpointDiscard: `${root}/maintenance/checkpoint/discard`, + checkpointCompact: `${root}/maintenance/checkpoint/compact`, } as const export const SyncApi = HttpApi.make("sync") @@ -139,6 +257,98 @@ export const SyncApi = HttpApi.make("sync") exclude: true, }), ), + HttpApiEndpoint.post("eventIndex", SyncPaths.eventIndex, { + query: WorkspaceRoutingQuery, + payload: EventIndexMaintenancePayload, + success: described(EventIndexMaintenanceResponse, "Advanced one bounded EventV2 sync-index backfill batch"), + error: [HttpApiError.BadRequest, ServiceUnavailableError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "sync.maintenance.event-index", + summary: "Backfill a bounded EventV2 sync-index batch", + description: "Run one authenticated, idempotent maintenance batch without starting an automatic table scan.", + exclude: true, + }), + ), + HttpApiEndpoint.post("fileArtifacts", SyncPaths.fileArtifacts, { + query: WorkspaceRoutingQuery, + payload: ArtifactMaintenancePayload, + success: described(ArtifactMaintenanceResponse, "Externalized legacy oversized file-part artifacts"), + error: HttpApiError.BadRequest, + }).annotateMerge( + OpenApi.annotations({ + identifier: "sync.maintenance.file-artifacts", + summary: "Externalize a bounded legacy file-part batch", + description: "Run one authenticated, idempotent file-part artifact maintenance batch.", + exclude: true, + }), + ), + HttpApiEndpoint.post("fileArtifactMetadata", SyncPaths.fileArtifactMetadata, { + query: WorkspaceRoutingQuery, + payload: FileArtifactMetadataPayload, + success: described(FilePartArtifact.Metadata, "Scoped file-part artifact metadata"), + error: [HttpApiError.BadRequest, HttpApiError.NotFound], + }).annotateMerge( + OpenApi.annotations({ + identifier: "sync.artifact.file.metadata", + summary: "Read scoped file-part artifact metadata", + description: "Resolve a content-addressed file-part descriptor only through its exact workspace event binding.", + }), + ), + HttpApiEndpoint.post("fileArtifactChunk", SyncPaths.fileArtifactChunk, { + query: WorkspaceRoutingQuery, + payload: FileArtifactChunkPayload, + success: described(FileArtifactChunkResponse, "One verified file-part artifact chunk"), + error: [HttpApiError.BadRequest, HttpApiError.NotFound], + }).annotateMerge( + OpenApi.annotations({ + identifier: "sync.artifact.file.chunk", + summary: "Read one verified file-part artifact chunk", + description: "Return one bounded base64 chunk after workspace scope and expected hash validation.", + }), + ), + HttpApiEndpoint.post("snapshotRows", SyncPaths.snapshotRows, { + query: WorkspaceRoutingQuery, + payload: SnapshotRowsPayload, + success: described(SnapshotRowsResponse, "One bounded scoped snapshot row page"), + error: [HttpApiError.BadRequest, HttpApiError.NotFound, ServiceUnavailableError], + }), + HttpApiEndpoint.post("snapshotChunks", SyncPaths.snapshotChunks, { + query: WorkspaceRoutingQuery, + payload: SnapshotChunksPayload, + success: described(SnapshotChunksResponse, "One bounded scoped snapshot chunk page"), + error: [HttpApiError.BadRequest, HttpApiError.NotFound, ServiceUnavailableError], + }), + HttpApiEndpoint.post("checkpointPrepare", SyncPaths.checkpointPrepare, { + query: WorkspaceRoutingQuery, + payload: CheckpointPreparePayload, + success: described(CheckpointAttempt, "Prepared bounded canonical checkpoint"), + error: [HttpApiError.BadRequest, HttpApiError.NotFound, ConflictError, ServiceUnavailableError], + }).annotateMerge(OpenApi.annotations({ identifier: "sync.maintenance.checkpoint.prepare", summary: "Prepare a canonical checkpoint", exclude: true })), + HttpApiEndpoint.post("checkpointStage", SyncPaths.checkpointStage, { + query: WorkspaceRoutingQuery, + payload: CheckpointStagePayload, + success: described(CheckpointAttempt, "Staged one bounded canonical checkpoint batch"), + error: [HttpApiError.BadRequest, HttpApiError.NotFound, ConflictError, ServiceUnavailableError], + }).annotateMerge(OpenApi.annotations({ identifier: "sync.maintenance.checkpoint.stage", summary: "Stage one bounded checkpoint batch", exclude: true })), + HttpApiEndpoint.post("checkpointFinalize", SyncPaths.checkpointFinalize, { + query: WorkspaceRoutingQuery, + payload: CheckpointFinalizePayload, + success: described(SnapshotDescriptor, "Finalized active canonical checkpoint"), + error: [HttpApiError.BadRequest, HttpApiError.NotFound, ConflictError, ServiceUnavailableError], + }).annotateMerge(OpenApi.annotations({ identifier: "sync.maintenance.checkpoint.finalize", summary: "Finalize a canonical checkpoint", exclude: true })), + HttpApiEndpoint.post("checkpointDiscard", SyncPaths.checkpointDiscard, { + query: WorkspaceRoutingQuery, + payload: CheckpointDiscardPayload, + success: described(CheckpointDiscardResponse, "Discarded one bounded prepared checkpoint batch"), + error: [HttpApiError.BadRequest, HttpApiError.NotFound, ConflictError, ServiceUnavailableError], + }).annotateMerge(OpenApi.annotations({ identifier: "sync.maintenance.checkpoint.discard", summary: "Discard one prepared checkpoint batch", exclude: true })), + HttpApiEndpoint.post("checkpointCompact", SyncPaths.checkpointCompact, { + query: WorkspaceRoutingQuery, + payload: CheckpointCompactPayload, + success: described(CheckpointCompactResponse, "Compacted one bounded retained EventV2 batch"), + error: [HttpApiError.BadRequest, HttpApiError.NotFound, ConflictError, ServiceUnavailableError], + }).annotateMerge(OpenApi.annotations({ identifier: "sync.maintenance.checkpoint.compact", summary: "Compact one active checkpoint event batch", exclude: true })), ) .annotateMerge( OpenApi.annotations({ diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts index b338ce211..d137cf231 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts @@ -31,6 +31,9 @@ import { InstanceHttpApi } from "../api" import { CommandPayload, ContextAttemptResolvePayload, + DiffArtifactFileQuery, + DiffArtifactMaintenancePayload, + DiffArtifactManifestQuery, DiffQuery, ForkPayload, InitPayload, @@ -50,6 +53,7 @@ import { ConflictError, PermissionNotFoundError, notFound } from "../errors" import * as SessionError from "./session-errors" import { randomUUID } from "node:crypto" import { getWorkspaceContext } from "../utils/workspace-context" +import { SessionDiffArtifact } from "@/session/diff-artifact" const tryParseJson = (text: string) => Effect.try({ @@ -154,6 +158,48 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", return yield* summary.diff({ sessionID: ctx.params.sessionID, messageID: ctx.query.messageID }) }) + const mapDiffArtifactError = (error: SessionDiffArtifact.Invalid | SessionDiffArtifact.NotFound) => + error instanceof SessionDiffArtifact.NotFound ? notFound(error.message) : new HttpApiError.BadRequest({}) + + const diffArtifactMaintenance = Effect.fn("SessionHttpApi.diffArtifactMaintenance")(function* (ctx: { + params: { sessionID: SessionID } + payload: typeof DiffArtifactMaintenancePayload.Type + }) { + yield* requireSession(ctx.params.sessionID) + return yield* SessionDiffArtifact.migrate({ + sessionID: ctx.params.sessionID, + ...(ctx.payload.limit ? { limit: ctx.payload.limit } : {}), + }).pipe(Effect.mapError(mapDiffArtifactError)) + }) + + const diffArtifactManifest = Effect.fn("SessionHttpApi.diffArtifactManifest")(function* (ctx: { + params: { sessionID: SessionID } + query: typeof DiffArtifactManifestQuery.Type + }) { + yield* requireSession(ctx.params.sessionID) + return yield* SessionDiffArtifact.manifest({ + sessionID: ctx.params.sessionID, + messageID: ctx.query.messageID, + artifactID: ctx.query.artifactID, + ...(ctx.query.cursor ? { cursor: ctx.query.cursor } : {}), + ...(ctx.query.limit ? { limit: ctx.query.limit } : {}), + }).pipe(Effect.mapError(mapDiffArtifactError)) + }) + + const diffArtifactFile = Effect.fn("SessionHttpApi.diffArtifactFile")(function* (ctx: { + params: { sessionID: SessionID } + query: typeof DiffArtifactFileQuery.Type + }) { + yield* requireSession(ctx.params.sessionID) + return yield* SessionDiffArtifact.file({ + sessionID: ctx.params.sessionID, + messageID: ctx.query.messageID, + artifactID: ctx.query.artifactID, + path: ctx.query.path, + ...(ctx.query.maxBytes ? { maxBytes: ctx.query.maxBytes } : {}), + }).pipe(Effect.mapError(mapDiffArtifactError)) + }) + const messages = Effect.fn("SessionHttpApi.messages")(function* (ctx: { params: { sessionID: SessionID } query: typeof MessagesQuery.Type @@ -703,6 +749,9 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", .handle("todo", todo) .handle("plan", plan) .handle("diff", diff) + .handle("diffArtifactMaintenance", diffArtifactMaintenance) + .handle("diffArtifactManifest", diffArtifactManifest) + .handle("diffArtifactFile", diffArtifactFile) .handle("messages", messages) .handle("message", message) .handleRaw("create", createRaw) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts index 6fea763b0..4f1e845a3 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts @@ -8,6 +8,10 @@ import { EventV2Bridge } from "@/event-v2-bridge" import { EventArtifactTable, EventSequenceTable, + EventSnapshotAttemptTable, + EventSnapshotRowTable, + EventSnapshotTable, + EventSyncIndexTable, EventSyncSequenceTable, EventTable, } from "@deepagent-code/core/event/sql" @@ -23,7 +27,22 @@ import { sql } from "drizzle-orm" import { Effect, Option, Schema, Scope } from "effect" import { HttpApiBuilder, HttpApiError } from "effect/unstable/httpapi" import { InstanceHttpApi } from "../api" -import { ArtifactMaintenancePayload, HistoryPayload, ReplayPayload, SessionPayload } from "../groups/sync" +import { + ArtifactMaintenancePayload, + CheckpointCompactPayload, + CheckpointDiscardPayload, + CheckpointFinalizePayload, + CheckpointPreparePayload, + CheckpointStagePayload, + EventIndexMaintenancePayload, + FileArtifactChunkPayload, + FileArtifactMetadataPayload, + HistoryPayload, + ReplayPayload, + SessionPayload, + SnapshotChunksPayload, + SnapshotRowsPayload, +} from "../groups/sync" import { encodeReplayRequestPrefix } from "@/sync/replay-protocol" import * as Log from "@deepagent-code/core/util/log" import { ServiceUnavailableError } from "../errors" @@ -35,6 +54,8 @@ import { SessionID } from "@/session/schema" import { Flag } from "@deepagent-code/core/flag/flag" import { WorkspaceAdapterRuntime } from "@/control-plane/workspace-adapter-runtime" import { createHmac, timingSafeEqual } from "node:crypto" +import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" +import { FilePartArtifactBindingTable } from "@deepagent-code/core/file-part-artifact.sql" const log = Log.create({ service: "server.sync" }) export const SyncHistoryLimits = { @@ -69,7 +90,7 @@ const historyCursor = (scope: string, generation: string, secret: string, highWa const decodeJson = Schema.decodeUnknownOption(Schema.UnknownFromJsonString) const record = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value) -const projectedEventData = sql`COALESCE(${EventArtifactTable.canonical_data}, CASE +const projectedEventData = sql`COALESCE(${FilePartArtifactBindingTable.canonical_data}, ${EventArtifactTable.canonical_data}, CASE WHEN ${EventTable.type} = ${EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1)} AND json_valid(${EventTable.data}) AND json_type(${EventTable.data}, '$.info.summary.diffs') = 'array' @@ -249,7 +270,12 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl const workspaceID = yield* InstanceState.workspaceID if (!workspaceID) return yield* new HttpApiError.BadRequest({}) const authority = yield* db - .select({ generation: EventSyncSequenceTable.generation, secret: EventSyncSequenceTable.cursor_secret, seq: EventSyncSequenceTable.seq }) + .select({ + generation: EventSyncSequenceTable.generation, + secret: EventSyncSequenceTable.cursor_secret, + seq: EventSyncSequenceTable.seq, + complete: EventSyncSequenceTable.backfill_complete, + }) .from(EventSyncSequenceTable) .where(eq(EventSyncSequenceTable.id, 1)) .get() @@ -259,6 +285,11 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl service: "sync.history", message: "Sync cursor authority is unavailable", }) + if (!authority.complete) + return yield* new ServiceUnavailableError({ + service: "event_sync_backfill_required", + message: "Event sync index backfill is incomplete; run bounded maintenance before reading history", + }) const envelope = "version" in ctx.payload ? ctx.payload : { version: 1 as const, known: ctx.payload } const known = envelope.known ?? {} if (Object.keys(known).length > SyncHistoryLimits.legacyKnownAggregates) @@ -270,22 +301,58 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl resource: `sync-cursor-reset:${workspaceID}`, }) const knownJson = JSON.stringify(known) + const resync = yield* db + .select({ + snapshotID: EventSnapshotTable.snapshot_id, + aggregateID: EventSnapshotTable.aggregate_id, + throughSeq: EventSnapshotTable.through_seq, + syncSeq: EventSnapshotTable.sync_seq, + codec: EventSnapshotTable.codec, + schemaVersion: EventSnapshotTable.schema_version, + snapshotHash: EventSnapshotTable.snapshot_hash, + body: EventSnapshotTable.body, + ownerID: EventSnapshotTable.owner_id, + createdAt: EventSnapshotTable.created_at, + }) + .from(EventSnapshotTable) + .innerJoin(EventSequenceTable, eq(EventSequenceTable.snapshot_id, EventSnapshotTable.snapshot_id)) + .innerJoin(SessionTable, eq(SessionTable.id, EventSnapshotTable.aggregate_id)) + .where( + and( + eq(SessionTable.workspace_id, workspaceID), + gt(EventSnapshotTable.sync_seq, after), + sql`COALESCE(( + SELECT CAST(value AS INTEGER) + FROM json_each(${knownJson}) + WHERE key = ${EventSnapshotTable.aggregate_id} + ), -1) < ${EventSnapshotTable.through_seq}`, + ), + ) + .orderBy(asc(EventSnapshotTable.sync_seq)) + .limit(1) + .get() + .pipe(Effect.orDie) const candidates = yield* db .all<{ kind: "event"; id: string; syncSeq: number; bytes: number }>(sql` SELECT 'event' AS kind, ${EventTable.id} AS id, - ${EventTable.sync_seq} AS syncSeq, - length(CAST(COALESCE(${EventArtifactTable.canonical_data}, ${EventTable.data}) AS BLOB)) AS bytes - FROM ${EventTable} + ${EventSyncIndexTable.sync_seq} AS syncSeq, + length(CAST(COALESCE(${FilePartArtifactBindingTable.canonical_data}, ${EventArtifactTable.canonical_data}, ${EventTable.data}) AS BLOB)) AS bytes + FROM ${EventSyncIndexTable} + INNER JOIN ${EventTable} + ON ${EventTable.id} = ${EventSyncIndexTable.event_id} INNER JOIN ${EventSequenceTable} ON ${EventSequenceTable.aggregate_id} = ${EventTable.aggregate_id} INNER JOIN ${SessionTable} ON ${SessionTable.id} = ${EventTable.aggregate_id} LEFT JOIN ${EventArtifactTable} ON ${EventArtifactTable.event_id} = ${EventTable.id} - WHERE ${EventTable.sync_seq} > ${after} - AND ${EventTable.sync_seq} <= ${authority.seq} + LEFT JOIN ${FilePartArtifactBindingTable} + ON ${FilePartArtifactBindingTable.event_id} = ${EventTable.id} + WHERE ${EventSyncIndexTable.sync_seq} > ${after} + AND ${EventSyncIndexTable.sync_seq} <= ${authority.seq} + ${resync ? sql`AND ${EventSyncIndexTable.sync_seq} < ${resync.syncSeq}` : sql``} AND ${SessionTable.workspace_id} = ${workspaceID} AND ${EventTable.seq} > COALESCE(${EventSequenceTable.retention_floor_seq}, -1) AND ${EventTable.seq} > COALESCE(( @@ -293,7 +360,7 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl FROM json_each(${knownJson}) WHERE key = ${EventTable.aggregate_id} ), -1) - ORDER BY ${EventTable.sync_seq} ASC + ORDER BY ${EventSyncIndexTable.sync_seq} ASC LIMIT ${SyncHistoryLimits.events + 1} `) .pipe(Effect.orDie) @@ -326,6 +393,27 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl service: "sync.history", message: `Legacy sync event ${page.oversized} exceeds the bounded history page and requires artifact migration`, }) + if (page.rows.length === 0 && resync) + return { + version: 1 as const, + items: [{ + kind: "resync_required" as const, + snapshot: { + snapshotID: resync.snapshotID, + aggregateID: resync.aggregateID, + throughSeq: resync.throughSeq, + syncSeq: resync.syncSeq, + codec: resync.codec, + schemaVersion: resync.schemaVersion, + snapshotHash: resync.snapshotHash, + body: resync.body, + ...(resync.ownerID ? { ownerID: resync.ownerID } : {}), + createdAt: resync.createdAt, + }, + }], + nextCursor: encodeHistoryCursor(workspaceID, authority.generation, authority.secret, resync.syncSeq), + complete: false, + } if (page.rows.length === 0) return { version: 1 as const, items: [], nextCursor: encodeHistoryCursor(workspaceID, authority.generation, authority.secret, authority.seq), complete: true } const selectedRows = page.rows.slice(0, SyncHistoryLimits.events) @@ -340,6 +428,7 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl }) .from(EventTable) .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) + .leftJoin(FilePartArtifactBindingTable, eq(FilePartArtifactBindingTable.event_id, EventTable.id)) .where(inArray(EventTable.id, eventIDs)) .all() .pipe(Effect.orDie) @@ -370,7 +459,7 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl service: "sync.history", message: `Projected sync item ${page.rows[0]!.id} exceeds the bounded history page`, }) - const complete = candidates.length <= selectedRows.length && bounded.items.length === selectedRows.length + const complete = !resync && candidates.length <= selectedRows.length && bounded.items.length === selectedRows.length const next = complete ? authority.seq : selectedRows[bounded.items.length - 1]?.syncSeq ?? after return { version: 1 as const, @@ -392,11 +481,287 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl return { processed: result.processed, ...(result.next ? { nextCursor: result.next } : {}) } }) + const eventIndex = Effect.fn("SyncHttpApi.eventIndex")(function* (ctx: { + payload: typeof EventIndexMaintenancePayload.Type + }) { + if (ctx.payload.limit !== undefined && (ctx.payload.limit < 1 || ctx.payload.limit > 1000)) + return yield* new HttpApiError.BadRequest({}) + if (!events.backfillSyncIndex) return yield* new ServiceUnavailableError({ + service: "event_sync_backfill_required", + message: "Event sync-index maintenance is unavailable in this runtime", + }) + return yield* events.backfillSyncIndex({ ...(ctx.payload.limit ? { limit: ctx.payload.limit } : {}) }) + }) + + const fileArtifacts = Effect.fn("SyncHttpApi.fileArtifacts")(function* (ctx: { + payload: typeof ArtifactMaintenancePayload.Type + }) { + if (ctx.payload.limit !== undefined && ctx.payload.limit !== EventV2.LEGACY_ARTIFACT_BATCH_EVENTS) + return yield* new HttpApiError.BadRequest({}) + const result = yield* FilePartArtifact.canonicalizeLegacy(db, { + ...(ctx.payload.cursor ? { afterID: ctx.payload.cursor } : {}), + ...(ctx.payload.limit ? { limit: ctx.payload.limit } : {}), + }) + return { processed: result.processed, ...(result.next ? { nextCursor: result.next } : {}) } + }) + + const fileArtifactMetadata = Effect.fn("SyncHttpApi.fileArtifactMetadata")(function* (ctx: { + payload: typeof FileArtifactMetadataPayload.Type + }) { + const workspaceID = yield* InstanceState.workspaceID + if (!workspaceID) return yield* new HttpApiError.BadRequest({}) + const scoped = yield* db + .select({ id: SessionTable.id }) + .from(SessionTable) + .where(and(eq(SessionTable.id, SessionID.make(ctx.payload.aggregateID)), eq(SessionTable.workspace_id, workspaceID))) + .get() + .pipe(Effect.orDie) + if (!scoped) return yield* new HttpApiError.NotFound() + return yield* FilePartArtifact.metadata(db, ctx.payload).pipe( + Effect.catchDefect((defect) => + Schema.is(FilePartArtifact.IntegrityError)(defect) + ? Effect.fail(new HttpApiError.NotFound()) + : Effect.die(defect), + ), + ) + }) + + const fileArtifactChunk = Effect.fn("SyncHttpApi.fileArtifactChunk")(function* (ctx: { + payload: typeof FileArtifactChunkPayload.Type + }) { + const workspaceID = yield* InstanceState.workspaceID + if (!workspaceID) return yield* new HttpApiError.BadRequest({}) + const scoped = yield* db + .select({ id: SessionTable.id }) + .from(SessionTable) + .where( + and( + eq(SessionTable.workspace_id, workspaceID), + eq(SessionTable.id, SessionID.make(ctx.payload.aggregateID)), + ), + ) + .get() + .pipe(Effect.orDie) + if (!scoped) return yield* new HttpApiError.NotFound() + const metadata = yield* FilePartArtifact.metadata(db, ctx.payload).pipe( + Effect.catchDefect((defect) => + Schema.is(FilePartArtifact.IntegrityError)(defect) + ? Effect.fail(new HttpApiError.NotFound()) + : Effect.die(defect), + ), + ) + if (metadata.chunkHashes[ctx.payload.index] !== ctx.payload.hash) + return yield* new HttpApiError.NotFound() + const data = yield* FilePartArtifact.chunk(db, { + artifactID: ctx.payload.artifactID, + index: ctx.payload.index, + expectedHash: ctx.payload.hash, + }).pipe( + Effect.catchDefect((defect) => + Schema.is(FilePartArtifact.IntegrityError)(defect) + ? Effect.fail(new HttpApiError.NotFound()) + : Effect.die(defect), + ), + ) + return { + artifactID: ctx.payload.artifactID, + index: ctx.payload.index, + hash: ctx.payload.hash, + data: data.toString("base64"), + } + }) + + const snapshotRows = Effect.fn("SyncHttpApi.snapshotRows")(function* (ctx: { + payload: typeof SnapshotRowsPayload.Type + }) { + if (ctx.payload.limit !== undefined && (ctx.payload.limit < 1 || ctx.payload.limit > EventV2.SNAPSHOT_TRANSFER_ROWS)) + return yield* new HttpApiError.BadRequest({}) + const workspaceID = yield* InstanceState.workspaceID + if (!workspaceID || !events.snapshotRows) return yield* new ServiceUnavailableError({ service: "sync.snapshot", message: "Snapshot row transfer is unavailable" }) + const scoped = yield* db + .select({ id: EventSnapshotTable.snapshot_id }) + .from(EventSnapshotTable) + .innerJoin(EventSequenceTable, eq(EventSequenceTable.snapshot_id, EventSnapshotTable.snapshot_id)) + .innerJoin(SessionTable, eq(SessionTable.id, EventSnapshotTable.aggregate_id)) + .where(and( + eq(SessionTable.workspace_id, workspaceID), + eq(EventSnapshotTable.aggregate_id, ctx.payload.aggregateID), + eq(EventSnapshotTable.snapshot_id, ctx.payload.snapshotID), + eq(EventSnapshotTable.snapshot_hash, ctx.payload.snapshotHash), + )) + .get() + .pipe(Effect.orDie) + if (!scoped) return yield* new HttpApiError.NotFound() + const rows = yield* events.snapshotRows({ + snapshotID: ctx.payload.snapshotID, + ...(ctx.payload.after !== undefined ? { after: ctx.payload.after } : {}), + ...(ctx.payload.limit !== undefined ? { limit: ctx.payload.limit } : {}), + }) + return { rows, complete: rows.length < (ctx.payload.limit ?? EventV2.SNAPSHOT_TRANSFER_ROWS) } + }) + + const snapshotChunks = Effect.fn("SyncHttpApi.snapshotChunks")(function* (ctx: { + payload: typeof SnapshotChunksPayload.Type + }) { + const max = 8 + if (ctx.payload.limit !== undefined && (ctx.payload.limit < 1 || ctx.payload.limit > max)) + return yield* new HttpApiError.BadRequest({}) + const workspaceID = yield* InstanceState.workspaceID + if (!workspaceID || !events.snapshotChunks) return yield* new ServiceUnavailableError({ service: "sync.snapshot", message: "Snapshot chunk transfer is unavailable" }) + const scoped = yield* db + .select({ id: EventSnapshotRowTable.row_hash }) + .from(EventSnapshotRowTable) + .innerJoin(EventSnapshotTable, eq(EventSnapshotTable.snapshot_id, EventSnapshotRowTable.snapshot_id)) + .innerJoin(EventSequenceTable, eq(EventSequenceTable.snapshot_id, EventSnapshotTable.snapshot_id)) + .innerJoin(SessionTable, eq(SessionTable.id, EventSnapshotTable.aggregate_id)) + .where(and( + eq(SessionTable.workspace_id, workspaceID), + eq(EventSnapshotTable.aggregate_id, ctx.payload.aggregateID), + eq(EventSnapshotTable.snapshot_id, ctx.payload.snapshotID), + eq(EventSnapshotTable.snapshot_hash, ctx.payload.snapshotHash), + eq(EventSnapshotRowTable.row_hash, ctx.payload.rowHash), + )) + .get() + .pipe(Effect.orDie) + if (!scoped) return yield* new HttpApiError.NotFound() + const chunks = yield* events.snapshotChunks({ + rowHash: ctx.payload.rowHash, + ...(ctx.payload.after !== undefined ? { after: ctx.payload.after } : {}), + limit: ctx.payload.limit ?? max, + }) + return { + chunks: chunks.map((chunk) => ({ ...chunk, data: chunk.data.toString("base64") })), + complete: chunks.length < (ctx.payload.limit ?? max), + } + }) + + const maintenanceFailure = ( + aggregateID: string, + defect: unknown, + ): Effect.Effect => { + if (Schema.is(EventV2.MaintenanceRequiredError)(defect)) + return Effect.fail(new ServiceUnavailableError({ service: defect.reason, message: defect.message })) + if (Schema.is(EventV2.InvalidSyncEventError)(defect)) + return Effect.fail(new ConflictError({ resource: `session:${aggregateID}`, message: defect.message })) + return Effect.die(defect) + } + const maintenanceSession = Effect.fn("SyncHttpApi.maintenanceSession")(function* (aggregateID: SessionID) { + const instance = yield* InstanceState.context + const workspaceID = yield* InstanceState.workspaceID + const row = yield* db + .select({ + id: SessionTable.id, + projectID: SessionTable.project_id, + directory: SessionTable.directory, + workspaceID: SessionTable.workspace_id, + seq: EventSequenceTable.seq, + ownerID: EventSequenceTable.owner_id, + floor: EventSequenceTable.retention_floor_seq, + }) + .from(SessionTable) + .innerJoin(EventSequenceTable, eq(EventSequenceTable.aggregate_id, SessionTable.id)) + .where(eq(SessionTable.id, aggregateID)) + .get() + .pipe(Effect.orDie) + if (!row) return yield* new HttpApiError.NotFound() + if ( + row.projectID !== instance.project.id || + row.directory !== instance.directory || + (row.workspaceID ?? undefined) !== workspaceID + ) + return yield* new ConflictError({ + resource: `session:${aggregateID}`, + message: "Checkpoint maintenance must match the routed project, directory, and workspace authority", + }) + return row + }) + const checkpointAttemptSession = Effect.fn("SyncHttpApi.checkpointAttemptSession")(function* (snapshotID: string) { + const attempt = yield* db + .select({ aggregateID: EventSnapshotAttemptTable.aggregate_id }) + .from(EventSnapshotAttemptTable) + .where(eq(EventSnapshotAttemptTable.snapshot_id, snapshotID)) + .get() + .pipe(Effect.orDie) + if (!attempt) return yield* new HttpApiError.NotFound() + return yield* maintenanceSession(SessionID.make(attempt.aggregateID)) + }) + const checkpointPrepare = Effect.fn("SyncHttpApi.checkpointPrepare")(function* (ctx: { + payload: typeof CheckpointPreparePayload.Type + }) { + if (!events.prepareCheckpoint) return yield* new ServiceUnavailableError({ service: "sync.checkpoint", message: "Checkpoint preparation is unavailable" }) + const row = yield* maintenanceSession(ctx.payload.aggregateID) + return yield* events.prepareCheckpoint({ + aggregateID: row.id, + throughSeq: EventV2.Cursor.make(row.seq), + expectedLatest: EventV2.Cursor.make(row.seq), + codec: "session-projection", + schemaVersion: 1, + ...(row.ownerID ? { ownerID: row.ownerID } : {}), + }).pipe(Effect.catchDefect((defect) => maintenanceFailure(row.id, defect))) + }) + const checkpointStage = Effect.fn("SyncHttpApi.checkpointStage")(function* (ctx: { + payload: typeof CheckpointStagePayload.Type + }) { + if (!events.stageCheckpoint) return yield* new ServiceUnavailableError({ service: "sync.checkpoint", message: "Checkpoint staging is unavailable" }) + const row = yield* checkpointAttemptSession(ctx.payload.snapshotID) + if (ctx.payload.limit !== undefined && (ctx.payload.limit < 1 || ctx.payload.limit > EventV2.SNAPSHOT_TRANSFER_ROWS)) + return yield* new HttpApiError.BadRequest({}) + return yield* events.stageCheckpoint({ + snapshotID: ctx.payload.snapshotID, + ...(ctx.payload.limit ? { limit: ctx.payload.limit } : {}), + }).pipe(Effect.catchDefect((defect) => maintenanceFailure(row.id, defect))) + }) + const checkpointFinalize = Effect.fn("SyncHttpApi.checkpointFinalize")(function* (ctx: { + payload: typeof CheckpointFinalizePayload.Type + }) { + if (!events.finalizeCheckpoint) return yield* new ServiceUnavailableError({ service: "sync.checkpoint", message: "Checkpoint finalization is unavailable" }) + const row = yield* checkpointAttemptSession(ctx.payload.snapshotID) + return yield* events.finalizeCheckpoint({ snapshotID: ctx.payload.snapshotID }).pipe( + Effect.catchDefect((defect) => maintenanceFailure(row.id, defect)), + ) + }) + const checkpointDiscard = Effect.fn("SyncHttpApi.checkpointDiscard")(function* (ctx: { + payload: typeof CheckpointDiscardPayload.Type + }) { + if (!events.discardCheckpoint) return yield* new ServiceUnavailableError({ service: "sync.checkpoint", message: "Checkpoint discard is unavailable" }) + const row = yield* checkpointAttemptSession(ctx.payload.snapshotID) + if (ctx.payload.limit !== undefined && (ctx.payload.limit < 1 || ctx.payload.limit > EventV2.SNAPSHOT_TRANSFER_ROWS)) + return yield* new HttpApiError.BadRequest({}) + return yield* events.discardCheckpoint({ + snapshotID: ctx.payload.snapshotID, + ...(ctx.payload.limit ? { limit: ctx.payload.limit } : {}), + }).pipe(Effect.catchDefect((defect) => maintenanceFailure(row.id, defect))) + }) + const checkpointCompact = Effect.fn("SyncHttpApi.checkpointCompact")(function* (ctx: { + payload: typeof CheckpointCompactPayload.Type + }) { + const row = yield* maintenanceSession(ctx.payload.aggregateID) + if (row.floor === null) return yield* new ConflictError({ resource: `session:${row.id}`, message: "Compaction requires an active canonical checkpoint" }) + if (ctx.payload.limit !== undefined && (ctx.payload.limit < 1 || ctx.payload.limit > 100)) + return yield* new HttpApiError.BadRequest({}) + return yield* events.compact({ + aggregateID: row.id, + throughSeq: EventV2.Cursor.make(row.floor), + ...(ctx.payload.limit ? { limit: ctx.payload.limit } : {}), + }).pipe(Effect.catchDefect((defect) => maintenanceFailure(row.id, defect))) + }) + return handlers .handle("start", start) .handle("replay", replay) .handle("steal", steal) .handle("history", history) .handle("artifacts", artifacts) + .handle("eventIndex", eventIndex) + .handle("fileArtifacts", fileArtifacts) + .handle("fileArtifactMetadata", fileArtifactMetadata) + .handle("fileArtifactChunk", fileArtifactChunk) + .handle("snapshotRows", snapshotRows) + .handle("snapshotChunks", snapshotChunks) + .handle("checkpointPrepare", checkpointPrepare) + .handle("checkpointStage", checkpointStage) + .handle("checkpointFinalize", checkpointFinalize) + .handle("checkpointDiscard", checkpointDiscard) + .handle("checkpointCompact", checkpointCompact) }), ) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/server.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/server.ts index a012ec1b5..ccdaf7dc8 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/server.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/server.ts @@ -49,6 +49,7 @@ import { SessionSteer } from "@/session/steer" import { SessionRunState } from "@/session/run-state" import { SessionStatus } from "@/session/status" import { SessionSummary } from "@/session/summary" +import { SessionProjection } from "@/session/session-projector" import { SessionLegacyProviderResolution } from "@/session/legacy-provider-resolution" import { Todo } from "@/session/todo" import { SessionShare } from "@/share/session" @@ -380,6 +381,7 @@ export function createRoutes( ShareNext.defaultLayer, Snapshot.defaultLayer, EventV2Bridge.defaultLayer, + SessionProjection.defaultLayer, EventV2.defaultLayer, Skill.defaultLayer, Todo.defaultLayer, diff --git a/packages/deepagent-code/src/session/diff-artifact-schema.ts b/packages/deepagent-code/src/session/diff-artifact-schema.ts new file mode 100644 index 000000000..0e44bed3c --- /dev/null +++ b/packages/deepagent-code/src/session/diff-artifact-schema.ts @@ -0,0 +1,43 @@ +import { Schema } from "effect" + +export const Limits = { + batch: 4, + manifestFiles: 100, + patchBytes: 1024 * 1024, +} as const + +export const Descriptor = Schema.Struct({ + id: Schema.String, + hash: Schema.String, + codec: Schema.Literals(["legacy-message-diff.v1", "legacy-message-diff.v2"]), + fileCount: Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)), +}) + +export const ManifestFile = Schema.Struct({ + file: Schema.String, + additions: Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)), + deletions: Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)), + status: Schema.optional(Schema.Literals(["added", "deleted", "modified"])), + patchBytes: Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)), + patchHash: Schema.String, +}) + +export const Manifest = Schema.Struct({ + artifact: Descriptor, + files: Schema.Array(ManifestFile), + nextCursor: Schema.optional(Schema.String), + complete: Schema.Boolean, +}) + +export const File = Schema.Struct({ + artifactID: Schema.String, + file: Schema.String, + additions: Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)), + deletions: Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)), + status: Schema.optional(Schema.Literals(["added", "deleted", "modified"])), + patch: Schema.String, + patchBytes: Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)), + returnedBytes: Schema.Number.check(Schema.isInt(), Schema.isGreaterThanOrEqualTo(0)), + patchHash: Schema.String, + truncated: Schema.Boolean, +}) diff --git a/packages/deepagent-code/src/session/diff-artifact.sql.ts b/packages/deepagent-code/src/session/diff-artifact.sql.ts new file mode 100644 index 000000000..04339c69c --- /dev/null +++ b/packages/deepagent-code/src/session/diff-artifact.sql.ts @@ -0,0 +1,61 @@ +import { index, integer, primaryKey, sqliteTable, text, uniqueIndex, blob } from "drizzle-orm/sqlite-core" + +export const SessionDiffMigrationReceiptTable = sqliteTable( + "session_diff_migration_receipt", + { + message_id: text().notNull().primaryKey(), + session_id: text().notNull(), + artifact_id: text().notNull(), + source_event_id: text().notNull(), + expected_message_data_hash: text().notNull(), + committed_message_data_hash: text(), + expected_session_summary_hash: text().notNull(), + committed_session_summary_hash: text(), + canonicalizer_version: integer().notNull(), + canonicalization_version: integer().notNull(), + epoch_hashes: text({ mode: "json" }) + .$type>() + .notNull(), + state: text().$type<"prepared" | "committed" | "migration_validation_failed">().notNull(), + failure_reason: text(), + created_at: integer().notNull(), + updated_at: integer().notNull(), + committed_at: integer(), + }, + (table) => [ + uniqueIndex("session_diff_migration_receipt_artifact_idx").on(table.artifact_id), + index("session_diff_migration_receipt_session_state_idx").on(table.session_id, table.state), + ], +) + +export const SessionDiffArtifactFileTable = sqliteTable( + "session_diff_artifact_file", + { + artifact_id: text().notNull(), + file_index: integer().notNull(), + path: text().notNull(), + path_key: text().notNull(), + additions: integer().notNull(), + deletions: integer().notNull(), + status: text().$type<"added" | "deleted" | "modified" | null>(), + patch_hash: text().notNull(), + patch_bytes: integer().notNull(), + patch_chunk_count: integer().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.artifact_id, table.file_index] }), + uniqueIndex("session_diff_artifact_file_path_idx").on(table.artifact_id, table.path_key), + ], +) + +export const SessionDiffArtifactFileChunkTable = sqliteTable( + "session_diff_artifact_file_chunk", + { + artifact_id: text().notNull(), + file_index: integer().notNull(), + chunk_index: integer().notNull(), + data: blob({ mode: "buffer" }).notNull(), + chunk_hash: text().notNull(), + }, + (table) => [primaryKey({ columns: [table.artifact_id, table.file_index, table.chunk_index] })], +) diff --git a/packages/deepagent-code/src/session/diff-artifact.ts b/packages/deepagent-code/src/session/diff-artifact.ts new file mode 100644 index 000000000..85ac39305 --- /dev/null +++ b/packages/deepagent-code/src/session/diff-artifact.ts @@ -0,0 +1,804 @@ +import { Database } from "@deepagent-code/core/database/database" +import { EventArtifactChunkTable, EventArtifactTable } from "@deepagent-code/core/event/sql" +import { MessageTable, SessionPromptEpochMessageTable, SessionTable } from "@deepagent-code/core/session/sql" +import { CanonicalJson } from "@deepagent-code/core/util/canonical-json" +import { Hash } from "@deepagent-code/core/util/hash" +import { SessionV1 } from "@deepagent-code/core/v1/session" +import { MessageID, SessionID } from "./schema" +import { MessageV2 } from "./message-v2" +import { HistoryAuthority } from "./history-authority" +import { SessionPromptEpochTable } from "./prompt-epoch.sql" +import { + SessionDiffArtifactFileChunkTable, + SessionDiffArtifactFileTable, + SessionDiffMigrationReceiptTable, +} from "./diff-artifact.sql" +import { and, asc, eq, gt, sql } from "drizzle-orm" +import { createHash } from "node:crypto" +import { Data, Effect, Schema } from "effect" +import { Descriptor, File, Limits, Manifest, ManifestFile } from "./diff-artifact-schema" + +export { Descriptor, File, Limits, Manifest, ManifestFile } from "./diff-artifact-schema" + +const FILE_CHUNK_BYTES = 256 * 1024 +const MAX_MIGRATION_FILES = 10_000 +const MAX_MIGRATION_BODY_BYTES = 64 * 1024 * 1024 +const MAX_PATH_BYTES = 4096 +const NULL_SUMMARY_HASH = Hash.sha256("null") + +export class Invalid extends Data.TaggedError("SessionDiffArtifact.Invalid")<{ + readonly message: string +}> {} + +export class NotFound extends Data.TaggedError("SessionDiffArtifact.NotFound")<{ + readonly message: string +}> {} + +class RewriteCasLost extends Data.TaggedError("SessionDiffArtifact.RewriteCasLost")<{ + readonly message: string + readonly expectedMessageDataHash: string + readonly expectedSessionSummaryHash: string +}> {} + +type CanonicalArtifact = { + id: string + hash: string + codec: "legacy-message-diff.v1" | "legacy-message-diff.v2" + fileCount: number +} + +type Candidate = { + artifactID: string + sourceEventID: string + sourceSeq: number + sessionID: SessionID + messageID: MessageID + originalDataHash: string + canonicalDataHash: string + canonicalDataText: string + bodyHash: string + bodyBytes: number + chunkCount: number + codecVersion: number + canonicalData: Record +} + +export const migrate = Effect.fn("SessionDiffArtifact.migrate")(function* (input: { + sessionID: SessionID + limit?: number + now?: number +}) { + const { db } = yield* Database.Service + const limit = Math.min(Math.max(input.limit ?? Limits.batch, 1), Limits.batch) + const candidates = yield* db + .select({ + artifactID: EventArtifactTable.artifact_id, + sourceEventID: EventArtifactTable.event_id, + sourceSeq: EventArtifactTable.seq, + sessionID: EventArtifactTable.aggregate_id, + messageID: sql`json_extract(${EventArtifactTable.canonical_data}, '$.info.id')`, + originalDataHash: EventArtifactTable.original_data_hash, + canonicalDataHash: EventArtifactTable.canonical_data_hash, + canonicalDataText: sql`CAST(${EventArtifactTable.canonical_data} AS TEXT)`, + bodyHash: EventArtifactTable.body_hash, + bodyBytes: EventArtifactTable.body_bytes, + chunkCount: EventArtifactTable.chunk_count, + codecVersion: EventArtifactTable.codec_version, + canonicalData: EventArtifactTable.canonical_data, + }) + .from(EventArtifactTable) + .innerJoin( + MessageTable, + and( + eq(MessageTable.session_id, EventArtifactTable.aggregate_id), + sql`${MessageTable.id} = json_extract(${EventArtifactTable.canonical_data}, '$.info.id')`, + ), + ) + .leftJoin( + SessionDiffMigrationReceiptTable, + eq(SessionDiffMigrationReceiptTable.message_id, MessageTable.id), + ) + .where( + and( + eq(EventArtifactTable.aggregate_id, input.sessionID), + eq(EventArtifactTable.kind, "legacy_message_diff"), + sql`json_extract(${MessageTable.data}, '$.role') = 'user'`, + sql`json_type(${MessageTable.data}, '$.summary.diffs') = 'array'`, + sql`${SessionDiffMigrationReceiptTable.message_id} is null`, + ), + ) + .orderBy(asc(EventArtifactTable.event_id)) + .limit(limit) + .all() + .pipe(Effect.orDie) + const results = yield* Effect.forEach(candidates, (candidate) => + migrateCandidate( + db, + { ...candidate, sessionID: SessionID.make(candidate.sessionID) }, + input.now ?? Date.now(), + ), + ) + return { + processed: results.length, + committed: results.filter((result) => result.state === "committed").length, + failed: results.filter((result) => result.state === "migration_validation_failed").length, + } +}) + +function migrateCandidate(db: Database.Interface["db"], candidate: Candidate, now: number) { + return db + .transaction( + (tx) => + Effect.gen(function* () { + const existing = yield* tx + .select() + .from(SessionDiffMigrationReceiptTable) + .where(eq(SessionDiffMigrationReceiptTable.message_id, candidate.messageID)) + .get() + if (existing?.state === "committed") { + if ( + existing.session_id !== candidate.sessionID || + existing.artifact_id !== candidate.artifactID || + existing.source_event_id !== candidate.sourceEventID + ) + return yield* Effect.die(new Error(`Committed diff migration receipt conflict: ${candidate.messageID}`)) + return { state: "committed" as const } + } + if (existing?.state === "migration_validation_failed") + return { state: "migration_validation_failed" as const } + + const failure = (reason: string, expectedMessageDataHash = Hash.sha256("missing")) => + Effect.gen(function* () { + yield* tx + .delete(SessionDiffArtifactFileChunkTable) + .where(eq(SessionDiffArtifactFileChunkTable.artifact_id, candidate.artifactID)) + .run() + yield* tx + .delete(SessionDiffArtifactFileTable) + .where(eq(SessionDiffArtifactFileTable.artifact_id, candidate.artifactID)) + .run() + return yield* tx + .insert(SessionDiffMigrationReceiptTable) + .values({ + message_id: candidate.messageID, + session_id: candidate.sessionID, + artifact_id: candidate.artifactID, + source_event_id: candidate.sourceEventID, + expected_message_data_hash: expectedMessageDataHash, + expected_session_summary_hash: Hash.sha256("unknown"), + canonicalizer_version: candidate.codecVersion, + canonicalization_version: HistoryAuthority.CANONICALIZATION_VERSION, + epoch_hashes: [], + state: "migration_validation_failed" as const, + failure_reason: reason, + created_at: now, + updated_at: now, + }) + .onConflictDoUpdate({ + target: SessionDiffMigrationReceiptTable.message_id, + set: { state: "migration_validation_failed", failure_reason: reason, updated_at: now }, + }) + .run() + .pipe(Effect.orDie, Effect.as({ state: "migration_validation_failed" as const })) + }) + + const artifact = canonicalArtifact(candidate) + if (!artifact) return yield* failure("canonical artifact descriptor is invalid") + if (candidate.bodyBytes > MAX_MIGRATION_BODY_BYTES) + return yield* failure("artifact body exceeds the migration byte budget") + if (Hash.sha256(candidate.canonicalDataText) !== candidate.canonicalDataHash) + return yield* failure("artifact canonical data hash mismatch") + const source = yield* tx.get<{ bytes: number }>(sql` + SELECT length(CAST(data AS BLOB)) AS bytes + FROM event + WHERE id = ${candidate.sourceEventID} + AND aggregate_id = ${candidate.sessionID} + AND seq = ${candidate.sourceSeq} + `) + if (!source) return yield* failure("artifact source event is missing") + const sourceDigest = createHash("sha256") + const sourceChunkCount = Math.ceil(source.bytes / FILE_CHUNK_BYTES) || 1 + for (const index of Array.from({ length: sourceChunkCount }, (_, chunkIndex) => chunkIndex)) { + const sourceChunk = yield* tx.get<{ data: Buffer }>(sql` + SELECT substr(CAST(data AS BLOB), ${index * FILE_CHUNK_BYTES + 1}, ${FILE_CHUNK_BYTES}) AS data + FROM event + WHERE id = ${candidate.sourceEventID} + AND aggregate_id = ${candidate.sessionID} + `) + if (!sourceChunk) return yield* failure(`artifact source chunk ${index} is missing`) + sourceDigest.update(sourceChunk.data) + } + if (sourceDigest.digest("hex") !== candidate.originalDataHash) + return yield* failure("artifact source event hash mismatch") + if (candidate.chunkCount !== Math.max(1, Math.ceil(candidate.bodyBytes / FILE_CHUNK_BYTES))) + return yield* failure("artifact chunk metadata is invalid") + const artifactChunkCount = yield* tx + .select({ count: sql`count(*)` }) + .from(EventArtifactChunkTable) + .where(eq(EventArtifactChunkTable.artifact_id, candidate.artifactID)) + .get() + if (artifactChunkCount?.count !== candidate.chunkCount) + return yield* failure("artifact chunk count mismatch") + const bodyDigest = createHash("sha256") + let observedBodyBytes = 0 + for (const index of Array.from({ length: candidate.chunkCount }, (_, chunkIndex) => chunkIndex)) { + const chunk = yield* tx + .select() + .from(EventArtifactChunkTable) + .where( + and( + eq(EventArtifactChunkTable.artifact_id, candidate.artifactID), + eq(EventArtifactChunkTable.chunk_index, index), + ), + ) + .get() + if (!chunk || Hash.sha256(chunk.data) !== chunk.chunk_hash) + return yield* failure(`artifact chunk ${index} failed integrity validation`) + bodyDigest.update(chunk.data) + observedBodyBytes += chunk.data.length + } + if (observedBodyBytes !== candidate.bodyBytes) return yield* failure("artifact body byte count mismatch") + if (bodyDigest.digest("hex") !== candidate.bodyHash) return yield* failure("artifact body hash mismatch") + + const current = yield* tx.get<{ data: string; body: string }>(sql` + SELECT CAST(message.data AS TEXT) AS data, + json_object('diffs', json_extract(message.data, '$.summary.diffs')) AS body + FROM message + WHERE message.id = ${candidate.messageID} + AND message.session_id = ${candidate.sessionID} + AND json_extract(message.data, '$.role') = 'user' + `) + if (!current) return yield* failure("candidate user message is missing") + const expectedMessageDataHash = Hash.sha256(current.data) + const sessionSummary = yield* tx.get<{ data: string | null; body: string | null }>(sql` + SELECT CAST(summary_diffs AS TEXT) AS data, + CASE WHEN summary_diffs IS NULL THEN NULL ELSE json_object('diffs', json(summary_diffs)) END AS body + FROM session + WHERE id = ${candidate.sessionID} + `) + if (!sessionSummary) return yield* failure("candidate Session is missing", expectedMessageDataHash) + const expectedSessionSummaryHash = Hash.sha256(sessionSummary.data ?? "null") + if (Hash.sha256(current.body) !== candidate.bodyHash) + return yield* failure("message inline diff no longer matches the committed artifact", expectedMessageDataHash) + if (sessionSummary.body !== null && Hash.sha256(sessionSummary.body) !== candidate.bodyHash) + return yield* failure("Session summary no longer matches the committed artifact", expectedMessageDataHash) + + const decoded = Schema.decodeUnknownOption(Schema.UnknownFromJsonString)(current.data) + const parsed = decoded._tag === "Some" ? decoded.value : undefined + if (!record(parsed) || parsed.role !== "user" || !record(parsed.summary)) + return yield* failure("candidate message shape is invalid", expectedMessageDataHash) + const next = { + ...parsed, + summary: { + ...parsed.summary, + diffs: undefined, + diffArtifact: artifact, + }, + } + delete next.summary.diffs + const nextData = CanonicalJson.stringify(next) + const epochHashes = yield* validateEpochHashes( + tx as unknown as Database.Interface["db"], + candidate.sessionID, + candidate.messageID, + parsed, + next, + ) + if ("failure" in epochHashes) + return yield* failure(epochHashes.failure ?? "history authority validation failed", expectedMessageDataHash) + + yield* tx + .insert(SessionDiffMigrationReceiptTable) + .values({ + message_id: candidate.messageID, + session_id: candidate.sessionID, + artifact_id: candidate.artifactID, + source_event_id: candidate.sourceEventID, + expected_message_data_hash: expectedMessageDataHash, + expected_session_summary_hash: expectedSessionSummaryHash, + canonicalizer_version: candidate.codecVersion, + canonicalization_version: HistoryAuthority.CANONICALIZATION_VERSION, + epoch_hashes: epochHashes.values, + state: "prepared", + created_at: now, + updated_at: now, + }) + .run() + + const files = yield* tx.all<{ + file_index: number + path: string + additions: number + deletions: number + status: "added" | "deleted" | "modified" | null + patch_bytes: number + }>(sql` + SELECT CAST(diff.key AS INTEGER) AS file_index, + json_extract(diff.value, '$.file') AS path, + CAST(COALESCE(json_extract(diff.value, '$.additions'), 0) AS INTEGER) AS additions, + CAST(COALESCE(json_extract(diff.value, '$.deletions'), 0) AS INTEGER) AS deletions, + json_extract(diff.value, '$.status') AS status, + length(CAST(COALESCE(json_extract(diff.value, '$.patch'), '') AS BLOB)) AS patch_bytes + FROM message source, json_each(json_extract(source.data, '$.summary.diffs')) AS diff + WHERE source.id = ${candidate.messageID} + AND source.session_id = ${candidate.sessionID} + ORDER BY CAST(diff.key AS INTEGER) + LIMIT ${MAX_MIGRATION_FILES + 1} + `) + if (files.length !== artifact.fileCount || files.length > MAX_MIGRATION_FILES) + return yield* failure("artifact file manifest count is invalid", expectedMessageDataHash) + const pathKeys = new Set() + for (const file of files) { + const path = normalizePath(file.path) + if (!path) return yield* failure(`artifact file path ${file.file_index} is invalid`, expectedMessageDataHash) + const pathKey = path.toLocaleLowerCase("en-US") + if (pathKeys.has(pathKey)) return yield* failure("artifact contains a case-colliding file path", expectedMessageDataHash) + pathKeys.add(pathKey) + const chunkCount = Math.max(1, Math.ceil(file.patch_bytes / FILE_CHUNK_BYTES)) + const digest = createHash("sha256") + yield* tx + .insert(SessionDiffArtifactFileTable) + .values({ + artifact_id: candidate.artifactID, + file_index: file.file_index, + path, + path_key: pathKey, + additions: file.additions, + deletions: file.deletions, + status: file.status, + patch_hash: Hash.sha256("pending"), + patch_bytes: file.patch_bytes, + patch_chunk_count: chunkCount, + }) + .run() + for (const index of Array.from({ length: chunkCount }, (_, chunkIndex) => chunkIndex)) { + const patchChunk = yield* tx.get<{ data: Buffer }>(sql` + SELECT substr( + CAST(COALESCE(json_extract(diff.value, '$.patch'), '') AS BLOB), + ${index * FILE_CHUNK_BYTES + 1}, ${FILE_CHUNK_BYTES} + ) AS data + FROM message source, json_each(json_extract(source.data, '$.summary.diffs')) AS diff + WHERE source.id = ${candidate.messageID} + AND source.session_id = ${candidate.sessionID} + AND CAST(diff.key AS INTEGER) = ${file.file_index} + `) + if (!patchChunk) + return yield* failure(`artifact file ${file.file_index} chunk is missing`, expectedMessageDataHash) + digest.update(patchChunk.data) + yield* tx + .insert(SessionDiffArtifactFileChunkTable) + .values({ + artifact_id: candidate.artifactID, + file_index: file.file_index, + chunk_index: index, + data: patchChunk.data, + chunk_hash: Hash.sha256(patchChunk.data), + }) + .run() + } + yield* tx + .update(SessionDiffArtifactFileTable) + .set({ patch_hash: digest.digest("hex") }) + .where( + and( + eq(SessionDiffArtifactFileTable.artifact_id, candidate.artifactID), + eq(SessionDiffArtifactFileTable.file_index, file.file_index), + ), + ) + .run() + } + + const updated = yield* tx + .update(MessageTable) + .set({ + data: sql`json(${nextData})` as unknown as typeof MessageTable.$inferInsert.data, + time_updated: now, + }) + .where( + and( + eq(MessageTable.id, candidate.messageID), + eq(MessageTable.session_id, candidate.sessionID), + sql`CAST(${MessageTable.data} AS TEXT) = ${current.data}`, + ), + ) + .returning({ data: sql`CAST(${MessageTable.data} AS TEXT)` }) + .get() + if (!updated) + return yield* new RewriteCasLost({ + message: "message compare-and-swap lost", + expectedMessageDataHash, + expectedSessionSummaryHash, + }) + const committedMessageDataHash = Hash.sha256(updated.data) + if (committedMessageDataHash !== Hash.sha256(nextData)) + return yield* Effect.die(new Error(`Committed diff message bytes diverged: ${candidate.messageID}`)) + if (sessionSummary.data !== null) { + const sessionUpdated = yield* tx + .update(SessionTable) + .set({ summary_diffs: null }) + .where( + and( + eq(SessionTable.id, candidate.sessionID), + sql`CAST(${SessionTable.summary_diffs} AS TEXT) = ${sessionSummary.data}`, + ), + ) + .returning({ id: SessionTable.id }) + .get() + if (!sessionUpdated) + return yield* new RewriteCasLost({ + message: "Session summary compare-and-swap lost", + expectedMessageDataHash, + expectedSessionSummaryHash, + }) + } + yield* tx + .update(SessionDiffMigrationReceiptTable) + .set({ + state: "committed", + committed_message_data_hash: committedMessageDataHash, + committed_session_summary_hash: NULL_SUMMARY_HASH, + updated_at: now, + committed_at: now, + }) + .where( + and( + eq(SessionDiffMigrationReceiptTable.message_id, candidate.messageID), + eq(SessionDiffMigrationReceiptTable.state, "prepared"), + ), + ) + .run() + return { state: "committed" as const } + }), + { behavior: "immediate" }, + ) + .pipe( + Effect.catchTag("SessionDiffArtifact.RewriteCasLost", (error) => + db + .transaction( + (tx) => + tx + .insert(SessionDiffMigrationReceiptTable) + .values({ + message_id: candidate.messageID, + session_id: candidate.sessionID, + artifact_id: candidate.artifactID, + source_event_id: candidate.sourceEventID, + expected_message_data_hash: error.expectedMessageDataHash, + expected_session_summary_hash: error.expectedSessionSummaryHash, + canonicalizer_version: candidate.codecVersion, + canonicalization_version: HistoryAuthority.CANONICALIZATION_VERSION, + epoch_hashes: [], + state: "migration_validation_failed" as const, + failure_reason: error.message, + created_at: now, + updated_at: now, + }) + .onConflictDoUpdate({ + target: SessionDiffMigrationReceiptTable.message_id, + set: { + state: "migration_validation_failed", + failure_reason: error.message, + updated_at: now, + }, + }) + .run(), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie, Effect.as({ state: "migration_validation_failed" as const })), + ), + Effect.orDie, + ) +} + +function canonicalArtifact(candidate: Candidate): CanonicalArtifact | undefined { + const info = record(candidate.canonicalData.info) ? candidate.canonicalData.info : undefined + const summary = info && record(info.summary) ? info.summary : undefined + const descriptor = summary && record(summary.diffArtifact) ? summary.diffArtifact : undefined + if ( + info?.id !== candidate.messageID || + !descriptor || + descriptor.id !== candidate.artifactID || + descriptor.hash !== candidate.bodyHash || + (descriptor.codec !== "legacy-message-diff.v1" && descriptor.codec !== "legacy-message-diff.v2") || + typeof descriptor.fileCount !== "number" || + !Number.isSafeInteger(descriptor.fileCount) || + descriptor.fileCount < 0 + ) + return + return descriptor as CanonicalArtifact +} + +function validateEpochHashes( + tx: Database.Interface["db"], + sessionID: SessionID, + messageID: MessageID, + beforeInfo: Record, + afterInfo: Record, +) { + return Effect.gen(function* () { + const epochs = yield* tx + .select() + .from(SessionPromptEpochTable) + .where(eq(SessionPromptEpochTable.session_id, sessionID)) + .orderBy(asc(SessionPromptEpochTable.epoch)) + .all() + const physicalIDs = ( + yield* tx + .select({ id: MessageTable.id }) + .from(MessageTable) + .where(eq(MessageTable.session_id, sessionID)) + .orderBy(asc(MessageTable.time_created), asc(MessageTable.id)) + .all() + ).map((row) => row.id) + const values = [] as Array<{ epoch: number; before: string; after: string }> + for (const epoch of epochs) { + if ( + epoch.canonicalization_version !== HistoryAuthority.CANONICALIZATION_VERSION || + epoch.projection_version !== HistoryAuthority.PROJECTION_VERSION + ) + return { failure: `PromptEpoch ${epoch.epoch} uses an unsupported history authority version` } + const memberIDs = ( + yield* tx + .select({ message_id: SessionPromptEpochMessageTable.message_id }) + .from(SessionPromptEpochMessageTable) + .where( + and( + eq(SessionPromptEpochMessageTable.session_id, sessionID), + eq(SessionPromptEpochMessageTable.prompt_epoch, epoch.epoch), + ), + ) + .orderBy(asc(SessionPromptEpochMessageTable.ordinal)) + .all() + ).map((row) => row.message_id) + const messages = yield* MessageV2.messagesInTransaction(tx, sessionID, physicalIDs) + if (!messages) return { failure: `PromptEpoch ${epoch.epoch} physical history is incomplete` } + const baseIDs = memberIDs.length > 0 ? memberIDs : physicalIDs.slice(0, epoch.base_message_count ?? 0) + const base = yield* MessageV2.messagesInTransaction(tx, sessionID, baseIDs) + if (!base) return { failure: `PromptEpoch ${epoch.epoch} references a missing base message` } + if (epoch.base_message_count !== base.length || !epoch.effective_history_hash) + return { failure: `PromptEpoch ${epoch.epoch} immutable base authority is incomplete` } + if (HistoryAuthority.hash(base) !== epoch.effective_history_hash) + return { failure: `PromptEpoch ${epoch.epoch} immutable base hash is already inconsistent` } + const before = HistoryAuthority.hash(messages) + const after = HistoryAuthority.hash( + messages.map((message) => + message.info.id === messageID + ? { ...message, info: { ...afterInfo, id: messageID, sessionID } as SessionV1.User } + : message, + ), + ) + const isolatedBefore = HistoryAuthority.hash([ + { info: { ...beforeInfo, id: messageID, sessionID } as SessionV1.User, parts: [] }, + ]) + const isolatedAfter = HistoryAuthority.hash([ + { info: { ...afterInfo, id: messageID, sessionID } as SessionV1.User, parts: [] }, + ]) + if (before !== after || isolatedBefore !== isolatedAfter) + return { failure: `PromptEpoch ${epoch.epoch} history hash changed during user summary rewrite` } + values.push({ epoch: epoch.epoch, before, after }) + } + return { values } + }) +} + +export const manifest = Effect.fn("SessionDiffArtifact.manifest")(function* (input: { + sessionID: SessionID + messageID: MessageID + artifactID: string + cursor?: string + limit?: number +}) { + const { db } = yield* Database.Service + const receipt = yield* requireReceipt(db, input) + const after = decodeCursor(input.cursor) + if (input.cursor && after === undefined) return yield* new Invalid({ message: "artifact manifest cursor is invalid" }) + const limit = Math.min(Math.max(input.limit ?? Limits.manifestFiles, 1), Limits.manifestFiles) + const rows = yield* db + .select() + .from(SessionDiffArtifactFileTable) + .where( + and( + eq(SessionDiffArtifactFileTable.artifact_id, input.artifactID), + after === undefined ? undefined : gt(SessionDiffArtifactFileTable.file_index, after), + ), + ) + .orderBy(asc(SessionDiffArtifactFileTable.file_index)) + .limit(limit + 1) + .all() + .pipe(Effect.orDie) + const files = rows.slice(0, limit) + const descriptor = canonicalArtifact(receipt) + if (!descriptor) return yield* new Invalid({ message: "committed artifact descriptor is invalid" }) + const fileCount = yield* db + .select({ count: sql`count(*)` }) + .from(SessionDiffArtifactFileTable) + .where(eq(SessionDiffArtifactFileTable.artifact_id, input.artifactID)) + .get() + .pipe(Effect.orDie) + if (fileCount?.count !== descriptor.fileCount) + return yield* new Invalid({ message: "artifact file manifest count mismatch" }) + return { + artifact: descriptor, + files: files.map((row) => ({ + file: row.path, + additions: row.additions, + deletions: row.deletions, + ...(row.status ? { status: row.status } : {}), + patchBytes: row.patch_bytes, + patchHash: row.patch_hash, + })), + ...(rows.length > limit ? { nextCursor: encodeCursor(files.at(-1)!.file_index) } : {}), + complete: rows.length <= limit, + } +}) + +export const file = Effect.fn("SessionDiffArtifact.file")(function* (input: { + sessionID: SessionID + messageID: MessageID + artifactID: string + path: string + maxBytes?: number +}) { + const { db } = yield* Database.Service + yield* requireReceipt(db, input) + const path = normalizePath(input.path) + if (!path) return yield* new Invalid({ message: "artifact file path is invalid" }) + const row = yield* db + .select() + .from(SessionDiffArtifactFileTable) + .where( + and( + eq(SessionDiffArtifactFileTable.artifact_id, input.artifactID), + eq(SessionDiffArtifactFileTable.path_key, path.toLocaleLowerCase("en-US")), + ), + ) + .get() + .pipe(Effect.orDie) + if (!row || row.path !== path) return yield* new NotFound({ message: "artifact file was not found" }) + const maxBytes = Math.min(Math.max(input.maxBytes ?? Limits.patchBytes, 1), Limits.patchBytes) + if (row.patch_chunk_count !== Math.max(1, Math.ceil(row.patch_bytes / FILE_CHUNK_BYTES))) + return yield* new Invalid({ message: "artifact file chunk metadata is invalid" }) + const chunkCount = yield* db + .select({ count: sql`count(*)` }) + .from(SessionDiffArtifactFileChunkTable) + .where( + and( + eq(SessionDiffArtifactFileChunkTable.artifact_id, input.artifactID), + eq(SessionDiffArtifactFileChunkTable.file_index, row.file_index), + ), + ) + .get() + .pipe(Effect.orDie) + if (chunkCount?.count !== row.patch_chunk_count) + return yield* new Invalid({ message: "artifact file chunk count mismatch" }) + const digest = createHash("sha256") + const prefix = [] as Buffer[] + let remaining = maxBytes + let observedBytes = 0 + for (const index of Array.from({ length: row.patch_chunk_count }, (_, chunkIndex) => chunkIndex)) { + const chunk = yield* db + .select() + .from(SessionDiffArtifactFileChunkTable) + .where( + and( + eq(SessionDiffArtifactFileChunkTable.artifact_id, input.artifactID), + eq(SessionDiffArtifactFileChunkTable.file_index, row.file_index), + eq(SessionDiffArtifactFileChunkTable.chunk_index, index), + ), + ) + .get() + .pipe(Effect.orDie) + if (!chunk || Hash.sha256(chunk.data) !== chunk.chunk_hash) + return yield* new Invalid({ message: `artifact file chunk ${index} failed integrity validation` }) + digest.update(chunk.data) + observedBytes += chunk.data.length + if (remaining > 0) { + prefix.push(chunk.data.subarray(0, remaining)) + remaining -= Math.min(remaining, chunk.data.length) + } + } + if (observedBytes !== row.patch_bytes) + return yield* new Invalid({ message: "artifact file byte count mismatch" }) + if (digest.digest("hex") !== row.patch_hash) + return yield* new Invalid({ message: "artifact file hash mismatch" }) + const bytes = utf8Prefix(Buffer.concat(prefix), Math.min(row.patch_bytes, maxBytes)) + return { + artifactID: input.artifactID, + file: row.path, + additions: row.additions, + deletions: row.deletions, + ...(row.status ? { status: row.status } : {}), + patch: bytes.toString("utf8"), + patchBytes: row.patch_bytes, + returnedBytes: bytes.length, + patchHash: row.patch_hash, + truncated: bytes.length < row.patch_bytes, + } +}) + +function requireReceipt( + db: Database.Interface["db"], + input: { sessionID: SessionID; messageID: MessageID; artifactID: string }, +) { + return Effect.gen(function* () { + const row = yield* db + .select({ + receipt: SessionDiffMigrationReceiptTable, + artifact: EventArtifactTable, + canonicalDataText: sql`CAST(${EventArtifactTable.canonical_data} AS TEXT)`, + }) + .from(SessionDiffMigrationReceiptTable) + .innerJoin(EventArtifactTable, eq(EventArtifactTable.artifact_id, SessionDiffMigrationReceiptTable.artifact_id)) + .where( + and( + eq(SessionDiffMigrationReceiptTable.session_id, input.sessionID), + eq(SessionDiffMigrationReceiptTable.message_id, input.messageID), + eq(SessionDiffMigrationReceiptTable.artifact_id, input.artifactID), + eq(SessionDiffMigrationReceiptTable.state, "committed"), + ), + ) + .get() + .pipe(Effect.orDie) + if (!row) return yield* new NotFound({ message: "committed artifact was not found for this Session message" }) + if (Hash.sha256(row.canonicalDataText) !== row.artifact.canonical_data_hash) + return yield* new Invalid({ message: "committed artifact canonical data hash mismatch" }) + return { + artifactID: row.artifact.artifact_id, + sourceEventID: row.artifact.event_id, + sourceSeq: row.artifact.seq, + sessionID: SessionID.make(row.artifact.aggregate_id), + messageID: input.messageID, + originalDataHash: row.artifact.original_data_hash, + canonicalDataHash: row.artifact.canonical_data_hash, + canonicalDataText: row.canonicalDataText, + bodyHash: row.artifact.body_hash, + bodyBytes: row.artifact.body_bytes, + chunkCount: row.artifact.chunk_count, + codecVersion: row.artifact.codec_version, + canonicalData: row.artifact.canonical_data, + } satisfies Candidate + }) +} + +function normalizePath(value: unknown) { + if ( + typeof value !== "string" || + value.length === 0 || + Buffer.byteLength(value, "utf8") > MAX_PATH_BYTES || + value.includes("\0") + ) + return + const path = value.replaceAll("\\", "/") + if (path.startsWith("/") || /^[A-Za-z]:\//.test(path)) return + const segments = path.split("/") + if (segments.some((segment) => !segment || segment === "." || segment === "..")) return + return segments.join("/") +} + +function record(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value) +} + +function encodeCursor(index: number) { + return Buffer.from(JSON.stringify({ version: 1, index })).toString("base64url") +} + +function decodeCursor(value?: string) { + if (!value) return + const decoded = Schema.decodeUnknownOption(Schema.UnknownFromJsonString)(Buffer.from(value, "base64url").toString()) + if (!decoded._tag || decoded._tag === "None" || !record(decoded.value)) return + const index = decoded.value.index + if (typeof index !== "number" || !Number.isSafeInteger(index) || index < 0) return + return index +} + +function utf8Prefix(value: Buffer, limit: number) { + const length = Math.min(value.length, limit) + for (const end of Array.from({ length: Math.min(4, length + 1) }, (_, index) => length - index)) { + const prefix = value.subarray(0, end) + if (Buffer.from(prefix.toString("utf8"), "utf8").equals(prefix)) return prefix + } + return Buffer.alloc(0) +} + +export * as SessionDiffArtifact from "./diff-artifact" diff --git a/packages/deepagent-code/src/session/message-v2.ts b/packages/deepagent-code/src/session/message-v2.ts index 9385c3827..24e571a0c 100644 --- a/packages/deepagent-code/src/session/message-v2.ts +++ b/packages/deepagent-code/src/session/message-v2.ts @@ -26,6 +26,7 @@ import { import { NamedError } from "@deepagent-code/core/util/error" import { APICallError, convertToModelMessages, LoadAPIKeyError, type ModelMessage, type UIMessage } from "ai" import { Database } from "@deepagent-code/core/database/database" +import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" import { NotFoundError } from "@/storage/storage" import { and } from "drizzle-orm" import { desc } from "drizzle-orm" @@ -270,12 +271,18 @@ export const normalizeFormat = (data: Record): Record - ({ - ...normalizeFormat(row.data as Record), - id: row.id, - sessionID: row.session_id, - }) as Info +const info = (row: typeof MessageTable.$inferSelect) => { + const data = normalizeFormat(row.data as Record) + const summary = + data.summary && typeof data.summary === "object" && !Array.isArray(data.summary) + ? (data.summary as Record) + : undefined + const normalized = + data.role === "user" && summary + ? { ...data, summary: { ...summary, diffs: Array.isArray(summary.diffs) ? summary.diffs : [] } } + : data + return { ...normalized, id: row.id, sessionID: row.session_id } as Info +} export function stripActivityProgress(message: T): T { if (message.role !== "assistant" || !message.activityProgress) return message @@ -543,6 +550,7 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* ( const result: UIMessage[] = [] const toolNames = new Set() const reasoningReplay = reasoningReplayCapability(model) + const database = Option.getOrUndefined(yield* Effect.serviceOption(Database.Service)) // Track media from tool results that need to be injected as user messages // for providers that don't support that media type in tool results. // @@ -624,9 +632,22 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* ( text: `[Attached ${part.mime}: ${part.filename ?? "file"}]`, }) } else { + if (part.artifact && !database) + return yield* Effect.die( + new FilePartArtifact.IntegrityError({ + artifactID: part.artifact.id, + message: "File-part artifact materialization requires the durable database authority", + }), + ) + const body = part.artifact + ? yield* FilePartArtifact.read({ + aggregateID: part.sessionID, + descriptor: FilePartArtifact.Descriptor.make(part.artifact), + }).pipe(Effect.provideService(Database.Service, database!)) + : undefined userMessage.parts.push({ type: "file", - url: part.url, + url: body ? `data:${part.mime};base64,${body.toString("base64")}` : part.url, mediaType: part.mime, filename: part.filename, }) diff --git a/packages/deepagent-code/src/session/revert.ts b/packages/deepagent-code/src/session/revert.ts index f13ea715a..585e0097f 100644 --- a/packages/deepagent-code/src/session/revert.ts +++ b/packages/deepagent-code/src/session/revert.ts @@ -47,9 +47,10 @@ export const layer = Layer.effect( const revertUnlocked = Effect.fn("SessionRevert.revertUnlocked")(function* (input: RevertInput) { yield* state.assertNotBusy(input.sessionID) const session = yield* sessions.get(input.sessionID).pipe(Effect.orDie) - const target = yield* sessions - .getClientMessage({ sessionID: input.sessionID, messageID: input.messageID }) - .pipe(Effect.orDie) + const target = Option.getOrUndefined( + yield* sessions.getClientMessage({ sessionID: input.sessionID, messageID: input.messageID }).pipe(Effect.option), + ) + if (!target) return session const lastUser = Option.getOrUndefined( yield* sessions .findMessage(input.sessionID, (message) => message.info.role === "user" && message.info.id <= input.messageID) diff --git a/packages/deepagent-code/src/session/session-projector.ts b/packages/deepagent-code/src/session/session-projector.ts new file mode 100644 index 000000000..e22ae26bf --- /dev/null +++ b/packages/deepagent-code/src/session/session-projector.ts @@ -0,0 +1,18 @@ +export * as SessionProjection from "./session-projector" + +import { EventV2Bridge } from "@/event-v2-bridge" +import { Database } from "@deepagent-code/core/database/database" +import { EventV2 } from "@deepagent-code/core/event" +import { SessionProjector } from "@deepagent-code/core/session/projector" +import { Effect, Layer } from "effect" + +const eventLayer = Layer.effect( + EventV2.Service, + Effect.map(EventV2Bridge.Service, (events) => EventV2.Service.of(events)), +) + +export const layer = SessionProjector.layer.pipe(Layer.provide(eventLayer)) +export const defaultLayer = layer.pipe( + Layer.provide(EventV2Bridge.defaultLayer), + Layer.provide(Database.defaultLayer), +) diff --git a/packages/deepagent-code/test/control-plane/workspace.test.ts b/packages/deepagent-code/test/control-plane/workspace.test.ts index a5421cb4b..aef5a5e82 100644 --- a/packages/deepagent-code/test/control-plane/workspace.test.ts +++ b/packages/deepagent-code/test/control-plane/workspace.test.ts @@ -6,7 +6,7 @@ import path from "node:path" import { NodeHttpServer } from "@effect/platform-node" import { Cause, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect" import { FetchHttpClient, HttpServer, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" -import { and, eq } from "drizzle-orm" +import { and, eq, sql } from "drizzle-orm" import { FSUtil } from "@deepagent-code/core/fs-util" import * as Log from "@deepagent-code/core/util/log" import { GlobalBus, type GlobalEvent } from "@/bus/global" @@ -17,7 +17,7 @@ import { AbsolutePath } from "@deepagent-code/core/schema" import { Session as SessionNs } from "@/session/session" import { SessionID } from "@/session/schema" import { SessionTable } from "@deepagent-code/core/session/sql" -import { EventSequenceTable, EventTable, WorkspaceSyncCursorTable } from "@deepagent-code/core/event/sql" +import { EventSequenceTable, EventSyncSequenceTable, EventTable, WorkspaceSyncCursorTable } from "@deepagent-code/core/event/sql" import { SessionToolRequestReceiptTable } from "@/session/tool-request-receipt.sql" import { resetDatabase } from "../fixture/db" import { disposeAllInstances, provideTmpdirInstance, requireInstance, TestInstance } from "../fixture/fixture" @@ -39,6 +39,7 @@ import { EventV2 } from "@deepagent-code/core/event" import { SessionV1 } from "@deepagent-code/core/v1/session" import { SyncReplayLimits } from "@/sync/replay-protocol" import { Hash } from "@deepagent-code/core/util/hash" +import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" void Log.init({ print: false }) @@ -1294,6 +1295,13 @@ describe("workspace CRUD", () => { }, } expect(Buffer.byteLength(JSON.stringify(data))).toBe(EventV2.MAX_ENCODED_PAYLOAD_BYTES) + const sync = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) yield* db .insert(EventTable) .values({ @@ -1302,6 +1310,7 @@ describe("workspace CRUD", () => { seq: sequence + 1, type: EventV2.versionedType(SessionV1.Event.MessageUpdated.type, 1), data, + sync_seq: sync!.seq, }) .run() .pipe(Effect.orDie) @@ -1805,6 +1814,138 @@ describe("workspace sync state", () => { }) }, 30_000) + it.live("does not advance its durable cursor when file artifact import fails", () => { + let chunkRequests = 0 + let historyRequests = 0 + let metadataRequests = 0 + let sourceEvent: { + id: EventV2.ID + aggregateID: SessionID + seq: number + data: Record + metadata: FilePartArtifact.Metadata + } + return Effect.gen(function* () { + yield* HttpServer.serveEffect()( + Effect.gen(function* () { + const req = yield* HttpServerRequest.HttpServerRequest + const url = new URL(req.url, "http://localhost") + if (url.pathname === "/history-artifact-fail/global/event") + return HttpServerResponse.fromWeb(eventStreamResponse()) + if (url.pathname === "/history-artifact-fail/sync/history") + return ( + historyRequests++, + HttpServerResponse.fromWeb( + Response.json({ + version: 1, + items: [ + { + kind: "event", + id: sourceEvent.id, + aggregate_id: sourceEvent.aggregateID, + seq: sourceEvent.seq, + type: "message.part.updated.1", + data: sourceEvent.data, + }, + ], + nextCursor: "must-not-commit-artifact", + complete: true, + }), + ) + ) + if (url.pathname === "/history-artifact-fail/sync/artifact/file/metadata") + return (metadataRequests++, HttpServerResponse.fromWeb(Response.json(sourceEvent.metadata))) + if (url.pathname === "/history-artifact-fail/sync/artifact/file/chunk") + return yield* Effect.gen(function* () { + chunkRequests += 1 + return HttpServerResponse.fromWeb( + Response.json({ + artifactID: sourceEvent.metadata.descriptor.id, + index: 0, + hash: sourceEvent.metadata.chunkHashes[0], + data: Buffer.from("corrupt").toString("base64"), + }), + ) + }) + return HttpServerResponse.text("unexpected", { status: 500 }) + }), + ) + const url = yield* serverUrl() + yield* provideTmpdirInstance( + () => + Effect.gen(function* () { + const workspace = yield* Workspace.Service + const sessionSvc = yield* SessionNs.Service + const { db } = yield* Database.Service + const instance = yield* requireInstance + const type = unique("history-artifact-fail") + const info = workspaceInfo(instance.project.id, type) + yield* insertWorkspace(info) + registerAdapter(instance.project.id, type, remoteAdapter(`${url}/history-artifact-fail`).adapter) + const session = yield* sessionSvc.create({ title: "artifact target" }) + yield* attachSessionToWorkspace(session.id, info.id) + const messageID = SessionV1.MessageID.ascending() + yield* sessionSvc.updateMessage({ + id: messageID, + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: "build", + model: { providerID: "test" as never, modelID: "model" as never }, + }) + const nextSeq = ((yield* sessionSequence(session.id)) ?? -1) + 1 + const body = Buffer.alloc(FilePartArtifact.CHUNK_BYTES + 1, 0x71) + const prepared = FilePartArtifact.prepare( + SessionV1.Event.PartUpdated.type, + { + sessionID: session.id, + part: { + id: SessionV1.PartID.ascending(), + sessionID: session.id, + messageID, + type: "file", + mime: "application/octet-stream", + url: `data:application/octet-stream;base64,${body.toString("base64")}`, + }, + time: Date.now(), + }, + EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1, + EventV2.MAX_ENCODED_PAYLOAD_BYTES, + ) + const descriptor = prepared.artifacts[0]!.descriptor + const eventID = EventV2.ID.create() + sourceEvent = { + id: eventID, + aggregateID: session.id, + seq: nextSeq, + data: prepared.data, + metadata: FilePartArtifact.Metadata.make({ + eventID, + aggregateID: session.id, + seq: nextSeq, + originalDataHash: Hash.sha256(JSON.stringify(prepared.data)), + canonicalDataHash: Hash.sha256(JSON.stringify(prepared.data)), + canonicalData: prepared.data, + descriptor, + chunkHashes: prepared.artifacts[0]!.chunks.map((chunk) => Hash.sha256(chunk)), + }), + } + + yield* workspace.startWorkspaceSyncing(instance.project.id) + yield* eventuallyEffect(Effect.sync(() => expect(chunkRequests).toBeGreaterThanOrEqual(1)), 5_000) + expect(historyRequests).toBeGreaterThanOrEqual(1) + expect(metadataRequests).toBeGreaterThanOrEqual(1) + expect( + yield* db.select().from(WorkspaceSyncCursorTable).where(eq(WorkspaceSyncCursorTable.workspace_id, info.id)).get(), + ).toBeUndefined() + expect(yield* sessionSequence(session.id)).toBe(nextSeq - 1) + expect(yield* db.select().from(EventTable).where(eq(EventTable.id, eventID)).get()).toBeUndefined() + }), + { git: true }, + ) + }) + }, 30_000) + it.live("resumes from its durable cursor after reconnecting", () => { const historyBodies = new Array<{ version: 1; cursor?: string }>() return Effect.gen(function* () { diff --git a/packages/deepagent-code/test/server/httpapi-sync.test.ts b/packages/deepagent-code/test/server/httpapi-sync.test.ts index cf8fe1e74..7670d458c 100644 --- a/packages/deepagent-code/test/server/httpapi-sync.test.ts +++ b/packages/deepagent-code/test/server/httpapi-sync.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, mock, spyOn } from "bun:test" -import { Context, Effect, Layer } from "effect" +import { Context, Effect, Layer, Schema } from "effect" import { Flag } from "@deepagent-code/core/flag/flag" import { SyncPaths, SyncReplayLimits } from "../../src/server/routes/instance/httpapi/groups/sync" import { HttpApiApp } from "../../src/server/routes/instance/httpapi/server" @@ -15,14 +15,23 @@ import { testEffect } from "../lib/effect" import { httpApiLayer, requestInDirectory } from "./httpapi-layer" import { SyncHistoryLimits } from "@/server/routes/instance/httpapi/handlers/sync" import { Database } from "@deepagent-code/core/database/database" -import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { + EventSequenceTable, + EventSnapshotChunkTable, + EventSnapshotRowTable, + EventSnapshotTable, + EventSyncSequenceTable, + EventTable, +} from "@deepagent-code/core/event/sql" import { EventV2 } from "@deepagent-code/core/event" -import { eq } from "drizzle-orm" +import { eq, sql } from "drizzle-orm" import { HttpServer } from "effect/unstable/http" import { SessionV1 } from "@deepagent-code/core/v1/session" import { SessionTable } from "@deepagent-code/core/session/sql" import { encodeReplayRequestPrefix } from "@/sync/replay-protocol" import { WorkspaceV2 } from "@deepagent-code/core/workspace" +import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" +import { Hash } from "@deepagent-code/core/util/hash" void Log.init({ print: false }) @@ -61,6 +70,299 @@ afterEach(async () => { }) describe("sync HttpApi", () => { + it.instance( + "creates, exposes, and compacts a canonical checkpoint through bounded maintenance routes", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "maintenance checkpoint", workspaceID: syncWorkspaceID }) + const sibling = yield* Session.use.create({ title: "interleaved event", workspaceID: syncWorkspaceID }) + + const prepared = yield* requestInDirectory(SyncPaths.checkpointPrepare, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ aggregateID: session.id }), + }) + expect(prepared.status, yield* prepared.text).toBe(200) + const attempt = Schema.decodeUnknownSync( + Schema.Struct({ snapshotID: Schema.String, state: Schema.String, hasMore: Schema.Boolean }), + )(yield* prepared.json) + expect(attempt.state).toBe("prepared") + + let state = attempt.state + while (state === "prepared") { + const staged = yield* requestInDirectory(SyncPaths.checkpointStage, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ snapshotID: attempt.snapshotID, limit: 1 }), + }) + expect(staged.status, yield* staged.text).toBe(200) + state = String(((yield* staged.json) as Record).state) + } + expect(state).toBe("staged") + + const finalized = yield* requestInDirectory(SyncPaths.checkpointFinalize, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ snapshotID: attempt.snapshotID }), + }) + expect(finalized.status, yield* finalized.text).toBe(200) + const snapshot = (yield* finalized.json) as unknown as EventV2.SerializedSnapshot + + const history = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1 }), + }) + expect(history.status, yield* history.text).toBe(200) + const firstPage = (yield* history.json) as { + items: { kind: string; aggregate_id?: string; snapshot?: { snapshotID: string } }[] + nextCursor: string + } + expect(firstPage.items).toEqual([ + expect.objectContaining({ kind: "event", aggregate_id: sibling.id }), + ]) + + const resync = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1, cursor: firstPage.nextCursor }), + }) + expect(resync.status, yield* resync.text).toBe(200) + expect(((yield* resync.json) as { items: { kind: string; snapshot?: { snapshotID: string } }[] }).items).toEqual([ + { kind: "resync_required", snapshot: expect.objectContaining({ snapshotID: snapshot.snapshotID }) }, + ]) + + let compacted = false + while (!compacted) { + const response = yield* requestInDirectory(SyncPaths.checkpointCompact, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ aggregateID: session.id, limit: 1 }), + }) + expect(response.status, yield* response.text).toBe(200) + compacted = Boolean(((yield* response.json) as { complete: boolean }).complete) + } + const { db } = yield* Database.Service + expect( + yield* db + .select({ count: sql`count(*)` }) + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie, Effect.map((row) => row?.count)), + ).toBe(0) + }), + { git: true, config: { formatter: false, lsp: false } }, + 30_000, + ) + + it.instance( + "returns a scoped bounded snapshot before retained event history", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "snapshot resync", workspaceID: syncWorkspaceID }) + const { db } = yield* Database.Service + const sequence = yield* db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + const sync = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) + const snapshot = { + snapshotID: "snapshot_sync_http", + aggregateID: session.id, + throughSeq: sequence!.seq, + syncSeq: sync!.seq, + codec: "session-projection", + schemaVersion: 1, + snapshotHash: "1".repeat(64), + body: { format: "chunked-rows.v1" }, + ownerID: syncWorkspaceID, + createdAt: 1, + } + const rowHash = Hash.sha256("{}") + yield* db + .insert(EventSnapshotTable) + .values({ + snapshot_id: snapshot.snapshotID, + aggregate_id: snapshot.aggregateID, + through_seq: snapshot.throughSeq, + sync_seq: snapshot.syncSeq, + codec: snapshot.codec, + schema_version: snapshot.schemaVersion, + snapshot_hash: snapshot.snapshotHash, + body: snapshot.body, + owner_id: snapshot.ownerID, + created_at: snapshot.createdAt, + }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(EventSnapshotRowTable) + .values({ snapshot_id: snapshot.snapshotID, row_index: 0, table_name: "session", row_key: session.id, row_hash: rowHash, row_bytes: 2, chunk_count: 1, chain_hash: "2".repeat(64) }) + .run() + .pipe(Effect.orDie) + yield* db + .insert(EventSnapshotChunkTable) + .values({ row_hash: rowHash, chunk_index: 0, data: Buffer.from("{}"), chunk_hash: rowHash }) + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ snapshot_id: snapshot.snapshotID, retention_floor_seq: snapshot.throughSeq }) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .run() + .pipe(Effect.orDie) + const history = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1 }), + }) + expect(history.status, yield* history.text).toBe(200) + const envelope = (yield* history.json) as { items: { kind: string; snapshot: EventV2.SerializedSnapshot }[] } + expect(envelope.items).toHaveLength(1) + expect(envelope.items[0]).toMatchObject({ kind: "resync_required", snapshot: { snapshotID: snapshot.snapshotID } }) + + const rows = yield* requestInDirectory(SyncPaths.snapshotRows, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ + aggregateID: session.id, + snapshotID: snapshot.snapshotID, + snapshotHash: snapshot.snapshotHash, + limit: 1, + }), + }) + expect(rows.status, yield* rows.text).toBe(200) + const page = Schema.decodeUnknownSync( + Schema.Struct({ rows: Schema.Array(Schema.Struct({ rowHash: Schema.String })), complete: Schema.Boolean }), + )(yield* rows.json) + expect(page.rows).toHaveLength(1) + const chunks = yield* requestInDirectory(SyncPaths.snapshotChunks, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ + aggregateID: session.id, + snapshotID: snapshot.snapshotID, + snapshotHash: snapshot.snapshotHash, + rowHash: page.rows[0]!.rowHash, + limit: 1, + }), + }) + expect(chunks.status, yield* chunks.text).toBe(200) + const wrongScope = yield* requestInDirectory(SyncPaths.snapshotChunks, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ + aggregateID: session.id, + snapshotID: snapshot.snapshotID, + snapshotHash: "0".repeat(64), + rowHash: page.rows[0]!.rowHash, + limit: 1, + }), + }) + expect(wrongScope.status).toBe(404) + }), + { git: true, config: { formatter: false, lsp: false } }, + 20_000, + ) + + it.instance( + "serves file artifacts only through the exact workspace event binding", + () => + Effect.gen(function* () { + const tmp = yield* TestInstance + const headers = { "x-deepagent-code-directory": tmp.directory, "content-type": "application/json" } + const session = yield* Session.use.create({ title: "file artifact", workspaceID: syncWorkspaceID }) + const messageID = MessageID.ascending() + yield* Session.use.updateMessage({ + id: messageID, + sessionID: session.id, + role: "user", + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + }) + yield* Session.use.updatePart({ + id: SessionV1.PartID.ascending(), + sessionID: session.id, + messageID, + type: "file", + mime: "application/octet-stream", + url: `data:application/octet-stream;base64,${Buffer.alloc(FilePartArtifact.CHUNK_BYTES + 7, 0x63).toString("base64")}`, + }) + const { db } = yield* Database.Service + const event = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, session.id)) + .all() + .pipe(Effect.orDie, Effect.map((rows) => rows.find((row) => row.type === "message.part.updated.1"))) + expect(event).toBeDefined() + if (!event) return + const descriptor = FilePartArtifact.descriptor(event.data) + expect(descriptor).toBeDefined() + if (!descriptor) return + const scope = { eventID: event.id, aggregateID: session.id, seq: event.seq, artifactID: descriptor.id } + const metadataResponse = yield* requestInDirectory(SyncPaths.fileArtifactMetadata, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify(scope), + }) + expect(metadataResponse.status).toBe(200) + const metadata = Schema.decodeUnknownSync(FilePartArtifact.Metadata)(yield* metadataResponse.json) + expect(metadata.chunkHashes).toHaveLength(2) + const chunkResponse = yield* requestInDirectory(SyncPaths.fileArtifactChunk, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ ...scope, index: 0, hash: metadata.chunkHashes[0] }), + }) + expect(chunkResponse.status).toBe(200) + const chunk = Schema.decodeUnknownSync(Schema.Struct({ data: Schema.String }))(yield* chunkResponse.json) + expect(chunk.data).toHaveLength(4 * Math.ceil(FilePartArtifact.CHUNK_BYTES / 3)) + + const wrongEvent = yield* requestInDirectory(SyncPaths.fileArtifactChunk, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ ...scope, eventID: EventV2.ID.create(), index: 0, hash: metadata.chunkHashes[0] }), + }) + expect(wrongEvent.status).toBe(404) + const wrongSequence = yield* requestInDirectory(SyncPaths.fileArtifactChunk, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ ...scope, seq: event.seq + 1, index: 0, hash: metadata.chunkHashes[0] }), + }) + expect(wrongSequence.status).toBe(404) + const wrongHash = yield* requestInDirectory(SyncPaths.fileArtifactChunk, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ ...scope, index: 0, hash: "0".repeat(64) }), + }) + expect(wrongHash.status).toBe(404) + + Flag.DEEPAGENT_CODE_WORKSPACE_ID = WorkspaceV2.ID.make("wrk_other_scope") + const crossed = yield* requestInDirectory(SyncPaths.fileArtifactMetadata, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify(scope), + }) + expect(crossed.status).toBe(404) + }), + { git: true, config: { formatter: false, lsp: false } }, + 30_000, + ) + it.instance( "fails closed when history has no routed workspace scope", () => @@ -484,6 +786,13 @@ describe("sync HttpApi", () => { .where(eq(EventSequenceTable.aggregate_id, session.id)) .get() .pipe(Effect.orDie) + const sync = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) yield* db .insert(EventTable) .values({ @@ -492,6 +801,7 @@ describe("sync HttpApi", () => { seq: sequence!.seq + 1, type: "sync.legacy.1", data: { value: "x".repeat(SyncHistoryLimits.dataBytes + 1) }, + sync_seq: sync!.seq, }) .run() .pipe(Effect.orDie) @@ -523,6 +833,13 @@ describe("sync HttpApi", () => { .where(eq(EventSequenceTable.aggregate_id, session.id)) .get() .pipe(Effect.orDie) + const sync = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) yield* db .insert(EventTable) .values({ @@ -531,6 +848,7 @@ describe("sync HttpApi", () => { seq: sequence!.seq + 1, type: "sync.legacy.1", data: { value: "x".repeat(SyncHistoryLimits.dataBytes + 1) }, + sync_seq: sync!.seq, }) .run() .pipe(Effect.orDie) @@ -580,6 +898,13 @@ describe("sync HttpApi", () => { const unicode = "界".repeat(Math.floor(available / 3)) + "x".repeat(available % 3) const middle = { value: unicode } expect(Buffer.byteLength(JSON.stringify(middle))).toBe(target) + const sync = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 3` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) yield* db .insert(EventTable) .values([ @@ -589,6 +914,7 @@ describe("sync HttpApi", () => { seq: sequence!.seq + 1, type: "sync.test.1", data: { value: "before" }, + sync_seq: sync!.seq - 2, }, { id: EventV2.ID.make("evt_sync_wire_middle"), @@ -596,6 +922,7 @@ describe("sync HttpApi", () => { seq: sequence!.seq + 2, type: "sync.test.1", data: middle, + sync_seq: sync!.seq - 1, }, { id: EventV2.ID.make("evt_sync_wire_small_after"), @@ -603,6 +930,7 @@ describe("sync HttpApi", () => { seq: sequence!.seq + 3, type: "sync.test.1", data: { value: "after" }, + sync_seq: sync!.seq, }, ]) .run() diff --git a/packages/deepagent-code/test/session/diff-artifact.test.ts b/packages/deepagent-code/test/session/diff-artifact.test.ts new file mode 100644 index 000000000..de1508ae4 --- /dev/null +++ b/packages/deepagent-code/test/session/diff-artifact.test.ts @@ -0,0 +1,389 @@ +import { afterEach, describe, expect } from "bun:test" +import { Database } from "@deepagent-code/core/database/database" +import { EventV2 } from "@deepagent-code/core/event" +import { + EventArtifactChunkTable, + EventArtifactTable, + EventSequenceTable, + EventSyncSequenceTable, + EventTable, +} from "@deepagent-code/core/event/sql" +import { MessageTable, SessionHistoryStateTable, SessionTable } from "@deepagent-code/core/session/sql" +import { Hash } from "@deepagent-code/core/util/hash" +import { SessionV1 } from "@deepagent-code/core/v1/session" +import { ProviderV2 } from "@deepagent-code/core/provider" +import { ModelV2 } from "@deepagent-code/core/model" +import { HistoryAuthority } from "@/session/history-authority" +import { MessageV2 } from "@/session/message-v2" +import { MessageID } from "@/session/schema" +import { Session } from "@/session/session" +import { SessionDiffArtifact } from "@/session/diff-artifact" +import { SessionDiffMigrationReceiptTable } from "@/session/diff-artifact.sql" +import { SessionPromptEpochTable } from "@/session/prompt-epoch.sql" +import { SessionPaths } from "@/server/routes/instance/httpapi/groups/session" +import { and, asc, eq, sql } from "drizzle-orm" +import { Effect, Layer } from "effect" +import { HttpClientResponse } from "effect/unstable/http" +import { resetDatabase } from "../fixture/db" +import { disposeAllInstances, TestInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" +import { httpApiLayer, requestInDirectory } from "../server/httpapi-layer" + +const it = testEffect(Layer.mergeAll(Session.defaultLayer, Database.defaultLayer, EventV2.defaultLayer, httpApiLayer)) + +afterEach(async () => { + await disposeAllInstances() + await resetDatabase() +}) + +const seed = Effect.fn("Test.seedLegacyDiff")(function* (input?: { corruptMessage?: boolean }) { + const sessions = yield* Session.Service + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const session = yield* sessions.create({ title: "legacy diff" }) + const messageID = MessageID.ascending() + const patch = "你".repeat(Math.ceil((EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1024) / 3)) + const message = { + role: "user" as const, + time: { created: Date.now() }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, + summary: { + diffs: [{ file: "src/large.ts", patch, additions: 2, deletions: 1, status: "modified" as const }], + }, + } + yield* db + .insert(MessageTable) + .values({ + id: messageID, + session_id: session.id, + time_created: message.time.created, + time_updated: message.time.created, + data: message, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(SessionTable) + .set({ summary_diffs: message.summary.diffs }) + .where(eq(SessionTable.id, session.id)) + .run() + .pipe(Effect.orDie) + yield* db + .insert(SessionPromptEpochTable) + .values({ + session_id: session.id, + epoch: 0, + state: "active", + checkpoint_user_id: null, + checkpoint_assistant_id: null, + retained_tail_start_id: null, + source_end_message_id: null, + checkpoint_hash: null, + projection_version: HistoryAuthority.PROJECTION_VERSION, + canonicalization_version: HistoryAuthority.CANONICALIZATION_VERSION, + base_message_count: 0, + effective_history_hash: HistoryAuthority.hash([]), + first_window_id: "win_legacy_diff_first", + previous_window_id: null, + window_id: "win_legacy_diff_active", + world_state_baseline_hash: null, + authority_state: "ready", + recovery_reason: null, + reason: "bootstrap", + created_at: Date.now(), + retired_at: null, + }) + .run() + .pipe(Effect.orDie) + const sequence = yield* db + .select({ seq: EventSequenceTable.seq }) + .from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .get() + .pipe(Effect.orDie) + const eventID = EventV2.ID.make(`evt_legacy_diff_${messageID}`) + const syncSequence = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) + if (!syncSequence) return yield* Effect.die("sync sequence authority missing") + yield* db + .insert(EventTable) + .values({ + id: eventID, + aggregate_id: session.id, + seq: (sequence?.seq ?? -1) + 1, + type: EventV2.versionedType("message.updated", 1), + data: { sessionID: session.id, info: { ...message, id: messageID, sessionID: session.id } }, + sync_seq: syncSequence.seq, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ seq: (sequence?.seq ?? -1) + 1 }) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .run() + .pipe(Effect.orDie) + expect((yield* events.canonicalizeLegacyArtifacts({ limit: 1 })).processed).toBe(1) + const artifact = yield* db + .select() + .from(EventArtifactTable) + .where(eq(EventArtifactTable.event_id, eventID)) + .get() + .pipe(Effect.orDie) + if (!artifact) return yield* Effect.die("legacy artifact missing") + if (input?.corruptMessage) { + yield* db + .run(sql`UPDATE message SET data = json_set(data, '$.summary.diffs[0].patch', 'changed') WHERE id = ${messageID}`) + .pipe(Effect.orDie) + } + return { session, messageID, message, artifact, patch } +}) + +function pathFor(template: string, sessionID: string) { + return template.replace(":sessionID", sessionID) +} + +function request(path: string, init?: RequestInit) { + return TestInstance.pipe(Effect.flatMap((test) => requestInDirectory(path, test.directory, init))) +} + +function json(response: HttpClientResponse.HttpClientResponse) { + return response.json.pipe(Effect.map((body) => body as T)) +} + +describe("Session legacy diff physical migration", () => { + it.instance( + "revalidates committed chunks, preserves every PromptEpoch hash, CAS rewrites, and retries exactly", + () => + Effect.gen(function* () { + const input = yield* seed() + const { db } = yield* Database.Service + const before = HistoryAuthority.hash([{ info: { ...input.message, id: input.messageID, sessionID: input.session.id }, parts: [] }]) + + const maintenance = yield* request(pathFor(SessionPaths.diffArtifactMaintenance, input.session.id), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ limit: 1 }), + }) + expect(maintenance.status).toBe(200) + expect(yield* json<{ processed: number; committed: number; failed: number }>(maintenance)).toEqual({ + processed: 1, + committed: 1, + failed: 0, + }) + expect(yield* SessionDiffArtifact.migrate({ sessionID: input.session.id, now: 101 })).toEqual({ + processed: 0, + committed: 0, + failed: 0, + }) + const raw = yield* db.get<{ data: string }>(sql`SELECT CAST(data AS TEXT) AS data FROM message WHERE id = ${input.messageID}`) + expect(raw?.data).not.toContain('"patch"') + expect(raw?.data).not.toContain('"diffs"') + const sessionSummary = yield* db + .select({ diffs: SessionTable.summary_diffs }) + .from(SessionTable) + .where(eq(SessionTable.id, input.session.id)) + .get() + .pipe(Effect.orDie) + expect(sessionSummary?.diffs).toBeNull() + const stored = yield* MessageV2.get({ sessionID: input.session.id, messageID: input.messageID }) + expect(stored.info.role === "user" ? stored.info.summary?.diffs : undefined).toEqual([]) + expect(stored.info.role === "user" ? stored.info.summary?.diffArtifact?.id : undefined).toBe( + input.artifact.artifact_id, + ) + expect(HistoryAuthority.hash([stored])).toBe(before) + const receipt = yield* db + .select() + .from(SessionDiffMigrationReceiptTable) + .where(eq(SessionDiffMigrationReceiptTable.message_id, input.messageID)) + .get() + .pipe(Effect.orDie) + expect(receipt?.state).toBe("committed") + expect(receipt?.expected_session_summary_hash).not.toBe(receipt?.committed_session_summary_hash) + expect(receipt?.committed_session_summary_hash).toBe(Hash.sha256("null")) + expect(receipt?.epoch_hashes).toEqual([{ epoch: 0, before, after: before }]) + const manifestResponse = yield* request( + `${pathFor(SessionPaths.diffArtifactManifest, input.session.id)}?${new URLSearchParams({ + messageID: input.messageID, + artifactID: input.artifact.artifact_id, + limit: "1", + })}`, + ) + expect(manifestResponse.status).toBe(200) + const manifest = yield* json(manifestResponse) + expect(manifest.files).toEqual([ + expect.objectContaining({ file: "src/large.ts", patchBytes: Buffer.byteLength(input.patch) }), + ]) + const fileResponse = yield* request( + `${pathFor(SessionPaths.diffArtifactFile, input.session.id)}?${new URLSearchParams({ + messageID: input.messageID, + artifactID: input.artifact.artifact_id, + path: "src/large.ts", + maxBytes: "1024", + })}`, + ) + expect(fileResponse.status).toBe(200) + const file = yield* json(fileResponse) + expect(file.truncated).toBe(true) + expect(file.returnedBytes).toBeLessThanOrEqual(1024) + expect(Buffer.from(file.patch).toString()).toBe(file.patch) + + const other = yield* Session.Service.pipe(Effect.flatMap((sessions) => sessions.create({ title: "other" }))) + const unauthorized = yield* request( + `${pathFor(SessionPaths.diffArtifactManifest, other.id)}?${new URLSearchParams({ + messageID: input.messageID, + artifactID: input.artifact.artifact_id, + })}`, + ) + expect(unauthorized.status).toBe(404) + }), + { git: true }, + 30_000, + ) + + it.instance( + "records hash mismatch without mutating message or history recovery authority", + () => + Effect.gen(function* () { + const input = yield* seed({ corruptMessage: true }) + const { db } = yield* Database.Service + expect(yield* SessionDiffArtifact.migrate({ sessionID: input.session.id, now: 200 })).toEqual({ + processed: 1, + committed: 0, + failed: 1, + }) + const raw = yield* db.get<{ patch: string }>(sql` + SELECT json_extract(data, '$.summary.diffs[0].patch') AS patch FROM message WHERE id = ${input.messageID} + `) + expect(raw?.patch).toBe("changed") + const receipt = yield* db + .select() + .from(SessionDiffMigrationReceiptTable) + .where(eq(SessionDiffMigrationReceiptTable.message_id, input.messageID)) + .get() + .pipe(Effect.orDie) + expect(receipt?.state).toBe("migration_validation_failed") + expect(receipt?.failure_reason).toContain("no longer matches") + expect( + yield* db + .select() + .from(SessionHistoryStateTable) + .where(eq(SessionHistoryStateTable.session_id, input.session.id)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + const epoch = yield* db + .select({ authority: SessionPromptEpochTable.authority_state }) + .from(SessionPromptEpochTable) + .where(eq(SessionPromptEpochTable.session_id, input.session.id)) + .get() + .pipe(Effect.orDie) + expect(epoch?.authority).toBe("ready") + }), + { git: true }, + 30_000, + ) + + it.instance( + "fails closed when a committed artifact chunk is corrupted", + () => + Effect.gen(function* () { + const input = yield* seed() + const { db } = yield* Database.Service + const chunk = yield* db + .select() + .from(EventArtifactChunkTable) + .where(eq(EventArtifactChunkTable.artifact_id, input.artifact.artifact_id)) + .orderBy(asc(EventArtifactChunkTable.chunk_index)) + .get() + .pipe(Effect.orDie) + yield* db + .update(EventArtifactChunkTable) + .set({ data: Buffer.from("corrupt") }) + .where( + and( + eq(EventArtifactChunkTable.artifact_id, input.artifact.artifact_id), + eq(EventArtifactChunkTable.chunk_index, chunk!.chunk_index), + ), + ) + .run() + .pipe(Effect.orDie) + expect((yield* SessionDiffArtifact.migrate({ sessionID: input.session.id })).failed).toBe(1) + const row = yield* db + .select({ data: MessageTable.data }) + .from(MessageTable) + .where(eq(MessageTable.id, input.messageID)) + .get() + .pipe(Effect.orDie) + expect( + row?.data.role === "user" && row.data.summary && typeof row.data.summary === "object" + ? row.data.summary.diffs[0]?.patch + : undefined, + ).toBe(input.patch) + }), + { git: true }, + 30_000, + ) + + it.instance( + "rolls back message, Session summary, and sidecars before recording a CAS failure", + () => + Effect.gen(function* () { + const input = yield* seed() + const { db } = yield* Database.Service + yield* db + .run(sql.raw(` + CREATE TRIGGER test_diff_session_cas_loss + BEFORE UPDATE OF summary_diffs ON session + WHEN OLD.id = '${input.session.id}' + BEGIN + SELECT RAISE(IGNORE); + END + `)) + .pipe(Effect.orDie) + + expect(yield* SessionDiffArtifact.migrate({ sessionID: input.session.id, now: 300 })).toEqual({ + processed: 1, + committed: 0, + failed: 1, + }) + const message = yield* db + .select({ data: MessageTable.data }) + .from(MessageTable) + .where(eq(MessageTable.id, input.messageID)) + .get() + .pipe(Effect.orDie) + expect( + message?.data.role === "user" && message.data.summary && typeof message.data.summary === "object" + ? message.data.summary.diffs[0]?.patch + : undefined, + ).toBe(input.patch) + expect( + ( + yield* db + .select({ diffs: SessionTable.summary_diffs }) + .from(SessionTable) + .where(eq(SessionTable.id, input.session.id)) + .get() + .pipe(Effect.orDie) + )?.diffs?.[0]?.patch, + ).toBe(input.patch) + expect( + yield* db + .select() + .from(SessionDiffMigrationReceiptTable) + .where(eq(SessionDiffMigrationReceiptTable.message_id, input.messageID)) + .get() + .pipe(Effect.orDie), + ).toEqual(expect.objectContaining({ state: "migration_validation_failed", failure_reason: "Session summary compare-and-swap lost" })) + }), + { git: true }, + 30_000, + ) +}) diff --git a/packages/sdk/js/src/gen/sdk.gen.ts b/packages/sdk/js/src/gen/sdk.gen.ts index b8305cc02..bb1b21194 100644 --- a/packages/sdk/js/src/gen/sdk.gen.ts +++ b/packages/sdk/js/src/gen/sdk.gen.ts @@ -387,6 +387,10 @@ import type { SessionDeleteMessageErrors, SessionDeleteMessageResponses, SessionDeleteResponses, + SessionDiffArtifactFileErrors, + SessionDiffArtifactFileResponses, + SessionDiffArtifactManifestErrors, + SessionDiffArtifactManifestResponses, SessionDiffErrors, SessionDiffResponses, SessionForkErrors, @@ -436,10 +440,18 @@ import type { SessionUpdateErrors, SessionUpdateResponses, SubtaskPartInput, + SyncArtifactFileChunkErrors, + SyncArtifactFileChunkResponses, + SyncArtifactFileMetadataErrors, + SyncArtifactFileMetadataResponses, SyncHistoryListErrors, SyncHistoryListResponses, SyncReplayErrors, SyncReplayResponses, + SyncSnapshotChunksErrors, + SyncSnapshotChunksResponses, + SyncSnapshotRowsErrors, + SyncSnapshotRowsResponses, SyncStartErrors, SyncStartResponses, TextPartInput, @@ -7401,6 +7413,94 @@ export class Session2 extends HeyApiClient { }) } + /** + * Get a Session diff artifact manifest page + * + * Read only metadata for an artifact committed to the addressed Session message. + */ + public diffArtifactManifest( + parameters: { + sessionID: string + directory?: string + workspace?: string + messageID: string + artifactID: string + cursor?: string + limit?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "query", key: "messageID" }, + { in: "query", key: "artifactID" }, + { in: "query", key: "cursor" }, + { in: "query", key: "limit" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get< + SessionDiffArtifactManifestResponses, + SessionDiffArtifactManifestErrors, + ThrowOnError + >({ + url: "/session/{sessionID}/diff-artifact/manifest", + ...options, + ...params, + }) + } + + /** + * Get one bounded Session diff artifact file patch + * + * Verify content-addressed chunks and return at most the requested UTF-8 byte budget for one authorized path. + */ + public diffArtifactFile( + parameters: { + sessionID: string + directory?: string + workspace?: string + messageID: string + artifactID: string + path: string + maxBytes?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "query", key: "messageID" }, + { in: "query", key: "artifactID" }, + { in: "query", key: "path" }, + { in: "query", key: "maxBytes" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get< + SessionDiffArtifactFileResponses, + SessionDiffArtifactFileErrors, + ThrowOnError + >({ + url: "/session/{sessionID}/diff-artifact/file", + ...options, + ...params, + }) + } + /** * Get session messages * @@ -8509,6 +8609,113 @@ export class History extends HeyApiClient { } } +export class File2 extends HeyApiClient { + /** + * Read scoped file-part artifact metadata + * + * Resolve a content-addressed file-part descriptor only through its exact workspace event binding. + */ + public metadata( + parameters: { + directory?: string + workspace?: string + eventID: string + aggregateID: string + seq: number + artifactID: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "eventID" }, + { in: "body", key: "aggregateID" }, + { in: "body", key: "seq" }, + { in: "body", key: "artifactID" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + SyncArtifactFileMetadataResponses, + SyncArtifactFileMetadataErrors, + ThrowOnError + >({ + url: "/sync/artifact/file/metadata", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read one verified file-part artifact chunk + * + * Return one bounded base64 chunk after workspace scope and expected hash validation. + */ + public chunk( + parameters: { + directory?: string + workspace?: string + eventID: string + aggregateID: string + seq: number + artifactID: string + index: number + hash: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "eventID" }, + { in: "body", key: "aggregateID" }, + { in: "body", key: "seq" }, + { in: "body", key: "artifactID" }, + { in: "body", key: "index" }, + { in: "body", key: "hash" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + SyncArtifactFileChunkResponses, + SyncArtifactFileChunkErrors, + ThrowOnError + >({ + url: "/sync/artifact/file/chunk", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Artifact extends HeyApiClient { + private _file?: File2 + get file(): File2 { + return (this._file ??= new File2({ client: this.client })) + } +} + export class Sync extends HeyApiClient { /** * Start workspace sync @@ -8595,10 +8802,97 @@ export class Sync extends HeyApiClient { }) } + public snapshotRows( + parameters: { + directory?: string + workspace?: string + aggregateID: string + snapshotID: string + snapshotHash: string + after?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + limit?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "aggregateID" }, + { in: "body", key: "snapshotID" }, + { in: "body", key: "snapshotHash" }, + { in: "body", key: "after" }, + { in: "body", key: "limit" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/sync/snapshot/rows", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + public snapshotChunks( + parameters: { + directory?: string + workspace?: string + aggregateID: string + snapshotID: string + snapshotHash: string + rowHash: string + after?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + limit?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "aggregateID" }, + { in: "body", key: "snapshotID" }, + { in: "body", key: "snapshotHash" }, + { in: "body", key: "rowHash" }, + { in: "body", key: "after" }, + { in: "body", key: "limit" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/sync/snapshot/chunks", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + private _history?: History get history(): History { return (this._history ??= new History({ client: this.client })) } + + private _artifact?: Artifact + get artifact(): Artifact { + return (this._artifact ??= new Artifact({ client: this.client })) + } } export class Control extends HeyApiClient { diff --git a/packages/sdk/js/src/gen/types.gen.ts b/packages/sdk/js/src/gen/types.gen.ts index ccee0dbd1..b752bebb7 100644 --- a/packages/sdk/js/src/gen/types.gen.ts +++ b/packages/sdk/js/src/gen/types.gen.ts @@ -568,6 +568,14 @@ export type FilePart = { mime: string filename?: string url: string + artifact?: { + codec: "file-part.v1" + id: string + hash: string + bytes: number + chunkBytes: 262144 + chunks: number + } source?: FilePartSource } @@ -5432,6 +5440,28 @@ export type SessionLegacyProviderResolutionDescriptor = worldStateBaselineHash?: string } +export type FilePartArtifactDescriptor = { + codec: "file-part.v1" + id: string + hash: string + bytes: number + chunkBytes: 262144 + chunks: number +} + +export type FilePartArtifactMetadata = { + eventID: string + aggregateID: string + seq: number + originalDataHash: string + canonicalDataHash: string + canonicalData: { + [key: string]: unknown + } + descriptor: FilePartArtifactDescriptor + chunkHashes: Array +} + export type LocationInfo = { directory: string workspaceID?: string @@ -13574,6 +13604,112 @@ export type SessionDiffResponses = { export type SessionDiffResponse = SessionDiffResponses[keyof SessionDiffResponses] +export type SessionDiffArtifactManifestData = { + body?: never + path: { + sessionID: string + } + query: { + directory?: string + workspace?: string + messageID: string + artifactID: string + cursor?: string + limit?: string + } + url: "/session/{sessionID}/diff-artifact/manifest" +} + +export type SessionDiffArtifactManifestErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * NotFoundError + */ + 404: NotFoundError +} + +export type SessionDiffArtifactManifestError = + SessionDiffArtifactManifestErrors[keyof SessionDiffArtifactManifestErrors] + +export type SessionDiffArtifactManifestResponses = { + /** + * Bounded Session diff artifact manifest page + */ + 200: { + artifact: { + id: string + hash: string + codec: "legacy-message-diff.v1" | "legacy-message-diff.v2" + fileCount: number + } + files: Array<{ + file: string + additions: number + deletions: number + status?: "added" | "deleted" | "modified" + patchBytes: number + patchHash: string + }> + nextCursor?: string + complete: boolean + } +} + +export type SessionDiffArtifactManifestResponse = + SessionDiffArtifactManifestResponses[keyof SessionDiffArtifactManifestResponses] + +export type SessionDiffArtifactFileData = { + body?: never + path: { + sessionID: string + } + query: { + directory?: string + workspace?: string + messageID: string + artifactID: string + path: string + maxBytes?: string + } + url: "/session/{sessionID}/diff-artifact/file" +} + +export type SessionDiffArtifactFileErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * NotFoundError + */ + 404: NotFoundError +} + +export type SessionDiffArtifactFileError = SessionDiffArtifactFileErrors[keyof SessionDiffArtifactFileErrors] + +export type SessionDiffArtifactFileResponses = { + /** + * Bounded Session diff artifact file patch + */ + 200: { + artifactID: string + file: string + additions: number + deletions: number + status?: "added" | "deleted" | "modified" + patch: string + patchBytes: number + returnedBytes: number + patchHash: string + truncated: boolean + } +} + +export type SessionDiffArtifactFileResponse = SessionDiffArtifactFileResponses[keyof SessionDiffArtifactFileResponses] + export type SessionMessagesData = { body?: never path: { @@ -15280,16 +15416,35 @@ export type SyncHistoryListResponses = { */ 200: { version: 1 - items: Array<{ - kind: "event" - id: string - aggregate_id: string - seq: number - type: string - data: { - [key: string]: unknown - } - }> + items: Array< + | { + kind: "event" + id: string + aggregate_id: string + seq: number + type: string + data: { + [key: string]: unknown + } + } + | { + kind: "resync_required" + snapshot: { + snapshotID: string + aggregateID: string + throughSeq: number + syncSeq: number + codec: string + schemaVersion: number + snapshotHash: string + body: { + [key: string]: unknown + } + ownerID?: string + createdAt: number + } + } + > nextCursor: string complete: boolean } @@ -15297,6 +15452,193 @@ export type SyncHistoryListResponses = { export type SyncHistoryListResponse = SyncHistoryListResponses[keyof SyncHistoryListResponses] +export type SyncArtifactFileMetadataData = { + body?: { + eventID: string + aggregateID: string + seq: number + artifactID: string + } + path?: never + query?: { + directory?: string + workspace?: string + } + url: "/sync/artifact/file/metadata" +} + +export type SyncArtifactFileMetadataErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type SyncArtifactFileMetadataError = SyncArtifactFileMetadataErrors[keyof SyncArtifactFileMetadataErrors] + +export type SyncArtifactFileMetadataResponses = { + /** + * Scoped file-part artifact metadata + */ + 200: FilePartArtifactMetadata +} + +export type SyncArtifactFileMetadataResponse = + SyncArtifactFileMetadataResponses[keyof SyncArtifactFileMetadataResponses] + +export type SyncArtifactFileChunkData = { + body?: { + eventID: string + aggregateID: string + seq: number + artifactID: string + index: number + hash: string + } + path?: never + query?: { + directory?: string + workspace?: string + } + url: "/sync/artifact/file/chunk" +} + +export type SyncArtifactFileChunkErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type SyncArtifactFileChunkError = SyncArtifactFileChunkErrors[keyof SyncArtifactFileChunkErrors] + +export type SyncArtifactFileChunkResponses = { + /** + * One verified file-part artifact chunk + */ + 200: { + artifactID: string + index: number + hash: string + data: string + } +} + +export type SyncArtifactFileChunkResponse = SyncArtifactFileChunkResponses[keyof SyncArtifactFileChunkResponses] + +export type SyncSnapshotRowsData = { + body?: { + aggregateID: string + snapshotID: string + snapshotHash: string + after?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + limit?: number + } + path?: never + query?: { + directory?: string + workspace?: string + } + url: "/sync/snapshot/rows" +} + +export type SyncSnapshotRowsErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * Not found + */ + 404: NotFoundError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError +} + +export type SyncSnapshotRowsError = SyncSnapshotRowsErrors[keyof SyncSnapshotRowsErrors] + +export type SyncSnapshotRowsResponses = { + /** + * One bounded scoped snapshot row page + */ + 200: { + rows: Array<{ + snapshotID: string + rowIndex: number + tableName: string + rowKey: string + rowHash: string + rowBytes: number + chunkCount: number + chainHash: string + }> + complete: boolean + } +} + +export type SyncSnapshotRowsResponse = SyncSnapshotRowsResponses[keyof SyncSnapshotRowsResponses] + +export type SyncSnapshotChunksData = { + body?: { + aggregateID: string + snapshotID: string + snapshotHash: string + rowHash: string + after?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + limit?: number + } + path?: never + query?: { + directory?: string + workspace?: string + } + url: "/sync/snapshot/chunks" +} + +export type SyncSnapshotChunksErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * Not found + */ + 404: NotFoundError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError +} + +export type SyncSnapshotChunksError = SyncSnapshotChunksErrors[keyof SyncSnapshotChunksErrors] + +export type SyncSnapshotChunksResponses = { + /** + * One bounded scoped snapshot chunk page + */ + 200: { + chunks: Array<{ + rowHash: string + chunkIndex: number + data: string + chunkHash: string + }> + complete: boolean + } +} + +export type SyncSnapshotChunksResponse = SyncSnapshotChunksResponses[keyof SyncSnapshotChunksResponses] + export type TuiAppendPromptData = { body?: { text: string diff --git a/packages/sdk/js/src/v2/gen/sdk.gen.ts b/packages/sdk/js/src/v2/gen/sdk.gen.ts index b8305cc02..bb1b21194 100644 --- a/packages/sdk/js/src/v2/gen/sdk.gen.ts +++ b/packages/sdk/js/src/v2/gen/sdk.gen.ts @@ -387,6 +387,10 @@ import type { SessionDeleteMessageErrors, SessionDeleteMessageResponses, SessionDeleteResponses, + SessionDiffArtifactFileErrors, + SessionDiffArtifactFileResponses, + SessionDiffArtifactManifestErrors, + SessionDiffArtifactManifestResponses, SessionDiffErrors, SessionDiffResponses, SessionForkErrors, @@ -436,10 +440,18 @@ import type { SessionUpdateErrors, SessionUpdateResponses, SubtaskPartInput, + SyncArtifactFileChunkErrors, + SyncArtifactFileChunkResponses, + SyncArtifactFileMetadataErrors, + SyncArtifactFileMetadataResponses, SyncHistoryListErrors, SyncHistoryListResponses, SyncReplayErrors, SyncReplayResponses, + SyncSnapshotChunksErrors, + SyncSnapshotChunksResponses, + SyncSnapshotRowsErrors, + SyncSnapshotRowsResponses, SyncStartErrors, SyncStartResponses, TextPartInput, @@ -7401,6 +7413,94 @@ export class Session2 extends HeyApiClient { }) } + /** + * Get a Session diff artifact manifest page + * + * Read only metadata for an artifact committed to the addressed Session message. + */ + public diffArtifactManifest( + parameters: { + sessionID: string + directory?: string + workspace?: string + messageID: string + artifactID: string + cursor?: string + limit?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "query", key: "messageID" }, + { in: "query", key: "artifactID" }, + { in: "query", key: "cursor" }, + { in: "query", key: "limit" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get< + SessionDiffArtifactManifestResponses, + SessionDiffArtifactManifestErrors, + ThrowOnError + >({ + url: "/session/{sessionID}/diff-artifact/manifest", + ...options, + ...params, + }) + } + + /** + * Get one bounded Session diff artifact file patch + * + * Verify content-addressed chunks and return at most the requested UTF-8 byte budget for one authorized path. + */ + public diffArtifactFile( + parameters: { + sessionID: string + directory?: string + workspace?: string + messageID: string + artifactID: string + path: string + maxBytes?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "path", key: "sessionID" }, + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "query", key: "messageID" }, + { in: "query", key: "artifactID" }, + { in: "query", key: "path" }, + { in: "query", key: "maxBytes" }, + ], + }, + ], + ) + return (options?.client ?? this.client).get< + SessionDiffArtifactFileResponses, + SessionDiffArtifactFileErrors, + ThrowOnError + >({ + url: "/session/{sessionID}/diff-artifact/file", + ...options, + ...params, + }) + } + /** * Get session messages * @@ -8509,6 +8609,113 @@ export class History extends HeyApiClient { } } +export class File2 extends HeyApiClient { + /** + * Read scoped file-part artifact metadata + * + * Resolve a content-addressed file-part descriptor only through its exact workspace event binding. + */ + public metadata( + parameters: { + directory?: string + workspace?: string + eventID: string + aggregateID: string + seq: number + artifactID: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "eventID" }, + { in: "body", key: "aggregateID" }, + { in: "body", key: "seq" }, + { in: "body", key: "artifactID" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + SyncArtifactFileMetadataResponses, + SyncArtifactFileMetadataErrors, + ThrowOnError + >({ + url: "/sync/artifact/file/metadata", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Read one verified file-part artifact chunk + * + * Return one bounded base64 chunk after workspace scope and expected hash validation. + */ + public chunk( + parameters: { + directory?: string + workspace?: string + eventID: string + aggregateID: string + seq: number + artifactID: string + index: number + hash: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "eventID" }, + { in: "body", key: "aggregateID" }, + { in: "body", key: "seq" }, + { in: "body", key: "artifactID" }, + { in: "body", key: "index" }, + { in: "body", key: "hash" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post< + SyncArtifactFileChunkResponses, + SyncArtifactFileChunkErrors, + ThrowOnError + >({ + url: "/sync/artifact/file/chunk", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + +export class Artifact extends HeyApiClient { + private _file?: File2 + get file(): File2 { + return (this._file ??= new File2({ client: this.client })) + } +} + export class Sync extends HeyApiClient { /** * Start workspace sync @@ -8595,10 +8802,97 @@ export class Sync extends HeyApiClient { }) } + public snapshotRows( + parameters: { + directory?: string + workspace?: string + aggregateID: string + snapshotID: string + snapshotHash: string + after?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + limit?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "aggregateID" }, + { in: "body", key: "snapshotID" }, + { in: "body", key: "snapshotHash" }, + { in: "body", key: "after" }, + { in: "body", key: "limit" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/sync/snapshot/rows", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + public snapshotChunks( + parameters: { + directory?: string + workspace?: string + aggregateID: string + snapshotID: string + snapshotHash: string + rowHash: string + after?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + limit?: number + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "query", key: "directory" }, + { in: "query", key: "workspace" }, + { in: "body", key: "aggregateID" }, + { in: "body", key: "snapshotID" }, + { in: "body", key: "snapshotHash" }, + { in: "body", key: "rowHash" }, + { in: "body", key: "after" }, + { in: "body", key: "limit" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/sync/snapshot/chunks", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + private _history?: History get history(): History { return (this._history ??= new History({ client: this.client })) } + + private _artifact?: Artifact + get artifact(): Artifact { + return (this._artifact ??= new Artifact({ client: this.client })) + } } export class Control extends HeyApiClient { diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index ccee0dbd1..b752bebb7 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -568,6 +568,14 @@ export type FilePart = { mime: string filename?: string url: string + artifact?: { + codec: "file-part.v1" + id: string + hash: string + bytes: number + chunkBytes: 262144 + chunks: number + } source?: FilePartSource } @@ -5432,6 +5440,28 @@ export type SessionLegacyProviderResolutionDescriptor = worldStateBaselineHash?: string } +export type FilePartArtifactDescriptor = { + codec: "file-part.v1" + id: string + hash: string + bytes: number + chunkBytes: 262144 + chunks: number +} + +export type FilePartArtifactMetadata = { + eventID: string + aggregateID: string + seq: number + originalDataHash: string + canonicalDataHash: string + canonicalData: { + [key: string]: unknown + } + descriptor: FilePartArtifactDescriptor + chunkHashes: Array +} + export type LocationInfo = { directory: string workspaceID?: string @@ -13574,6 +13604,112 @@ export type SessionDiffResponses = { export type SessionDiffResponse = SessionDiffResponses[keyof SessionDiffResponses] +export type SessionDiffArtifactManifestData = { + body?: never + path: { + sessionID: string + } + query: { + directory?: string + workspace?: string + messageID: string + artifactID: string + cursor?: string + limit?: string + } + url: "/session/{sessionID}/diff-artifact/manifest" +} + +export type SessionDiffArtifactManifestErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * NotFoundError + */ + 404: NotFoundError +} + +export type SessionDiffArtifactManifestError = + SessionDiffArtifactManifestErrors[keyof SessionDiffArtifactManifestErrors] + +export type SessionDiffArtifactManifestResponses = { + /** + * Bounded Session diff artifact manifest page + */ + 200: { + artifact: { + id: string + hash: string + codec: "legacy-message-diff.v1" | "legacy-message-diff.v2" + fileCount: number + } + files: Array<{ + file: string + additions: number + deletions: number + status?: "added" | "deleted" | "modified" + patchBytes: number + patchHash: string + }> + nextCursor?: string + complete: boolean + } +} + +export type SessionDiffArtifactManifestResponse = + SessionDiffArtifactManifestResponses[keyof SessionDiffArtifactManifestResponses] + +export type SessionDiffArtifactFileData = { + body?: never + path: { + sessionID: string + } + query: { + directory?: string + workspace?: string + messageID: string + artifactID: string + path: string + maxBytes?: string + } + url: "/session/{sessionID}/diff-artifact/file" +} + +export type SessionDiffArtifactFileErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * NotFoundError + */ + 404: NotFoundError +} + +export type SessionDiffArtifactFileError = SessionDiffArtifactFileErrors[keyof SessionDiffArtifactFileErrors] + +export type SessionDiffArtifactFileResponses = { + /** + * Bounded Session diff artifact file patch + */ + 200: { + artifactID: string + file: string + additions: number + deletions: number + status?: "added" | "deleted" | "modified" + patch: string + patchBytes: number + returnedBytes: number + patchHash: string + truncated: boolean + } +} + +export type SessionDiffArtifactFileResponse = SessionDiffArtifactFileResponses[keyof SessionDiffArtifactFileResponses] + export type SessionMessagesData = { body?: never path: { @@ -15280,16 +15416,35 @@ export type SyncHistoryListResponses = { */ 200: { version: 1 - items: Array<{ - kind: "event" - id: string - aggregate_id: string - seq: number - type: string - data: { - [key: string]: unknown - } - }> + items: Array< + | { + kind: "event" + id: string + aggregate_id: string + seq: number + type: string + data: { + [key: string]: unknown + } + } + | { + kind: "resync_required" + snapshot: { + snapshotID: string + aggregateID: string + throughSeq: number + syncSeq: number + codec: string + schemaVersion: number + snapshotHash: string + body: { + [key: string]: unknown + } + ownerID?: string + createdAt: number + } + } + > nextCursor: string complete: boolean } @@ -15297,6 +15452,193 @@ export type SyncHistoryListResponses = { export type SyncHistoryListResponse = SyncHistoryListResponses[keyof SyncHistoryListResponses] +export type SyncArtifactFileMetadataData = { + body?: { + eventID: string + aggregateID: string + seq: number + artifactID: string + } + path?: never + query?: { + directory?: string + workspace?: string + } + url: "/sync/artifact/file/metadata" +} + +export type SyncArtifactFileMetadataErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type SyncArtifactFileMetadataError = SyncArtifactFileMetadataErrors[keyof SyncArtifactFileMetadataErrors] + +export type SyncArtifactFileMetadataResponses = { + /** + * Scoped file-part artifact metadata + */ + 200: FilePartArtifactMetadata +} + +export type SyncArtifactFileMetadataResponse = + SyncArtifactFileMetadataResponses[keyof SyncArtifactFileMetadataResponses] + +export type SyncArtifactFileChunkData = { + body?: { + eventID: string + aggregateID: string + seq: number + artifactID: string + index: number + hash: string + } + path?: never + query?: { + directory?: string + workspace?: string + } + url: "/sync/artifact/file/chunk" +} + +export type SyncArtifactFileChunkErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * Not found + */ + 404: NotFoundError +} + +export type SyncArtifactFileChunkError = SyncArtifactFileChunkErrors[keyof SyncArtifactFileChunkErrors] + +export type SyncArtifactFileChunkResponses = { + /** + * One verified file-part artifact chunk + */ + 200: { + artifactID: string + index: number + hash: string + data: string + } +} + +export type SyncArtifactFileChunkResponse = SyncArtifactFileChunkResponses[keyof SyncArtifactFileChunkResponses] + +export type SyncSnapshotRowsData = { + body?: { + aggregateID: string + snapshotID: string + snapshotHash: string + after?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + limit?: number + } + path?: never + query?: { + directory?: string + workspace?: string + } + url: "/sync/snapshot/rows" +} + +export type SyncSnapshotRowsErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * Not found + */ + 404: NotFoundError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError +} + +export type SyncSnapshotRowsError = SyncSnapshotRowsErrors[keyof SyncSnapshotRowsErrors] + +export type SyncSnapshotRowsResponses = { + /** + * One bounded scoped snapshot row page + */ + 200: { + rows: Array<{ + snapshotID: string + rowIndex: number + tableName: string + rowKey: string + rowHash: string + rowBytes: number + chunkCount: number + chainHash: string + }> + complete: boolean + } +} + +export type SyncSnapshotRowsResponse = SyncSnapshotRowsResponses[keyof SyncSnapshotRowsResponses] + +export type SyncSnapshotChunksData = { + body?: { + aggregateID: string + snapshotID: string + snapshotHash: string + rowHash: string + after?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + limit?: number + } + path?: never + query?: { + directory?: string + workspace?: string + } + url: "/sync/snapshot/chunks" +} + +export type SyncSnapshotChunksErrors = { + /** + * BadRequest | InvalidRequestError + */ + 400: EffectHttpApiErrorBadRequest | InvalidRequestError + /** + * Not found + */ + 404: NotFoundError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError +} + +export type SyncSnapshotChunksError = SyncSnapshotChunksErrors[keyof SyncSnapshotChunksErrors] + +export type SyncSnapshotChunksResponses = { + /** + * One bounded scoped snapshot chunk page + */ + 200: { + chunks: Array<{ + rowHash: string + chunkIndex: number + data: string + chunkHash: string + }> + complete: boolean + } +} + +export type SyncSnapshotChunksResponse = SyncSnapshotChunksResponses[keyof SyncSnapshotChunksResponses] + export type TuiAppendPromptData = { body?: { text: string From e323f19c734cd67bb9bf754340cc2985870c6b69 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 16:41:25 +0800 Subject: [PATCH 23/44] fix(core): preserve synthetic file artifact metadata --- packages/core/src/file-part-artifact.ts | 18 +++++++++++++----- packages/core/src/v1/session.ts | 1 + packages/core/test/file-part-artifact.test.ts | 9 +++++++-- packages/sdk/js/src/gen/types.gen.ts | 1 + packages/sdk/js/src/v2/gen/types.gen.ts | 1 + 5 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/core/src/file-part-artifact.ts b/packages/core/src/file-part-artifact.ts index 7db78e111..e28c5d58b 100644 --- a/packages/core/src/file-part-artifact.ts +++ b/packages/core/src/file-part-artifact.ts @@ -713,16 +713,24 @@ export function canonicalizeLegacy( const sourceData = Object.fromEntries( Object.entries(sourcePart).filter(([key]) => key !== "id" && key !== "sessionID" && key !== "messageID"), ) - if (!isDeepStrictEqual(projected.data, sourceData)) + // Older FilePart event schemas dropped the runtime-owned synthetic marker while + // the local Part projection retained it. Accept only that exact historical skew. + const legacySynthetic = + !("synthetic" in sourceData) && "synthetic" in projected.data && projected.data.synthetic === true + const expectedData = legacySynthetic ? { ...sourceData, synthetic: true } : sourceData + if (!isDeepStrictEqual(projected.data, expectedData)) return yield* fail( prepared.artifacts[0]!.descriptor.id, `Legacy file-part event ${row.id} no longer matches its projected part`, ) - const canonicalData = Object.fromEntries( - Object.entries(canonicalPart as Record).filter( - ([key]) => key !== "id" && key !== "sessionID" && key !== "messageID", + const canonicalData = { + ...Object.fromEntries( + Object.entries(canonicalPart as Record).filter( + ([key]) => key !== "id" && key !== "sessionID" && key !== "messageID", + ), ), - ) as typeof PartTable.$inferInsert.data + ...(legacySynthetic ? { synthetic: true } : {}), + } as typeof PartTable.$inferInsert.data yield* db .update(PartTable) .set({ data: canonicalData }) diff --git a/packages/core/src/v1/session.ts b/packages/core/src/v1/session.ts index 4a39ab89f..8e5cc0010 100644 --- a/packages/core/src/v1/session.ts +++ b/packages/core/src/v1/session.ts @@ -189,6 +189,7 @@ export const FilePart = Schema.Struct({ mime: Schema.String, filename: Schema.optional(Schema.String), url: Schema.String, + synthetic: Schema.optional(Schema.Boolean), artifact: Schema.optional( Schema.Struct({ codec: Schema.Literal("file-part.v1"), diff --git a/packages/core/test/file-part-artifact.test.ts b/packages/core/test/file-part-artifact.test.ts index 3cfb93a73..293bc70dc 100644 --- a/packages/core/test/file-part-artifact.test.ts +++ b/packages/core/test/file-part-artifact.test.ts @@ -91,7 +91,8 @@ describe("FilePartArtifact", () => { message_id: messageID, session_id: sessionID, time_created: 1, - data: { type: "file", mime: "application/octet-stream", url: data.part.url }, + // v4.0.4 kept this runtime marker in Part while its FilePart event schema dropped it. + data: { type: "file", mime: "application/octet-stream", url: data.part.url, synthetic: true }, } as typeof PartTable.$inferInsert) .run() .pipe(Effect.orDie) @@ -143,7 +144,11 @@ describe("FilePartArtifact", () => { expect(Buffer.byteLength(JSON.stringify(binding!.canonical_data))).toBeLessThan(2_000) const projected = yield* db.select().from(PartTable).where(eq(PartTable.id, partID)).get().pipe(Effect.orDie) expect(Buffer.byteLength(JSON.stringify(projected!.data))).toBeLessThan(1_000) - expect(projected!.data).toMatchObject({ url: `artifact:${descriptor.id}`, artifact: descriptor }) + expect(projected!.data).toMatchObject({ + url: `artifact:${descriptor.id}`, + artifact: descriptor, + synthetic: true, + }) expect( yield* FilePartArtifact.read({ aggregateID: sessionID, descriptor }).pipe(Effect.provide(source)), ).toEqual(body) diff --git a/packages/sdk/js/src/gen/types.gen.ts b/packages/sdk/js/src/gen/types.gen.ts index b752bebb7..1dfc35757 100644 --- a/packages/sdk/js/src/gen/types.gen.ts +++ b/packages/sdk/js/src/gen/types.gen.ts @@ -568,6 +568,7 @@ export type FilePart = { mime: string filename?: string url: string + synthetic?: boolean artifact?: { codec: "file-part.v1" id: string diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index b752bebb7..1dfc35757 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -568,6 +568,7 @@ export type FilePart = { mime: string filename?: string url: string + synthetic?: boolean artifact?: { codec: "file-part.v1" id: string From 1298431b957ca2d6a66be5f60b8691906555bfd5 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 16:47:47 +0800 Subject: [PATCH 24/44] fix(session): migrate metadata-only diff artifacts --- .../src/session/diff-artifact.ts | 11 +-- .../test/session/diff-artifact.test.ts | 67 ++++++++++++++++++- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/packages/deepagent-code/src/session/diff-artifact.ts b/packages/deepagent-code/src/session/diff-artifact.ts index 85ac39305..46b27cde0 100644 --- a/packages/deepagent-code/src/session/diff-artifact.ts +++ b/packages/deepagent-code/src/session/diff-artifact.ts @@ -352,7 +352,7 @@ function migrateCandidate(db: Database.Interface["db"], candidate: Candidate, no }) .run() for (const index of Array.from({ length: chunkCount }, (_, chunkIndex) => chunkIndex)) { - const patchChunk = yield* tx.get<{ data: Buffer }>(sql` + const patchChunk = yield* tx.get<{ data: Buffer | null }>(sql` SELECT substr( CAST(COALESCE(json_extract(diff.value, '$.patch'), '') AS BLOB), ${index * FILE_CHUNK_BYTES + 1}, ${FILE_CHUNK_BYTES} @@ -364,15 +364,18 @@ function migrateCandidate(db: Database.Interface["db"], candidate: Candidate, no `) if (!patchChunk) return yield* failure(`artifact file ${file.file_index} chunk is missing`, expectedMessageDataHash) - digest.update(patchChunk.data) + // SQLite exposes substr(X'', 1, n) as NULL through the Bun driver. A legacy + // JSON null patch is the canonical empty patch, represented by one empty chunk. + const data = patchChunk.data ?? Buffer.alloc(0) + digest.update(data) yield* tx .insert(SessionDiffArtifactFileChunkTable) .values({ artifact_id: candidate.artifactID, file_index: file.file_index, chunk_index: index, - data: patchChunk.data, - chunk_hash: Hash.sha256(patchChunk.data), + data, + chunk_hash: Hash.sha256(data), }) .run() } diff --git a/packages/deepagent-code/test/session/diff-artifact.test.ts b/packages/deepagent-code/test/session/diff-artifact.test.ts index de1508ae4..b6e08dd9d 100644 --- a/packages/deepagent-code/test/session/diff-artifact.test.ts +++ b/packages/deepagent-code/test/session/diff-artifact.test.ts @@ -18,7 +18,7 @@ import { MessageV2 } from "@/session/message-v2" import { MessageID } from "@/session/schema" import { Session } from "@/session/session" import { SessionDiffArtifact } from "@/session/diff-artifact" -import { SessionDiffMigrationReceiptTable } from "@/session/diff-artifact.sql" +import { SessionDiffArtifactFileChunkTable, SessionDiffMigrationReceiptTable } from "@/session/diff-artifact.sql" import { SessionPromptEpochTable } from "@/session/prompt-epoch.sql" import { SessionPaths } from "@/server/routes/instance/httpapi/groups/session" import { and, asc, eq, sql } from "drizzle-orm" @@ -36,20 +36,26 @@ afterEach(async () => { await resetDatabase() }) -const seed = Effect.fn("Test.seedLegacyDiff")(function* (input?: { corruptMessage?: boolean }) { +const seed = Effect.fn("Test.seedLegacyDiff")(function* (input?: { corruptMessage?: boolean; nullPatch?: boolean }) { const sessions = yield* Session.Service const events = yield* EventV2.Service const { db } = yield* Database.Service const session = yield* sessions.create({ title: "legacy diff" }) const messageID = MessageID.ascending() const patch = "你".repeat(Math.ceil((EventV2.MAX_ENCODED_PAYLOAD_BYTES + 1024) / 3)) + const diffs = [ + { file: "src/large.ts", patch, additions: 2, deletions: 1, status: "modified" as const }, + ...(input?.nullPatch + ? [{ file: "src/metadata-only.ts", patch: null, additions: 0, deletions: 0, status: "modified" as const }] + : []), + ] as unknown as NonNullable["diffs"] const message = { role: "user" as const, time: { created: Date.now() }, agent: "build", model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("model") }, summary: { - diffs: [{ file: "src/large.ts", patch, additions: 2, deletions: 1, status: "modified" as const }], + diffs, }, } yield* db @@ -290,6 +296,61 @@ describe("Session legacy diff physical migration", () => { 30_000, ) + it.instance( + "preserves a legacy null patch as one verified empty chunk", + () => + Effect.gen(function* () { + const input = yield* seed({ nullPatch: true }) + expect(yield* SessionDiffArtifact.migrate({ sessionID: input.session.id, now: 150 })).toEqual({ + processed: 1, + committed: 1, + failed: 0, + }) + const manifestResponse = yield* request( + `${pathFor(SessionPaths.diffArtifactManifest, input.session.id)}?${new URLSearchParams({ + messageID: input.messageID, + artifactID: input.artifact.artifact_id, + })}`, + ) + expect(manifestResponse.status).toBe(200) + const manifest = yield* json(manifestResponse) + expect(manifest.files).toContainEqual( + expect.objectContaining({ file: "src/metadata-only.ts", patchBytes: 0 }), + ) + const { db } = yield* Database.Service + const emptyChunks = yield* db + .select() + .from(SessionDiffArtifactFileChunkTable) + .where( + and( + eq(SessionDiffArtifactFileChunkTable.artifact_id, input.artifact.artifact_id), + eq(SessionDiffArtifactFileChunkTable.file_index, 1), + ), + ) + .all() + .pipe(Effect.orDie) + expect(emptyChunks).toHaveLength(1) + expect(emptyChunks[0]?.data).toEqual(Buffer.alloc(0)) + expect(emptyChunks[0]?.chunk_hash).toBe(Hash.sha256(Buffer.alloc(0))) + const fileResponse = yield* request( + `${pathFor(SessionPaths.diffArtifactFile, input.session.id)}?${new URLSearchParams({ + messageID: input.messageID, + artifactID: input.artifact.artifact_id, + path: "src/metadata-only.ts", + })}`, + ) + expect(fileResponse.status).toBe(200) + expect(yield* json(fileResponse)).toMatchObject({ + patch: "", + patchBytes: 0, + returnedBytes: 0, + truncated: false, + }) + }), + { git: true }, + 30_000, + ) + it.instance( "fails closed when a committed artifact chunk is corrupted", () => From d93ab4cef0f5c7a604300c499446c01a3d9add99 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 16:58:15 +0800 Subject: [PATCH 25/44] fix(session): select authoritative diff artifacts --- .../script/bug-407-010-release-gate.ts | 29 ++++- .../src/session/diff-artifact.ts | 8 ++ .../test/session/diff-artifact.test.ts | 100 +++++++++++++++++- 3 files changed, 133 insertions(+), 4 deletions(-) diff --git a/packages/deepagent-code/script/bug-407-010-release-gate.ts b/packages/deepagent-code/script/bug-407-010-release-gate.ts index f30e51725..aec46471d 100644 --- a/packages/deepagent-code/script/bug-407-010-release-gate.ts +++ b/packages/deepagent-code/script/bug-407-010-release-gate.ts @@ -50,6 +50,9 @@ const stage = Effect.fn("Bug407010ReleaseGate.stage")(function* ( ) { const started = Bun.nanoseconds() const result = yield* effect + const { db } = yield* Database.Service + yield* db.run("PRAGMA shrink_memory").pipe(Effect.orDie) + yield* Effect.sync(() => Bun.gc(true)) console.log(JSON.stringify({ stage: name, elapsedMs: Math.round((Bun.nanoseconds() - started) / 1_000_000), @@ -98,6 +101,10 @@ const program = Effect.gen(function* () { processed += result.processed if (!result.next) return { batches, processed } cursor = result.next + if (batches % 8 === 0) { + yield* db.run("PRAGMA shrink_memory").pipe(Effect.orDie) + yield* Effect.sync(() => Bun.gc(true)) + } } })) @@ -120,7 +127,7 @@ const program = Effect.gen(function* () { .where(eq(EventArtifactTable.kind, "legacy_message_diff")) .all() .pipe(Effect.orDie) - yield* stage("legacy-session-diffs", Effect.gen(function* () { + const diffMigration = yield* stage("legacy-session-diffs", Effect.gen(function* () { let processed = 0 let committed = 0 let failed = 0 @@ -131,10 +138,14 @@ const program = Effect.gen(function* () { committed += result.committed failed += result.failed if (result.processed === 0) break + yield* db.run("PRAGMA shrink_memory").pipe(Effect.orDie) + yield* Effect.sync(() => Bun.gc(true)) } } return { sessions: artifactSessions.length, processed, committed, failed } })) + if (diffMigration.failed > 0) + return yield* Effect.die(new Error(`Legacy Session diff migration failed: ${JSON.stringify(diffMigration)}`)) yield* stage("checkpoint-and-compact", Effect.forEach(sessions, (sessionID) => Effect.gen(function* () { const session = yield* db @@ -184,6 +195,8 @@ const program = Effect.gen(function* () { deleted += compacted.deleted batches += 1 } + yield* db.run("PRAGMA shrink_memory").pipe(Effect.orDie) + yield* Effect.sync(() => Bun.gc(true)) return { sessionID, snapshotID: snapshot.snapshotID, throughSeq: snapshot.throughSeq, deleted, batches } }), { concurrency: 1 })) @@ -216,6 +229,12 @@ const program = Effect.gen(function* () { const activeSnapshots = yield* db.select({ count: sql`count(*)` }).from(EventSnapshotTable).get().pipe(Effect.orDie) const compactions = yield* db.select({ count: sql`count(*)` }).from(EventCompactionReceiptTable).get().pipe(Effect.orDie) const diffReceipts = yield* db.select({ count: sql`count(*)` }).from(SessionDiffMigrationReceiptTable).get().pipe(Effect.orDie) + const failedDiffReceipts = yield* db + .select({ count: sql`count(*)` }) + .from(SessionDiffMigrationReceiptTable) + .where(eq(SessionDiffMigrationReceiptTable.state, "migration_validation_failed")) + .get() + .pipe(Effect.orDie) const fileArtifacts = yield* db.select({ count: sql`count(*)` }).from(FilePartArtifactBindingTable).get().pipe(Effect.orDie) const result = { ...after[0], @@ -223,13 +242,19 @@ const program = Effect.gen(function* () { activeSnapshots: activeSnapshots?.count ?? 0, compactionReceipts: compactions?.count ?? 0, diffMigrationReceipts: diffReceipts?.count ?? 0, + failedDiffMigrationReceipts: failedDiffReceipts?.count ?? 0, fileArtifactBindings: fileArtifacts?.count ?? 0, backfill, quickCheck: quickCheck.map((row) => row.quick_check), foreignKeyViolations: foreignKeys.length, rssBytes: process.memoryUsage.rss(), } - if (backfill?.state !== "complete" || quickCheck.some((row) => row.quick_check !== "ok") || foreignKeys.length > 0) + if ( + backfill?.state !== "complete" || + quickCheck.some((row) => row.quick_check !== "ok") || + foreignKeys.length > 0 || + result.failedDiffMigrationReceipts > 0 + ) return yield* Effect.die(new Error(`Release gate failed: ${JSON.stringify(result)}`)) console.log(JSON.stringify({ stage: "after", ...result })) }) diff --git a/packages/deepagent-code/src/session/diff-artifact.ts b/packages/deepagent-code/src/session/diff-artifact.ts index 46b27cde0..012a606b3 100644 --- a/packages/deepagent-code/src/session/diff-artifact.ts +++ b/packages/deepagent-code/src/session/diff-artifact.ts @@ -105,6 +105,14 @@ export const migrate = Effect.fn("SessionDiffArtifact.migrate")(function* (input sql`json_extract(${MessageTable.data}, '$.role') = 'user'`, sql`json_type(${MessageTable.data}, '$.summary.diffs') = 'array'`, sql`${SessionDiffMigrationReceiptTable.message_id} is null`, + sql`NOT EXISTS ( + SELECT 1 + FROM event_artifact newer + WHERE newer.aggregate_id = ${EventArtifactTable.aggregate_id} + AND newer.kind = 'legacy_message_diff' + AND newer.seq > ${EventArtifactTable.seq} + AND json_extract(newer.canonical_data, '$.info.id') = ${MessageTable.id} + )`, ), ) .orderBy(asc(EventArtifactTable.event_id)) diff --git a/packages/deepagent-code/test/session/diff-artifact.test.ts b/packages/deepagent-code/test/session/diff-artifact.test.ts index b6e08dd9d..c35cc627b 100644 --- a/packages/deepagent-code/test/session/diff-artifact.test.ts +++ b/packages/deepagent-code/test/session/diff-artifact.test.ts @@ -36,7 +36,11 @@ afterEach(async () => { await resetDatabase() }) -const seed = Effect.fn("Test.seedLegacyDiff")(function* (input?: { corruptMessage?: boolean; nullPatch?: boolean }) { +const seed = Effect.fn("Test.seedLegacyDiff")(function* (input?: { + corruptMessage?: boolean + nullPatch?: boolean + supersededArtifact?: boolean +}) { const sessions = yield* Session.Service const events = yield* EventV2.Service const { db } = yield* Database.Service @@ -143,12 +147,77 @@ const seed = Effect.fn("Test.seedLegacyDiff")(function* (input?: { corruptMessag .get() .pipe(Effect.orDie) if (!artifact) return yield* Effect.die("legacy artifact missing") + const latest = input?.supersededArtifact + ? yield* Effect.gen(function* () { + const latestPatch = `${patch}\nlatest` + const latestMessage = { + ...message, + summary: { + diffs: [{ file: "src/large.ts", patch: latestPatch, additions: 3, deletions: 1, status: "modified" as const }], + }, + } + yield* db + .update(MessageTable) + .set({ data: latestMessage }) + .where(eq(MessageTable.id, messageID)) + .run() + .pipe(Effect.orDie) + yield* db + .update(SessionTable) + .set({ summary_diffs: latestMessage.summary.diffs }) + .where(eq(SessionTable.id, session.id)) + .run() + .pipe(Effect.orDie) + const latestSync = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) + if (!latestSync) return yield* Effect.die("latest sync sequence authority missing") + const latestEventID = EventV2.ID.make(`evt_legacy_diff_latest_${messageID}`) + yield* db + .insert(EventTable) + .values({ + id: latestEventID, + aggregate_id: session.id, + seq: (sequence?.seq ?? -1) + 2, + type: EventV2.versionedType("message.updated", 1), + data: { sessionID: session.id, info: { ...latestMessage, id: messageID, sessionID: session.id } }, + sync_seq: latestSync.seq, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ seq: (sequence?.seq ?? -1) + 2 }) + .where(eq(EventSequenceTable.aggregate_id, session.id)) + .run() + .pipe(Effect.orDie) + expect((yield* events.canonicalizeLegacyArtifacts({ limit: 1 })).processed).toBe(1) + const latestArtifact = yield* db + .select() + .from(EventArtifactTable) + .where(eq(EventArtifactTable.event_id, latestEventID)) + .get() + .pipe(Effect.orDie) + if (!latestArtifact) return yield* Effect.die("latest legacy artifact missing") + return { message: latestMessage, artifact: latestArtifact, patch: latestPatch } + }) + : undefined if (input?.corruptMessage) { yield* db .run(sql`UPDATE message SET data = json_set(data, '$.summary.diffs[0].patch', 'changed') WHERE id = ${messageID}`) .pipe(Effect.orDie) } - return { session, messageID, message, artifact, patch } + return { + session, + messageID, + message: latest?.message ?? message, + artifact: latest?.artifact ?? artifact, + patch: latest?.patch ?? patch, + } }) function pathFor(template: string, sessionID: string) { @@ -351,6 +420,33 @@ describe("Session legacy diff physical migration", () => { 30_000, ) + it.instance( + "rewrites from the latest artifact when one Message has multiple historical diff events", + () => + Effect.gen(function* () { + const input = yield* seed({ supersededArtifact: true }) + const { db } = yield* Database.Service + expect(yield* SessionDiffArtifact.migrate({ sessionID: input.session.id, now: 175 })).toEqual({ + processed: 1, + committed: 1, + failed: 0, + }) + const receipt = yield* db + .select() + .from(SessionDiffMigrationReceiptTable) + .where(eq(SessionDiffMigrationReceiptTable.message_id, input.messageID)) + .get() + .pipe(Effect.orDie) + expect(receipt).toMatchObject({ + artifact_id: input.artifact.artifact_id, + source_event_id: input.artifact.event_id, + state: "committed", + }) + }), + { git: true }, + 30_000, + ) + it.instance( "fails closed when a committed artifact chunk is corrupted", () => From b270120177dcef46ea70c5208d778cdf56f6a5d3 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 18:50:52 +0800 Subject: [PATCH 26/44] fix: close bounded artifact release gates --- packages/core/src/database/migration.gen.ts | 1 + packages/core/src/database/migration.ts | 25 +- ...20260812120000_legacy_provider_recovery.ts | 57 +++- .../20260813135000_event_immutable.ts | 122 +++++++ packages/core/src/event.ts | 278 +++++++++------- packages/core/src/file-part-artifact.ts | 312 +++++++++++------- packages/core/src/session/projector.ts | 20 ++ packages/core/src/v1/session.ts | 22 +- packages/core/test/database-migration.test.ts | 71 ++++ packages/core/test/event-snapshot.test.ts | 175 ++++++++++ packages/core/test/event.test.ts | 47 +++ packages/core/test/file-part-artifact.test.ts | 124 ++++++- .../script/bug-407-010-release-gate.ts | 109 +++++- .../src/control-plane/workspace.ts | 4 +- .../routes/instance/httpapi/groups/session.ts | 6 +- .../instance/httpapi/handlers/session.ts | 12 +- .../src/session/diff-artifact.ts | 257 ++++++++++++--- .../test/control-plane/workspace.test.ts | 4 +- .../test/session/diff-artifact.test.ts | 100 +++++- packages/sdk/js/src/gen/types.gen.ts | 41 ++- packages/sdk/js/src/v2/gen/types.gen.ts | 41 ++- 21 files changed, 1471 insertions(+), 357 deletions(-) create mode 100644 packages/core/src/database/migration/20260813135000_event_immutable.ts diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index dd9609e29..4e6cc97d3 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -93,5 +93,6 @@ export const migrations = ( import("./migration/20260813132000_session_diff_artifact"), import("./migration/20260813133000_session_transfer_authority"), import("./migration/20260813134000_database_capability"), + import("./migration/20260813135000_event_immutable"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration.ts b/packages/core/src/database/migration.ts index ba2170f40..816e249f0 100644 --- a/packages/core/src/database/migration.ts +++ b/packages/core/src/database/migration.ts @@ -8,6 +8,7 @@ import { migrations } from "./migration.gen" type Database = EffectDrizzleSqlite.EffectSQLiteDatabase type Transaction = Parameters[0]>[0] const lock = Semaphore.makeUnsafe(1) +const historicalAliases = new Map([["20260530232709_lovely_romulus", "20260511173437_session-metadata"]]) export type Migration = { id: string @@ -15,10 +16,14 @@ export type Migration = { } export function apply(db: Database) { - return lock.withPermit(applyOnly(db, migrations)) + return lock.withPermit(applyMigrations(db, migrations, true)) } export function applyOnly(db: Database, input: Migration[]) { + return applyMigrations(db, input, false) +} + +function applyMigrations(db: Database, input: Migration[], requireLinearHistory: boolean) { return Effect.gen(function* () { const duplicate = input.find( (migration, index) => input.findIndex((candidate) => candidate.id === migration.id) !== index, @@ -48,6 +53,24 @@ export function applyOnly(db: Database, input: Migration[]) { } } + if (requireLinearHistory) { + const known = new Set(input.map((migration) => migration.id)) + for (const [alias, canonical] of historicalAliases) { + if (completed.has(alias) && known.has(canonical)) completed.add(canonical) + } + const unknown = [...completed].filter((id) => !known.has(id) && !historicalAliases.has(id)).sort() + if (unknown.length > 0) + return yield* Effect.die( + new Error(`database migration history belongs to an incompatible lineage: ${unknown.join(", ")}`), + ) + const firstMissing = input.findIndex((migration) => !completed.has(migration.id)) + const gap = firstMissing < 0 ? undefined : input.slice(firstMissing + 1).find((migration) => completed.has(migration.id)) + if (gap) + return yield* Effect.die( + new Error(`database migration history has a gap before ${gap.id}: ${input[firstMissing]!.id} is missing`), + ) + } + for (const migration of input) { if (completed.has(migration.id)) continue yield* db.transaction((tx) => diff --git a/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts b/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts index b00c708aa..c952ca034 100644 --- a/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts +++ b/packages/core/src/database/migration/20260812120000_legacy_provider_recovery.ts @@ -12,12 +12,16 @@ export default { // Historical orphan receipts are intentionally omitted from the rebuilt authority. yield* tx.run("DROP TRIGGER IF EXISTS compaction_run_continuation_response_validate") yield* tx.run(` - CREATE TEMP TABLE legacy_provider_recovery_argument_receipt AS - SELECT * FROM session_tool_argument_receipt + CREATE TEMP TABLE legacy_provider_recovery_argument_receipt AS SELECT + receipt_id, layer, ordinal, call_id, tool_name, event_type, payload_hash, + payload_length, payload_keys, unavailable_reason, created_at, validation_outcome + FROM session_tool_argument_receipt `) yield* tx.run(` - CREATE TEMP TABLE legacy_provider_recovery_activity_progress AS - SELECT * FROM session_activity_progress + CREATE TEMP TABLE legacy_provider_recovery_activity_progress AS SELECT + activity_id, revision, assistant_message_id, text_part_id, provider_receipt_id, + state, finish_observed, response_fingerprint, created_at, settled_at + FROM session_activity_progress `) yield* tx.run("DROP TABLE session_tool_argument_receipt") yield* tx.run("DROP TABLE session_activity_progress") @@ -68,8 +72,37 @@ export default { ) `) yield* tx.run(` - INSERT INTO session_tool_request_receipt_rebuilt - SELECT receipt.* + INSERT INTO session_tool_request_receipt_rebuilt ( + receipt_id, request_ordinal, session_id, user_message_id, assistant_message_id, + provider_attempt_id, provider_id, model_id, protocol, registry_tool_ids, + permission_filtered_tool_ids, final_offered_tool_ids, call_ids, + tool_definition_hash, tool_choice_mode, adapter_tool_capability, + adapter_lowering_outcome, estimated_input_tokens, physical_input_budget, + reserved_output_tokens, safety_margin_tokens, context_limit_provenance, + request_state, request_error_code, created_at, prompt_epoch, prompt_window_id, + effective_history_hash, world_state_baseline_hash, prompt_cache_key, + provider_request_hash, response_chain_reuse_decision, + response_chain_refusal_reason, request_input_hash, final_request_hash, + provider_state, adapter_prepared_at, dispatching_at, streaming_at, terminal_at, + response_fingerprint, owner_token + ) + SELECT + receipt.receipt_id, receipt.request_ordinal, receipt.session_id, + receipt.user_message_id, receipt.assistant_message_id, receipt.provider_attempt_id, + receipt.provider_id, receipt.model_id, receipt.protocol, receipt.registry_tool_ids, + receipt.permission_filtered_tool_ids, receipt.final_offered_tool_ids, receipt.call_ids, + receipt.tool_definition_hash, receipt.tool_choice_mode, receipt.adapter_tool_capability, + receipt.adapter_lowering_outcome, receipt.estimated_input_tokens, + receipt.physical_input_budget, receipt.reserved_output_tokens, + receipt.safety_margin_tokens, receipt.context_limit_provenance, receipt.request_state, + receipt.request_error_code, receipt.created_at, receipt.prompt_epoch, + receipt.prompt_window_id, receipt.effective_history_hash, + receipt.world_state_baseline_hash, receipt.prompt_cache_key, + receipt.provider_request_hash, receipt.response_chain_reuse_decision, + receipt.response_chain_refusal_reason, receipt.request_input_hash, + receipt.final_request_hash, receipt.provider_state, receipt.adapter_prepared_at, + receipt.dispatching_at, receipt.streaming_at, receipt.terminal_at, + receipt.response_fingerprint, receipt.owner_token FROM session_tool_request_receipt receipt JOIN session ON session.id = receipt.session_id `) @@ -198,7 +231,11 @@ export default { `) yield* tx.run(` INSERT INTO session_tool_argument_receipt - SELECT argument.* + SELECT + argument.receipt_id, argument.layer, argument.ordinal, argument.call_id, + argument.tool_name, argument.event_type, argument.payload_hash, + argument.payload_length, argument.payload_keys, argument.unavailable_reason, + argument.created_at, argument.validation_outcome FROM legacy_provider_recovery_argument_receipt argument JOIN session_tool_request_receipt receipt ON receipt.receipt_id = argument.receipt_id `) @@ -232,7 +269,11 @@ export default { `) yield* tx.run(` INSERT INTO session_activity_progress - SELECT progress.* + SELECT + progress.activity_id, progress.revision, progress.assistant_message_id, + progress.text_part_id, progress.provider_receipt_id, progress.state, + progress.finish_observed, progress.response_fingerprint, + progress.created_at, progress.settled_at FROM legacy_provider_recovery_activity_progress progress JOIN session_tool_request_receipt receipt ON receipt.receipt_id = progress.provider_receipt_id `) diff --git a/packages/core/src/database/migration/20260813135000_event_immutable.ts b/packages/core/src/database/migration/20260813135000_event_immutable.ts new file mode 100644 index 000000000..a538ed71d --- /dev/null +++ b/packages/core/src/database/migration/20260813135000_event_immutable.ts @@ -0,0 +1,122 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813135000_event_immutable", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TRIGGER event_update_immutable + BEFORE UPDATE ON event + BEGIN + SELECT RAISE(ABORT, 'event_update_immutable'); + END + `) + for (const table of [ + "event_artifact", + "event_artifact_chunk", + "file_part_artifact_chunk", + "file_part_artifact_binding", + "session_diff_artifact_file", + "session_diff_artifact_file_chunk", + ]) { + yield* tx.run(` + CREATE TRIGGER ${table}_update_immutable + BEFORE UPDATE ON ${table} + BEGIN + SELECT RAISE(ABORT, '${table}_update_immutable'); + END + `) + } + yield* tx.run(` + CREATE TRIGGER file_part_artifact_update_immutable + BEFORE UPDATE ON file_part_artifact + WHEN NOT ( + OLD.complete = 0 AND NEW.complete = 1 + AND NEW.artifact_id = OLD.artifact_id + AND NEW.body_hash = OLD.body_hash + AND NEW.body_bytes = OLD.body_bytes + AND NEW.chunk_bytes = OLD.chunk_bytes + AND NEW.chunk_count = OLD.chunk_count + AND NEW.codec_version = OLD.codec_version + AND NEW.created_at = OLD.created_at + ) + BEGIN + SELECT RAISE(ABORT, 'file_part_artifact_update_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_artifact_delete_immutable + BEFORE DELETE ON event_artifact + WHEN EXISTS (SELECT 1 FROM event_sequence WHERE aggregate_id = OLD.aggregate_id) + BEGIN + SELECT RAISE(ABORT, 'event_artifact_delete_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_artifact_chunk_delete_immutable + BEFORE DELETE ON event_artifact_chunk + WHEN EXISTS (SELECT 1 FROM event_artifact WHERE artifact_id = OLD.artifact_id) + BEGIN + SELECT RAISE(ABORT, 'event_artifact_chunk_delete_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER file_part_artifact_binding_delete_immutable + BEFORE DELETE ON file_part_artifact_binding + WHEN EXISTS (SELECT 1 FROM event_sequence WHERE aggregate_id = OLD.aggregate_id) + BEGIN + SELECT RAISE(ABORT, 'file_part_artifact_binding_delete_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER file_part_artifact_delete_guard + BEFORE DELETE ON file_part_artifact + WHEN EXISTS (SELECT 1 FROM file_part_artifact_binding WHERE artifact_id = OLD.artifact_id) + OR EXISTS (SELECT 1 FROM file_part_artifact_import WHERE artifact_id = OLD.artifact_id) + BEGIN + SELECT RAISE(ABORT, 'file_part_artifact_is_referenced'); + END + `) + yield* tx.run(` + CREATE TRIGGER file_part_artifact_chunk_delete_immutable + BEFORE DELETE ON file_part_artifact_chunk + WHEN EXISTS (SELECT 1 FROM file_part_artifact WHERE artifact_id = OLD.artifact_id) + BEGIN + SELECT RAISE(ABORT, 'file_part_artifact_chunk_delete_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_diff_artifact_file_delete_immutable + BEFORE DELETE ON session_diff_artifact_file + WHEN EXISTS ( + SELECT 1 FROM session_diff_migration_receipt receipt + WHERE receipt.artifact_id = OLD.artifact_id AND receipt.state = 'committed' + ) + BEGIN + SELECT RAISE(ABORT, 'session_diff_artifact_file_delete_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_diff_artifact_file_chunk_delete_immutable + BEFORE DELETE ON session_diff_artifact_file_chunk + WHEN EXISTS ( + SELECT 1 FROM session_diff_migration_receipt receipt + WHERE receipt.artifact_id = OLD.artifact_id AND receipt.state = 'committed' + ) + BEGIN + SELECT RAISE(ABORT, 'session_diff_artifact_file_chunk_delete_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_diff_migration_receipt_delete_immutable + BEFORE DELETE ON session_diff_migration_receipt + WHEN OLD.state = 'committed' + AND EXISTS (SELECT 1 FROM session WHERE id = OLD.session_id) + BEGIN + SELECT RAISE(ABORT, 'session_diff_migration_receipt_delete_immutable'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index 85917311a..10a5d114b 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -191,6 +191,9 @@ export const SNAPSHOT_TRANSFER_CHUNKS = 16 export const LEGACY_ARTIFACT_BATCH_EVENTS = 1 export const LEGACY_DIFF_DESCRIPTOR_FILES = 200 export const ARTIFACT_CHUNK_BYTES = 256 * 1024 +export const LEGACY_ARTIFACT_MAX_SOURCE_BYTES = 64 * 1024 * 1024 +export const LEGACY_ARTIFACT_MAX_BODY_BYTES = 64 * 1024 * 1024 +export const LEGACY_ARTIFACT_MAX_FILES = 10_000 export class EncodedPayloadTooLargeError extends Schema.TaggedErrorClass()( "EventV2.EncodedPayloadTooLarge", @@ -265,6 +268,25 @@ function hashJson(value: unknown) { return Hash.sha256(stableJson(value)) } +function isLegacyFileArtifactRetry( + type: string, + stored: Record, + original: Record, + encoded: Record, + binding: { readonly originalDataHash: string; readonly canonicalData: Record }, +) { + if (!isDeepStrictEqual(stored, original) || !FilePartArtifact.matchesDataHash(binding.originalDataHash, original)) + return false + const prepared = FilePartArtifact.prepare( + type, + original, + Buffer.byteLength(JSON.stringify(original)), + MAX_ENCODED_PAYLOAD_BYTES, + ) + if (prepared.artifacts.length !== 1 || !isDeepStrictEqual(prepared.data, encoded)) return false + return FilePartArtifact.isLegacySyntheticCanonical(binding.canonicalData, encoded) +} + // Synchronized events cross a JSON boundary, so their data schemas must encode and decode without services. const syncCodec = (definition: Definition) => definition.data as Schema.Codec @@ -592,9 +614,15 @@ export const layerWith = (layerOptions?: LayerOptions) => stored.type === versionedType(definition.type, sync.version) && (isDeepStrictEqual(stored.data, encoded) || isDeepStrictEqual(artifact?.canonicalData, encoded) || - (isDeepStrictEqual(fileArtifact?.canonicalData, encoded) && - (fileArtifact?.originalDataHash === Hash.sha256(JSON.stringify(original)) || - isDeepStrictEqual(stored.data, original))) || + (fileArtifact && + (isDeepStrictEqual(fileArtifact.canonicalData, encoded) || + isLegacyFileArtifactRetry( + definition.type, + stored.data, + original, + encoded, + fileArtifact, + ))) || (artifact && artifact.originalDataHash === (artifact.codecVersion >= 2 @@ -1707,131 +1735,155 @@ export const layerWith = (layerOptions?: LayerOptions) => } function canonicalizeLegacyArtifacts(input?: { readonly afterID?: ID; readonly limit?: number; readonly now?: number }) { - const bodyData = sql`json_object('diffs', json_extract(${EventTable.data}, '$.info.summary.diffs'))` - const chunk = (eventID: ID, kind: "source" | "body", index: number) => - db + return Effect.gen(function* () { + const row = yield* db .select({ - data: kind === "source" - ? sql`substr(CAST(${EventTable.data} AS BLOB), ${index * ARTIFACT_CHUNK_BYTES + 1}, ${ARTIFACT_CHUNK_BYTES})` - : sql`substr(CAST(${bodyData} AS BLOB), ${index * ARTIFACT_CHUNK_BYTES + 1}, ${ARTIFACT_CHUNK_BYTES})`, + id: EventTable.id, + aggregateID: EventTable.aggregate_id, + seq: EventTable.seq, + sourceBytes: sql`length(CAST(${EventTable.data} AS BLOB))`, }) .from(EventTable) - .where(eq(EventTable.id, eventID)) + .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) + .where( + and( + eq(EventTable.type, versionedType("message.updated", 1)), + sql`length(CAST(${EventTable.data} AS BLOB)) > ${MAX_ENCODED_PAYLOAD_BYTES}`, + input?.afterID ? gt(EventTable.id, input.afterID) : undefined, + sql`${EventArtifactTable.event_id} is null`, + ), + ) + .orderBy(asc(EventTable.id)) + .limit(1) .get() .pipe(Effect.orDie) - const hash = (eventID: ID, kind: "source" | "body", bytes: number) => - Effect.gen(function* () { - const digest = createHash("sha256") - yield* Effect.forEach( - Array.from({ length: Math.ceil(bytes / ARTIFACT_CHUNK_BYTES) }, (_, index) => index), - (index) => chunk(eventID, kind, index).pipe(Effect.tap((value) => Effect.sync(() => digest.update(value!.data)))), - { discard: true }, + if (!row) return { processed: 0, next: undefined } + if (row.sourceBytes > LEGACY_ARTIFACT_MAX_SOURCE_BYTES) + return yield* Effect.die( + new EncodedPayloadTooLargeError({ + type: `event.artifact.${row.id}`, + encodedBytes: row.sourceBytes, + limitBytes: LEGACY_ARTIFACT_MAX_SOURCE_BYTES, + message: `Legacy diff artifact ${row.id} exceeds the source maintenance budget`, + }), ) - return digest.digest("hex") - }) - return db - .transaction( - () => - Effect.gen(function* () { - const rows = yield* db - .select({ - id: EventTable.id, - aggregateID: EventTable.aggregate_id, - seq: EventTable.seq, - fileCount: sql`json_array_length(${EventTable.data}, '$.info.summary.diffs')`, - sourceBytes: sql`length(CAST(${EventTable.data} AS BLOB))`, - bodyBytes: sql`length(CAST(${bodyData} AS BLOB))`, - }) - .from(EventTable) - .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) - .where( - and( - eq(EventTable.type, versionedType("message.updated", 1)), - sql`length(CAST(${EventTable.data} AS BLOB)) > ${MAX_ENCODED_PAYLOAD_BYTES}`, - sql`json_valid(${EventTable.data})`, - sql`json_type(${EventTable.data}, '$.info.summary.diffs') = 'array'`, - sql`json_array_length(${EventTable.data}, '$.info.summary.diffs') > 0`, - input?.afterID ? gt(EventTable.id, input.afterID) : undefined, - sql`${EventArtifactTable.event_id} is null`, - ), - ) - .orderBy(asc(EventTable.id)) - .limit(Math.min(Math.max(input?.limit ?? LEGACY_ARTIFACT_BATCH_EVENTS, 1), LEGACY_ARTIFACT_BATCH_EVENTS)) - .all() - .pipe(Effect.orDie) - yield* Effect.forEach( - rows, - (row) => Effect.gen(function* () { - const originalDataHash = yield* hash(row.id, "source", row.sourceBytes) - const bodyHash = yield* hash(row.id, "body", row.bodyBytes) - const artifactID = `evtart_${Hash.sha256(`legacy-message-diff.v2:${row.id}:${originalDataHash}:${bodyHash}`)}` - const canonicalData = db - .select({ - data: sql`json_set( - ${EventTable.data}, - '$.info.summary.diffs', - json(COALESCE(( - SELECT json_group_array(json_remove(value, '$.patch')) - FROM ( - SELECT value - FROM json_each(json_extract(${EventTable.data}, '$.info.summary.diffs')) - LIMIT ${LEGACY_DIFF_DESCRIPTOR_FILES} - ) - ), '[]')), - '$.info.summary.diffArtifact', - json_object( - 'id', ${artifactID}, - 'hash', ${bodyHash}, - 'codec', 'legacy-message-diff.v2', - 'fileCount', ${row.fileCount} - ) - )`, - }) - .from(EventTable) - .where(eq(EventTable.id, row.id)) - .get() - .pipe(Effect.orDie) - return yield* canonicalData.pipe( - Effect.flatMap((canonical) => { - if (!canonical) return Effect.die(new Error(`Missing artifact source event ${row.id}`)) - const chunks = Math.ceil(row.bodyBytes / ARTIFACT_CHUNK_BYTES) || 1 - return db.run(sql` + const source = yield* db + .select({ text: sql`CAST(${EventTable.data} AS TEXT)` }) + .from(EventTable) + .where(eq(EventTable.id, row.id)) + .get() + .pipe(Effect.orDie) + if (!source || Buffer.byteLength(source.text) !== row.sourceBytes) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "event.artifact", + message: `Artifact source ${row.id} changed before validation`, + }), + ) + const decoded = Schema.decodeUnknownOption(Schema.UnknownFromJsonString)(source.text) + const data = decoded._tag === "Some" ? decoded.value : undefined + if (!data || typeof data !== "object" || Array.isArray(data)) return { processed: 0, next: row.id } + const info = (data as Record).info + const summary = info && typeof info === "object" && !Array.isArray(info) + ? (info as Record).summary + : undefined + const diffs = summary && typeof summary === "object" && !Array.isArray(summary) + ? (summary as Record).diffs + : undefined + if (!Array.isArray(diffs) || diffs.length === 0) return { processed: 0, next: row.id } + if (diffs.length > LEGACY_ARTIFACT_MAX_FILES) + return yield* Effect.die( + new EncodedPayloadTooLargeError({ + type: `event.artifact.${row.id}`, + encodedBytes: diffs.length, + limitBytes: LEGACY_ARTIFACT_MAX_FILES, + message: `Legacy diff artifact ${row.id} exceeds the file-count maintenance budget`, + }), + ) + const body = Buffer.from(JSON.stringify({ diffs })) + if (body.byteLength > LEGACY_ARTIFACT_MAX_BODY_BYTES) + return yield* Effect.die( + new EncodedPayloadTooLargeError({ + type: `event.artifact.${row.id}`, + encodedBytes: body.byteLength, + limitBytes: LEGACY_ARTIFACT_MAX_BODY_BYTES, + message: `Legacy diff artifact ${row.id} exceeds the body maintenance budget`, + }), + ) + const originalDataHash = Hash.sha256(source.text) + const bodyHash = Hash.sha256(body) + const artifactID = `evtart_${Hash.sha256(`legacy-message-diff.v2:${row.id}:${originalDataHash}:${bodyHash}`)}` + const canonical = { + ...(data as Record), + info: { + ...(info as Record), + summary: { + ...(summary as Record), + diffs: diffs.slice(0, LEGACY_DIFF_DESCRIPTOR_FILES).map((diff) => { + if (!diff || typeof diff !== "object" || Array.isArray(diff)) return diff + const { patch: _, ...descriptor } = diff as Record + return descriptor + }), + diffArtifact: { + id: artifactID, + hash: bodyHash, + codec: "legacy-message-diff.v2", + fileCount: diffs.length, + previewFileCount: Math.min(diffs.length, LEGACY_DIFF_DESCRIPTOR_FILES), + previewTruncated: diffs.length > LEGACY_DIFF_DESCRIPTOR_FILES, + }, + }, + }, + } + const canonicalText = JSON.stringify(canonical) + const chunks = Array.from( + { length: Math.ceil(body.byteLength / ARTIFACT_CHUNK_BYTES) || 1 }, + (_, index) => Buffer.from(body.subarray(index * ARTIFACT_CHUNK_BYTES, (index + 1) * ARTIFACT_CHUNK_BYTES)), + ) + return yield* db + .transaction( + () => + Effect.gen(function* () { + const current = yield* db.get<{ text: string }>(sql` + SELECT CAST(data AS TEXT) AS text FROM event + WHERE id = ${row.id} + AND aggregate_id = ${row.aggregateID} + AND seq = ${row.seq} + AND type = ${versionedType("message.updated", 1)} + `) + if (!current || current.text !== source.text) + return yield* Effect.die( + new InvalidSyncEventError({ + type: "event.artifact", + message: `Artifact source ${row.id} changed after validation`, + }), + ) + yield* db.run(sql` INSERT INTO event_artifact ( artifact_id, event_id, aggregate_id, seq, kind, original_data_hash, canonical_data_hash, canonical_data, body_hash, body_bytes, chunk_count, codec_version, created_at ) VALUES ( ${artifactID}, ${row.id}, ${row.aggregateID}, ${row.seq}, 'legacy_message_diff', - ${originalDataHash}, ${Hash.sha256(canonical.data)}, ${canonical.data}, - ${bodyHash}, ${row.bodyBytes}, ${chunks}, 2, ${input?.now ?? Date.now()} + ${originalDataHash}, ${Hash.sha256(canonicalText)}, ${canonicalText}, + ${bodyHash}, ${body.byteLength}, ${chunks.length}, 2, ${input?.now ?? Date.now()} ) ON CONFLICT DO NOTHING - `).pipe( - Effect.orDie, - Effect.andThen( - Effect.forEach( - Array.from({ length: chunks }, (_, index) => index), - (index) => chunk(row.id, "body", index).pipe( - Effect.flatMap((value) => db.run(sql` + `).pipe(Effect.orDie) + yield* Effect.forEach( + chunks, + (chunk, index) => db.run(sql` INSERT INTO event_artifact_chunk (artifact_id, chunk_index, data, chunk_hash) - VALUES (${artifactID}, ${index}, ${value!.data}, ${Hash.sha256(value!.data)}) + VALUES (${artifactID}, ${index}, ${chunk}, ${Hash.sha256(chunk)}) ON CONFLICT DO NOTHING - `)), - Effect.orDie, - ), - { discard: true }, - ), - ), - ) - }), - ) - }), - { discard: true }, - ) - return { processed: rows.length, next: rows.at(-1)?.id } - }), - { behavior: "immediate" }, - ) - .pipe(Effect.orDie) + `).pipe(Effect.orDie), + { discard: true }, + ) + return { processed: 1, next: row.id } + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie) + }) } function remove(aggregateID: string) { diff --git a/packages/core/src/file-part-artifact.ts b/packages/core/src/file-part-artifact.ts index e28c5d58b..49f9ff8d9 100644 --- a/packages/core/src/file-part-artifact.ts +++ b/packages/core/src/file-part-artifact.ts @@ -14,12 +14,16 @@ import { } from "./file-part-artifact.sql" import { NonNegativeInt } from "./schema" import { Hash } from "./util/hash" +import { CanonicalJson } from "./util/canonical-json" import { EventTable } from "./event/sql" import { PartTable } from "./session/sql" export const CHUNK_BYTES = 262_144 as const export const MAX_BYTES = 32 * 1024 * 1024 export const MAX_CHUNKS = MAX_BYTES / CHUNK_BYTES +export const MAX_ENCODED_EVENT_BYTES = 64 * 1024 * 1024 +export const MAX_BASE64_CHARS = Math.ceil(MAX_BYTES / 3) * 4 +export const MAX_DATA_URL_CHARS = MAX_BASE64_CHARS + 256 export const CODEC = "file-part.v1" as const const Digest = Schema.String.check(Schema.isPattern(/^[a-f0-9]{64}$/)) @@ -86,14 +90,50 @@ const descriptorFromData = (type: string, data: Record): Bindin const fail = (artifactID: string, message: string) => Effect.die(new IntegrityError({ artifactID, message })) +export function dataHash(data: unknown) { + return Hash.sha256(CanonicalJson.stringify(data)) +} + +export function matchesDataHash(hash: string, data: unknown) { + return hash === dataHash(data) +} + +export function isLegacySyntheticCanonical(canonical: Record, encoded: Record) { + const part = encoded.part + const canonicalPart = canonical.part + if ( + !part || + typeof part !== "object" || + Array.isArray(part) || + "synthetic" in part || + !canonicalPart || + typeof canonicalPart !== "object" || + Array.isArray(canonicalPart) || + (canonicalPart as Record).synthetic !== true + ) + return false + const { synthetic: _, ...rest } = canonicalPart as Record + return isDeepStrictEqual(encoded, { ...canonical, part: rest }) +} + export function prepare(type: string, data: Record, encodedBytes: number, limitBytes: number) { if (type !== "message.part.updated") return { data, artifacts: [] as Prepared[] } const part = data.part if (!part || typeof part !== "object" || Array.isArray(part)) return { data, artifacts: [] as Prepared[] } const value = part as Record if (value.type !== "file" || typeof value.url !== "string") return { data, artifacts: [] as Prepared[] } + if (value.url.startsWith("data:") && value.url.length > MAX_DATA_URL_CHARS) + throw new IntegrityError({ + artifactID: "pending", + message: `File part data URL is ${value.url.length} characters; limit is ${MAX_DATA_URL_CHARS}`, + }) const match = /^data:([^;,]*);base64,([A-Za-z0-9+/]*={0,2})$/.exec(value.url) if (!match) return { data, artifacts: [] as Prepared[] } + if (match[2]!.length > MAX_BASE64_CHARS) + throw new IntegrityError({ + artifactID: "pending", + message: `File part base64 payload is ${match[2]!.length} characters; limit is ${MAX_BASE64_CHARS}`, + }) const body = Buffer.from(match[2]!, "base64") if (body.toString("base64").replace(/=+$/, "") !== match[2]!.replace(/=+$/, "")) return { data, artifacts: [] as Prepared[] } @@ -221,7 +261,7 @@ export function bind( if (binding.aggregateID !== input.aggregateID) return yield* fail(binding.descriptor.id, "Artifact aggregate does not match its event") yield* requireAvailable(db, binding.descriptor) - const canonicalDataHash = Hash.sha256(JSON.stringify(input.data)) + const normalizedData = JSON.parse(CanonicalJson.stringify(input.data)) as Record const imported = input.requireImport ? yield* db .select() @@ -236,11 +276,13 @@ export function bind( imported.aggregate_id !== input.aggregateID || imported.seq !== input.seq || imported.artifact_id !== binding.descriptor.id || - imported.canonical_data_hash !== canonicalDataHash || + !matchesDataHash(imported.canonical_data_hash, imported.canonical_data) || !isDeepStrictEqual(imported.canonical_data, input.data)) ) return yield* fail(binding.descriptor.id, "Replayed artifact has no exact staged import receipt") - const originalDataHash = imported?.original_data_hash ?? Hash.sha256(JSON.stringify(input.originalData ?? input.data)) + const canonicalData = imported?.canonical_data ?? normalizedData + const canonicalDataHash = dataHash(canonicalData) + const originalDataHash = imported?.original_data_hash ?? dataHash(input.originalData ?? input.data) const existing = yield* db .select() .from(FilePartArtifactBindingTable) @@ -255,7 +297,7 @@ export function bind( existing.artifact_id !== binding.descriptor.id || existing.original_data_hash !== originalDataHash || existing.canonical_data_hash !== canonicalDataHash || - !isDeepStrictEqual(existing.canonical_data, input.data) + !isDeepStrictEqual(existing.canonical_data, canonicalData) ) return yield* fail(binding.descriptor.id, "Event identity is bound to a different artifact") return @@ -270,7 +312,7 @@ export function bind( artifact_id: binding.descriptor.id, original_data_hash: originalDataHash, canonical_data_hash: canonicalDataHash, - canonical_data: input.data, + canonical_data: canonicalData, created_at: input.now ?? Date.now(), }) .run() @@ -623,128 +665,141 @@ export function canonicalizeLegacy( db: Database.Interface["db"], input?: { readonly afterID?: EventV2.ID; readonly limit?: number; readonly now?: number }, ) { - const limit = Math.min(Math.max(input?.limit ?? 1, 1), 1) - return db - .transaction( - () => - Effect.gen(function* () { - const rows = yield* db - .select({ - id: EventTable.id, - aggregateID: EventTable.aggregate_id, - seq: EventTable.seq, - bytes: sql`length(CAST(${EventTable.data} AS BLOB))`, - }) - .from(EventTable) - .leftJoin(FilePartArtifactBindingTable, eq(FilePartArtifactBindingTable.event_id, EventTable.id)) - .where( - and( - eq(EventTable.type, "message.part.updated.1"), - sql`length(CAST(${EventTable.data} AS BLOB)) > ${4 * 1024 * 1024}`, - sql`json_valid(${EventTable.data})`, - sql`json_type(${EventTable.data}, '$.part.url') = 'text'`, - sql`json_extract(${EventTable.data}, '$.part.url') LIKE 'data:%;base64,%'`, - isNull(FilePartArtifactBindingTable.event_id), - input?.afterID ? gt(EventTable.id, input.afterID) : undefined, - ), + return Effect.gen(function* () { + const row = yield* db + .select({ + id: EventTable.id, + aggregateID: EventTable.aggregate_id, + seq: EventTable.seq, + bytes: sql`length(CAST(${EventTable.data} AS BLOB))`, + }) + .from(EventTable) + .leftJoin(FilePartArtifactBindingTable, eq(FilePartArtifactBindingTable.event_id, EventTable.id)) + .where( + and( + eq(EventTable.type, "message.part.updated.1"), + sql`length(CAST(${EventTable.data} AS BLOB)) > ${4 * 1024 * 1024}`, + isNull(FilePartArtifactBindingTable.event_id), + input?.afterID ? gt(EventTable.id, input.afterID) : undefined, + ), + ) + .orderBy(asc(EventTable.id)) + .limit(1) + .get() + .pipe(Effect.orDie) + if (!row) return { processed: 0, next: undefined } + if (row.bytes > MAX_ENCODED_EVENT_BYTES) + return yield* fail( + "pending", + `Legacy file-part event ${row.id} is ${row.bytes} bytes; limit is ${MAX_ENCODED_EVENT_BYTES}`, + ) + const source = yield* db + .select({ data: EventTable.data }) + .from(EventTable) + .where(eq(EventTable.id, row.id)) + .get() + .pipe(Effect.orDie) + if (!source) return yield* fail("pending", `Legacy file-part event ${row.id} disappeared during maintenance`) + if (Buffer.byteLength(JSON.stringify(source.data)) !== row.bytes) + return yield* fail("pending", `Legacy file-part event ${row.id} changed during maintenance`) + const originalPart = source.data.part + if (!originalPart || typeof originalPart !== "object" || Array.isArray(originalPart)) + return { processed: 0, next: row.id } + const sourcePart = originalPart as Record + if (sourcePart.type !== "file" || typeof sourcePart.url !== "string" || !sourcePart.url.startsWith("data:")) + return { processed: 0, next: row.id } + if (sourcePart.url.length > MAX_DATA_URL_CHARS) + return yield* fail( + "pending", + `Legacy file-part event ${row.id} data URL is ${sourcePart.url.length} characters; limit is ${MAX_DATA_URL_CHARS}`, + ) + const prepared = prepare("message.part.updated", source.data, row.bytes, 4 * 1024 * 1024) + if (prepared.artifacts.length !== 1) + return yield* fail("pending", `Legacy file-part event ${row.id} cannot be externalized canonically`) + const originalBinding = descriptorFromData("message.part.updated", prepared.data) + const canonicalPart = prepared.data.part + if (!originalBinding || !canonicalPart || typeof canonicalPart !== "object" || Array.isArray(canonicalPart)) + return yield* fail(prepared.artifacts[0]!.descriptor.id, `Legacy file-part event ${row.id} has invalid part data`) + return yield* db + .transaction( + () => + Effect.gen(function* () { + const current = yield* db + .select({ + aggregateID: EventTable.aggregate_id, + seq: EventTable.seq, + data: EventTable.data, + }) + .from(EventTable) + .where(eq(EventTable.id, row.id)) + .get() + .pipe(Effect.orDie) + if ( + !current || + current.aggregateID !== row.aggregateID || + current.seq !== row.seq || + !isDeepStrictEqual(current.data, source.data) ) - .orderBy(asc(EventTable.id)) - .limit(limit) - .all() - .pipe(Effect.orDie) - const oversized = rows.find((row) => row.bytes > MAX_BYTES) - if (oversized) - return yield* fail( - "pending", - `Legacy file-part event ${oversized.id} is ${oversized.bytes} bytes; limit is ${MAX_BYTES} bytes`, + return yield* fail("pending", `Legacy file-part event ${row.id} changed during maintenance`) + const projected = yield* db + .select() + .from(PartTable) + .where(eq(PartTable.id, originalBinding.partID as typeof PartTable.$inferSelect.id)) + .get() + .pipe(Effect.orDie) + if (!projected) + return yield* fail( + prepared.artifacts[0]!.descriptor.id, + `Legacy file-part event ${row.id} has no corresponding projected part`, + ) + if (projected.session_id !== row.aggregateID || projected.message_id !== sourcePart.messageID) + return yield* fail( + prepared.artifacts[0]!.descriptor.id, + `Legacy file-part event ${row.id} has a conflicting projected part`, + ) + const sourceData = Object.fromEntries( + Object.entries(sourcePart).filter(([key]) => key !== "id" && key !== "sessionID" && key !== "messageID"), ) - yield* Effect.forEach( - rows, - (row) => - Effect.gen(function* () { - const source = yield* db - .select({ data: EventTable.data }) - .from(EventTable) - .where(eq(EventTable.id, row.id)) - .get() - .pipe(Effect.orDie) - if (!source) - return yield* fail("pending", `Legacy file-part event ${row.id} disappeared during maintenance`) - const prepared = prepare("message.part.updated", source.data, row.bytes, 4 * 1024 * 1024) - if (prepared.artifacts.length !== 1) - return yield* fail("pending", `Legacy file-part event ${row.id} cannot be externalized canonically`) - yield* put(db, prepared.artifacts[0]!, input?.now) - yield* bind(db, { - eventID: row.id, - aggregateID: row.aggregateID, - seq: row.seq, - type: "message.part.updated", - data: prepared.data, - originalData: source.data, - now: input?.now, - }) - const originalBinding = descriptorFromData("message.part.updated", prepared.data) - const originalPart = source.data.part - const canonicalPart = prepared.data.part - if ( - !originalBinding || - !originalPart || - typeof originalPart !== "object" || - Array.isArray(originalPart) || - !canonicalPart || - typeof canonicalPart !== "object" || - Array.isArray(canonicalPart) - ) - return yield* fail(prepared.artifacts[0]!.descriptor.id, `Legacy file-part event ${row.id} has invalid part data`) - const projected = yield* db - .select() - .from(PartTable) - .where(eq(PartTable.id, originalBinding.partID as typeof PartTable.$inferSelect.id)) - .get() - .pipe(Effect.orDie) - if (!projected) - return yield* fail( - prepared.artifacts[0]!.descriptor.id, - `Legacy file-part event ${row.id} has no corresponding projected part`, - ) - const sourcePart = originalPart as Record - if (projected.session_id !== row.aggregateID || projected.message_id !== sourcePart.messageID) - return yield* fail(prepared.artifacts[0]!.descriptor.id, `Legacy file-part event ${row.id} has a conflicting projected part`) - const sourceData = Object.fromEntries( - Object.entries(sourcePart).filter(([key]) => key !== "id" && key !== "sessionID" && key !== "messageID"), - ) - // Older FilePart event schemas dropped the runtime-owned synthetic marker while - // the local Part projection retained it. Accept only that exact historical skew. - const legacySynthetic = - !("synthetic" in sourceData) && "synthetic" in projected.data && projected.data.synthetic === true - const expectedData = legacySynthetic ? { ...sourceData, synthetic: true } : sourceData - if (!isDeepStrictEqual(projected.data, expectedData)) - return yield* fail( - prepared.artifacts[0]!.descriptor.id, - `Legacy file-part event ${row.id} no longer matches its projected part`, - ) - const canonicalData = { - ...Object.fromEntries( - Object.entries(canonicalPart as Record).filter( - ([key]) => key !== "id" && key !== "sessionID" && key !== "messageID", - ), - ), - ...(legacySynthetic ? { synthetic: true } : {}), - } as typeof PartTable.$inferInsert.data - yield* db - .update(PartTable) - .set({ data: canonicalData }) - .where(eq(PartTable.id, originalBinding.partID as typeof PartTable.$inferSelect.id)) - .run() - .pipe(Effect.orDie) - }), - { discard: true }, - ) - return { processed: rows.length, next: rows.at(-1)?.id } - }), - { behavior: "immediate" }, - ) - .pipe(Effect.orDie) + // Older FilePart event schemas dropped the runtime-owned synthetic marker while + // the local Part projection retained it. Accept only that exact historical skew. + const legacySynthetic = + !("synthetic" in sourceData) && "synthetic" in projected.data && projected.data.synthetic === true + const expectedData = legacySynthetic ? { ...sourceData, synthetic: true } : sourceData + if (!isDeepStrictEqual(projected.data, expectedData)) + return yield* fail( + prepared.artifacts[0]!.descriptor.id, + `Legacy file-part event ${row.id} no longer matches its projected part`, + ) + const canonicalEventData = legacySynthetic + ? { ...prepared.data, part: { ...(canonicalPart as Record), synthetic: true } } + : prepared.data + const canonicalData = Object.fromEntries( + Object.entries(canonicalEventData.part as Record).filter( + ([key]) => key !== "id" && key !== "sessionID" && key !== "messageID", + ), + ) as typeof PartTable.$inferInsert.data + yield* put(db, prepared.artifacts[0]!, input?.now) + yield* bind(db, { + eventID: row.id, + aggregateID: row.aggregateID, + seq: row.seq, + type: "message.part.updated", + data: canonicalEventData, + originalData: source.data, + now: input?.now, + }) + yield* db + .update(PartTable) + .set({ data: canonicalData }) + .where(eq(PartTable.id, originalBinding.partID as typeof PartTable.$inferSelect.id)) + .run() + .pipe(Effect.orDie) + return { processed: 1, next: row.id } + }), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie) + }) } function validate(descriptor: Descriptor, chunks: readonly Uint8Array[]) { @@ -774,6 +829,9 @@ function validate(descriptor: Descriptor, chunks: readonly Uint8Array[]) { function stage(db: Database.Interface["db"], metadata: Metadata) { return Effect.gen(function* () { yield* requireAvailable(db, metadata.descriptor) + if (!matchesDataHash(metadata.canonicalDataHash, metadata.canonicalData)) + return yield* fail(metadata.descriptor.id, "Artifact import metadata canonical hash is invalid") + const canonicalData = JSON.parse(CanonicalJson.stringify(metadata.canonicalData)) as Record const chunks = yield* db .select({ index: FilePartArtifactChunkTable.chunk_index, hash: FilePartArtifactChunkTable.chunk_hash, data: FilePartArtifactChunkTable.data }) .from(FilePartArtifactChunkTable) @@ -806,7 +864,7 @@ function stage(db: Database.Interface["db"], metadata: Metadata) { existing.artifact_id !== metadata.descriptor.id || existing.original_data_hash !== metadata.originalDataHash || existing.canonical_data_hash !== metadata.canonicalDataHash || - !isDeepStrictEqual(existing.canonical_data, metadata.canonicalData) + !isDeepStrictEqual(existing.canonical_data, canonicalData) ) return yield* fail(metadata.descriptor.id, "Artifact import receipt conflicts with durable CAS") return @@ -838,7 +896,7 @@ function stage(db: Database.Interface["db"], metadata: Metadata) { artifact_id: metadata.descriptor.id, original_data_hash: metadata.originalDataHash, canonical_data_hash: metadata.canonicalDataHash, - canonical_data: metadata.canonicalData, + canonical_data: canonicalData, created_at: Date.now(), }) .run() diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index a349116b9..dffa414a6 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -390,6 +390,26 @@ export const layer = Layer.effectDiscard( and( eq(EventArtifactTable.aggregate_id, aggregateID), sql`json_extract(${EventArtifactTable.canonical_data}, '$.info.id') = ${MessageTable.id}`, + sql`( + EXISTS ( + SELECT 1 FROM session_diff_migration_receipt receipt + WHERE receipt.message_id = ${MessageTable.id} + AND receipt.artifact_id = ${EventArtifactTable.artifact_id} + AND receipt.state = 'committed' + ) OR ( + NOT EXISTS ( + SELECT 1 FROM session_diff_migration_receipt receipt + WHERE receipt.message_id = ${MessageTable.id} + AND receipt.state = 'committed' + ) AND NOT EXISTS ( + SELECT 1 FROM event_artifact newer + WHERE newer.aggregate_id = ${EventArtifactTable.aggregate_id} + AND newer.kind = ${EventArtifactTable.kind} + AND newer.seq > ${EventArtifactTable.seq} + AND json_extract(newer.canonical_data, '$.info.id') = ${MessageTable.id} + ) + ) + )`, ), ) .where( diff --git a/packages/core/src/v1/session.ts b/packages/core/src/v1/session.ts index 8e5cc0010..79676f1e2 100644 --- a/packages/core/src/v1/session.ts +++ b/packages/core/src/v1/session.ts @@ -390,12 +390,22 @@ const DiffManifestDescriptor = Schema.Struct({ truncatedFiles: NonNegativeInt, }).annotate({ identifier: "SnapshotDiffManifestDescriptor" }) -const DiffArtifactDescriptor = Schema.Struct({ - id: Schema.String, - hash: Schema.String, - codec: Schema.Literals(["legacy-message-diff.v1", "legacy-message-diff.v2"]), - fileCount: NonNegativeInt, -}).annotate({ identifier: "SnapshotDiffArtifactDescriptor" }) +const DiffArtifactDescriptor = Schema.Union([ + Schema.Struct({ + id: Schema.String, + hash: Schema.String, + codec: Schema.Literal("legacy-message-diff.v1"), + fileCount: NonNegativeInt, + }), + Schema.Struct({ + id: Schema.String, + hash: Schema.String, + codec: Schema.Literal("legacy-message-diff.v2"), + fileCount: NonNegativeInt, + previewFileCount: NonNegativeInt, + previewTruncated: Schema.Boolean, + }), +]).annotate({ identifier: "SnapshotDiffArtifactDescriptor" }) export const User = Schema.Struct({ ...messageBase, diff --git a/packages/core/test/database-migration.test.ts b/packages/core/test/database-migration.test.ts index a67348b8f..ef03d6d26 100644 --- a/packages/core/test/database-migration.test.ts +++ b/packages/core/test/database-migration.test.ts @@ -29,6 +29,7 @@ import partIntegrityBackfillMigration from "@deepagent-code/core/database/migrat import legacyActivityProgressMigration from "@deepagent-code/core/database/migration/20260811090000_legacy_activity_progress" import legacyActivityOwnerMigration from "@deepagent-code/core/database/migration/20260811100000_legacy_activity_owner" import eventSnapshotAuthorityMigration from "@deepagent-code/core/database/migration/20260813100000_event_snapshot_authority" +import eventImmutableMigration from "@deepagent-code/core/database/migration/20260813135000_event_immutable" import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" import { Database } from "@deepagent-code/core/database/database" import { tmpdir } from "./fixture/tmpdir" @@ -89,6 +90,47 @@ describe("DatabaseMigration", () => { }), ) }) + + test("keeps EventV2 rows append-only after bounded maintenance is installed", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`CREATE TABLE event (id TEXT PRIMARY KEY, data TEXT NOT NULL)`) + yield* db.run(sql`CREATE TABLE event_sequence (aggregate_id TEXT PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE event_artifact (artifact_id TEXT PRIMARY KEY, aggregate_id TEXT NOT NULL)`) + yield* db.run(sql`CREATE TABLE event_artifact_chunk (artifact_id TEXT NOT NULL)`) + yield* db.run(sql`CREATE TABLE file_part_artifact ( + artifact_id TEXT PRIMARY KEY, body_hash TEXT NOT NULL, body_bytes INTEGER NOT NULL, + chunk_bytes INTEGER NOT NULL, chunk_count INTEGER NOT NULL, codec_version INTEGER NOT NULL, + complete INTEGER NOT NULL, created_at INTEGER NOT NULL + )`) + yield* db.run(sql`CREATE TABLE file_part_artifact_chunk (artifact_id TEXT NOT NULL)`) + yield* db.run(sql`CREATE TABLE file_part_artifact_binding ( + aggregate_id TEXT NOT NULL, artifact_id TEXT NOT NULL + )`) + yield* db.run(sql`CREATE TABLE file_part_artifact_import (artifact_id TEXT NOT NULL)`) + yield* db.run(sql`CREATE TABLE session_diff_artifact_file (artifact_id TEXT NOT NULL)`) + yield* db.run(sql`CREATE TABLE session_diff_artifact_file_chunk (artifact_id TEXT NOT NULL)`) + yield* db.run(sql`CREATE TABLE session (id TEXT PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE session_diff_migration_receipt ( + artifact_id TEXT NOT NULL, session_id TEXT NOT NULL, state TEXT NOT NULL + )`) + yield* db.run(sql`INSERT INTO event VALUES ('event-a', '{}')`) + yield* DatabaseMigration.applyOnly(db, [eventImmutableMigration]) + + const updated = yield* db.run(sql`UPDATE event SET data = '{"changed":true}' WHERE id = 'event-a'`).pipe(Effect.exit) + expect(String(updated)).toContain("event_update_immutable") + expect(yield* db.get(sql`SELECT data FROM event WHERE id = 'event-a'`)).toEqual({ data: "{}" }) + + yield* db.run(sql`INSERT INTO file_part_artifact VALUES ('artifact-a', 'hash-a', 1, 1, 1, 1, 0, 1)`) + yield* db.run(sql`UPDATE file_part_artifact SET complete = 1 WHERE artifact_id = 'artifact-a'`) + const artifactUpdated = yield* db + .run(sql`UPDATE file_part_artifact SET body_hash = 'hash-b' WHERE artifact_id = 'artifact-a'`) + .pipe(Effect.exit) + expect(String(artifactUpdated)).toContain("file_part_artifact_update_immutable") + }), + ) + }) test("serializes concurrent embedded initialization for one database path", async () => { await using tmp = await tmpdir() const filename = path.join(tmp.path, "embedded.sqlite") @@ -260,6 +302,35 @@ describe("DatabaseMigration", () => { ) }) + test("rejects divergent and gapped migration histories before running DDL", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`CREATE TABLE migration (id TEXT PRIMARY KEY, time_completed INTEGER NOT NULL)`) + yield* db.run(sql`CREATE TABLE sentinel (value TEXT NOT NULL)`) + yield* db.run(sql`INSERT INTO sentinel VALUES ('unchanged')`) + yield* db.run(sql`INSERT INTO migration VALUES (${migrations[1]!.id}, 1)`) + const marker = { id: "migration-a", up: () => db.run(sql`DELETE FROM sentinel`) } + const later = { id: "migration-b", up: () => Effect.void } + + const gapped = yield* DatabaseMigration.apply(db).pipe(Effect.exit) + expect(String(gapped)).toContain("database migration history") + expect(yield* db.all(sql`SELECT value FROM sentinel`)).toEqual([{ value: "unchanged" }]) + + yield* db.run(sql`DELETE FROM migration`) + yield* db.run(sql`INSERT INTO migration VALUES ('incompatible-lineage', 1)`) + const divergent = yield* DatabaseMigration.apply(db).pipe(Effect.exit) + expect(String(divergent)).toContain("incompatible lineage") + expect(yield* db.all(sql`SELECT value FROM sentinel`)).toEqual([{ value: "unchanged" }]) + + // Focused migration tests intentionally apply slices and remain independent of the global journal. + yield* db.run(sql`DELETE FROM migration`) + yield* DatabaseMigration.applyOnly(db, [marker, later]) + expect(yield* db.all(sql`SELECT value FROM sentinel`)).toEqual([]) + }), + ) + }) + test("legacy activity migration backfills V2 source identity and rejects mismatched legacy admissions", async () => { await run( Effect.gen(function* () { diff --git a/packages/core/test/event-snapshot.test.ts b/packages/core/test/event-snapshot.test.ts index 671ca8b59..2c2528d70 100644 --- a/packages/core/test/event-snapshot.test.ts +++ b/packages/core/test/event-snapshot.test.ts @@ -5,6 +5,8 @@ import { asc, eq } from "drizzle-orm" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" import { + EventArtifactChunkTable, + EventArtifactTable, EventCompactionReceiptTable, EventDedupeTable, EventSequenceTable, @@ -21,6 +23,7 @@ import { MessageTable, PartTable, SessionInputTable, SessionHistoryStateTable, S import { SessionMessage } from "@deepagent-code/core/session/message" import { SessionProjector } from "@deepagent-code/core/session/projector" import { SessionV1 } from "@deepagent-code/core/v1/session" +import { Hash } from "@deepagent-code/core/util/hash" import { testEffect } from "./lib/effect" import { tmpdir } from "./fixture/tmpdir" @@ -44,6 +47,47 @@ const message = (id: string, text: string) => ({ }, }) +const artifact = (input: { id: string; eventID: EventV2.ID; seq: number; messageID: string; label: string }) => { + const canonicalData = { + sessionID, + info: { + id: input.messageID, + sessionID, + role: "user", + time: { created: 1 }, + agent: "build", + model: { providerID: "test", modelID: "test" }, + system: input.label, + summary: { + diffs: [], + diffArtifact: { + id: input.id, + hash: Hash.sha256(`body-${input.label}`), + codec: "legacy-message-diff.v2", + fileCount: 300, + previewFileCount: 200, + previewTruncated: true, + }, + }, + }, + } + return { + artifact_id: input.id, + event_id: input.eventID, + aggregate_id: sessionID, + seq: input.seq, + kind: "legacy_message_diff" as const, + original_data_hash: Hash.sha256(`source-${input.label}`), + canonical_data_hash: Hash.sha256(JSON.stringify(canonicalData)), + canonical_data: canonicalData, + body_hash: Hash.sha256(`body-${input.label}`), + body_bytes: 0, + chunk_count: 1, + codec_version: 2, + created_at: input.seq, + } +} + describe("EventV2 canonical projection snapshots", () => { it.effect("stages bounded checkpoints, imports atomically into a fresh database, and continues at the next event", () => Effect.gen(function* () { @@ -192,4 +236,135 @@ describe("EventV2 canonical projection snapshots", () => { expect(yield* sourceDb.select().from(EventCompactionReceiptTable).where(eq(EventCompactionReceiptTable.aggregate_id, sessionID)).get().pipe(Effect.orDie)).toMatchObject({ state: "complete", cursor_seq: 2 }) }), ) + + it.effect("checkpoints the newest diff artifact when one Message has multiple historical artifacts", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const db = (yield* Database.Service).db + yield* db + .insert(ProjectTable) + .values({ id: projectID, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + .pipe(Effect.orDie) + const info = SessionV1.SessionInfo.make({ + id: sessionID, + slug: "snapshot-artifact-authority", + version: "test", + projectID, + directory: "/project", + title: "snapshot-artifact-authority", + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 1, updated: 1 }, + }) + const messageID = SessionV1.MessageID.make("msg_snapshot_artifact_authority") + yield* events.publish(SessionV1.Event.Created, { sessionID, info }) + yield* events.publish(SessionV1.Event.MessageUpdated, message(messageID, "physical")) + const sourceEvents = yield* db + .select() + .from(EventTable) + .where(eq(EventTable.aggregate_id, sessionID)) + .orderBy(asc(EventTable.seq)) + .all() + .pipe(Effect.orDie) + const old = artifact({ + id: "evtart_old", + eventID: sourceEvents[0]!.id, + seq: sourceEvents[0]!.seq, + messageID, + label: "old", + }) + const latest = artifact({ + id: "evtart_latest", + eventID: sourceEvents[1]!.id, + seq: sourceEvents[1]!.seq, + messageID, + label: "latest", + }) + yield* db.insert(EventArtifactTable).values([old, latest]).run().pipe(Effect.orDie) + yield* db + .insert(EventArtifactChunkTable) + .values([ + { artifact_id: old.artifact_id, chunk_index: 0, data: Buffer.alloc(0), chunk_hash: Hash.sha256(Buffer.alloc(0)) }, + { artifact_id: latest.artifact_id, chunk_index: 0, data: Buffer.alloc(0), chunk_hash: Hash.sha256(Buffer.alloc(0)) }, + ]) + .run() + .pipe(Effect.orDie) + yield* db + .update(MessageTable) + .set({ data: latest.canonical_data.info as typeof MessageTable.$inferInsert.data }) + .where(eq(MessageTable.id, messageID)) + .run() + .pipe(Effect.orDie) + yield* db.run(` + INSERT INTO session_diff_migration_receipt ( + message_id, session_id, artifact_id, source_event_id, + expected_message_data_hash, committed_message_data_hash, + expected_session_summary_hash, committed_session_summary_hash, + canonicalizer_version, canonicalization_version, epoch_hashes, + state, created_at, updated_at, committed_at + ) VALUES ( + '${messageID}', '${sessionID}', '${latest.artifact_id}', '${latest.event_id}', + '${Hash.sha256("expected-message")}', '${Hash.sha256("committed-message")}', + '${Hash.sha256("expected-session")}', '${Hash.sha256("null")}', + 2, 1, '[]', 'committed', 1, 1, 1 + ) + `).pipe(Effect.orDie) + + const snapshot = yield* events.checkpoint({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(1), + expectedLatest: EventV2.Cursor.make(1), + codec: "session-projection", + schemaVersion: 1, + }) + const rows = yield* events.snapshotRows!({ snapshotID: snapshot.snapshotID, after: -1, limit: 100 }) + const row = rows.find((candidate) => candidate.tableName === "message" && candidate.rowKey === messageID)! + const chunks = yield* events.snapshotChunks!({ rowHash: row.rowHash, after: -1 }) + const value = JSON.parse(Buffer.concat(chunks.map((chunk) => chunk.data)).toString()) as { + data: { summary?: { diffArtifact?: { id: string; previewFileCount: number; previewTruncated: boolean } } } + } + expect(value.data.summary?.diffArtifact).toEqual( + expect.objectContaining({ id: latest.artifact_id, previewFileCount: 200, previewTruncated: true }), + ) + expect(JSON.stringify(value)).not.toContain(old.artifact_id) + + const tmp = yield* Effect.acquireRelease( + Effect.promise(() => tmpdir()), + (value) => Effect.promise(() => value[Symbol.asyncDispose]()), + ) + const targetDatabase = Database.layerFromPath(path.join(tmp.path, "artifact-target.sqlite")) + const targetEvents = EventV2.layer.pipe(Layer.provide(targetDatabase)) + const targetProjector = SessionProjector.layer.pipe(Layer.provide(targetEvents), Layer.provide(targetDatabase)) + yield* Effect.gen(function* () { + const target = yield* EventV2.Service + const targetDb = (yield* Database.Service).db + yield* targetDb + .insert(ProjectTable) + .values({ id: projectID, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + .pipe(Effect.orDie) + yield* target.stageSnapshotRows!(snapshot, rows) + for (const snapshotRow of rows) + yield* target.stageSnapshotChunks!( + snapshot, + snapshotRow, + yield* events.snapshotChunks!({ rowHash: snapshotRow.rowHash, after: -1 }), + ) + yield* target.importSnapshot(snapshot) + const imported = yield* targetDb + .select({ data: MessageTable.data }) + .from(MessageTable) + .where(eq(MessageTable.id, messageID)) + .get() + .pipe(Effect.orDie) + expect( + imported?.data.role === "user" && imported.data.summary && typeof imported.data.summary === "object" + ? imported.data.summary.diffArtifact?.id + : undefined, + ).toBe(latest.artifact_id) + expect(JSON.stringify(imported)).not.toContain('"patch"') + }).pipe(Effect.provide(Layer.fresh(Layer.mergeAll(targetDatabase, targetEvents, targetProjector)))) + }), + ) }) diff --git a/packages/core/test/event.test.ts b/packages/core/test/event.test.ts index 5624178fd..4a8226e4e 100644 --- a/packages/core/test/event.test.ts +++ b/packages/core/test/event.test.ts @@ -998,6 +998,53 @@ describe("EventV2", () => { }), ) + it.effect("fails closed before writing an overlarge legacy diff manifest", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const { db } = yield* Database.Service + const aggregateID = EventV2.ID.create() + yield* events.publish(SyncMessage, { id: aggregateID, text: "zero" }) + const id = EventV2.ID.make("evt_legacy_message_diff_overlarge_manifest") + const diffs = Array.from({ length: EventV2.LEGACY_ARTIFACT_MAX_FILES + 1 }, (_, index) => ({ + file: `file-${index}.patch`, + patch: "x".repeat(450), + additions: 1, + deletions: 0, + })) + const data = { sessionID: aggregateID, info: { summary: { diffs } } } + const syncSeq = yield* db + .update(EventSyncSequenceTable) + .set({ seq: sql`${EventSyncSequenceTable.seq} + 1` }) + .where(eq(EventSyncSequenceTable.id, 1)) + .returning({ seq: EventSyncSequenceTable.seq }) + .get() + .pipe(Effect.orDie) + yield* db + .insert(EventTable) + .values({ + id, + aggregate_id: aggregateID, + seq: 1, + type: EventV2.versionedType("message.updated", 1), + data, + sync_seq: syncSeq!.seq, + }) + .run() + .pipe(Effect.orDie) + yield* db + .update(EventSequenceTable) + .set({ seq: 1 }) + .where(eq(EventSequenceTable.aggregate_id, aggregateID)) + .run() + .pipe(Effect.orDie) + + const result = yield* events.canonicalizeLegacyArtifacts({ limit: 1, now: 11 }).pipe(Effect.exit) + expect(String(result)).toContain("EventV2.EncodedPayloadTooLarge") + expect(yield* db.select().from(EventArtifactTable).where(eq(EventArtifactTable.event_id, id)).all()).toEqual([]) + expect(yield* db.select().from(EventArtifactChunkTable).all()).toEqual([]) + }), + ) + it.effect("uses custom sync aggregate field", () => Effect.gen(function* () { const events = yield* EventV2.Service diff --git a/packages/core/test/file-part-artifact.test.ts b/packages/core/test/file-part-artifact.test.ts index 293bc70dc..60e0b2229 100644 --- a/packages/core/test/file-part-artifact.test.ts +++ b/packages/core/test/file-part-artifact.test.ts @@ -18,6 +18,7 @@ import { AbsolutePath } from "@deepagent-code/core/schema" import { Project } from "@deepagent-code/core/project" import { ProjectTable } from "@deepagent-code/core/project/sql" import { MessageTable, PartTable, SessionTable } from "@deepagent-code/core/session/sql" +import { SessionProjector } from "@deepagent-code/core/session/projector" import { testEffect } from "./lib/effect" const Prelude = EventV2.define({ @@ -31,12 +32,19 @@ const runtime = () => { return Layer.mergeAll(database, EventV2.layer.pipe(Layer.provide(database))) } +const projectedRuntime = () => { + const database = Database.layerFromPath(":memory:") + const events = EventV2.layer.pipe(Layer.provide(database)) + const projector = SessionProjector.layer.pipe(Layer.provide(events), Layer.provide(database)) + return Layer.mergeAll(database, events, projector) +} + const it = testEffect(Layer.empty) describe("FilePartArtifact", () => { it.effect("externalizes one bounded legacy inline file-part batch idempotently", () => Effect.gen(function* () { - const source = yield* Layer.buildWithScope(Layer.fresh(runtime()), yield* Effect.scope) + const source = yield* Layer.buildWithScope(Layer.fresh(projectedRuntime()), yield* Effect.scope) const events = Context.get(source, EventV2.Service) const db = Context.get(source, Database.Service).db const sessionID = SessionSchema.ID.make("ses_legacy_file_part") @@ -137,10 +145,11 @@ describe("FilePartArtifact", () => { .where(eq(FilePartArtifactBindingTable.event_id, eventID)) .get() .pipe(Effect.orDie) - expect(binding!.original_data_hash).toBe(Hash.sha256(JSON.stringify(data))) + expect(binding!.original_data_hash).toBe(FilePartArtifact.dataHash(data)) const descriptor = FilePartArtifact.descriptor(binding!.canonical_data) expect(descriptor).toBeDefined() if (!descriptor) return + expect(binding!.canonical_data).toMatchObject({ part: { synthetic: true } }) expect(Buffer.byteLength(JSON.stringify(binding!.canonical_data))).toBeLessThan(2_000) const projected = yield* db.select().from(PartTable).where(eq(PartTable.id, partID)).get().pipe(Effect.orDie) expect(Buffer.byteLength(JSON.stringify(projected!.data))).toBeLessThan(1_000) @@ -162,8 +171,119 @@ describe("FilePartArtifact", () => { type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), data, }) + const divergent = yield* events + .replay({ + id: eventID, + aggregateID: sessionID, + seq: 1, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + data: { ...data, part: { ...data.part, synthetic: false } }, + }) + .pipe(Effect.exit) + expect(divergent._tag).toBe("Failure") expect(yield* db.select().from(EventTable).where(eq(EventTable.id, eventID)).all()).toHaveLength(1) expect((yield* db.select().from(EventTable).where(eq(EventTable.id, eventID)).get())!.data).toEqual(data) + + const metadata = yield* FilePartArtifact.metadata(db, { + eventID, + aggregateID: sessionID, + seq: 1, + artifactID: descriptor.id, + }) + const artifactChunks = yield* Effect.forEach(metadata.chunkHashes, (hash, index) => + FilePartArtifact.chunk(db, { artifactID: descriptor.id, index, expectedHash: hash }), + ) + + const replayTarget = yield* Layer.buildWithScope(Layer.fresh(projectedRuntime()), yield* Effect.scope) + const replayDb = Context.get(replayTarget, Database.Service).db + const replayEvents = Context.get(replayTarget, EventV2.Service) + yield* replayDb + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + .pipe(Effect.orDie) + yield* replayDb + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "legacy-replay", + directory: AbsolutePath.make("/project"), + title: "legacy-replay", + version: "test", + }) + .run() + .pipe(Effect.orDie) + yield* replayDb + .insert(MessageTable) + .values({ id: messageID, session_id: sessionID, time_created: 1, data: { role: "user", time: { created: 1 } } } as typeof MessageTable.$inferInsert) + .run() + .pipe(Effect.orDie) + yield* replayDb + .insert(EventSequenceTable) + .values({ aggregate_id: sessionID, seq: 0 }) + .run() + .pipe(Effect.orDie) + yield* Effect.forEach( + artifactChunks, + (chunk, index) => + FilePartArtifact.importChunk({ + metadata, + index, + hash: metadata.chunkHashes[index]!, + data: chunk, + }).pipe(Effect.provide(replayTarget)), + { discard: true }, + ) + yield* FilePartArtifact.stageImport(metadata).pipe(Effect.provide(replayTarget)) + yield* replayEvents.replay({ + id: eventID, + aggregateID: sessionID, + seq: 1, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + data: binding!.canonical_data, + }) + expect((yield* replayDb.select().from(PartTable).where(eq(PartTable.id, partID)).get())?.data).toEqual(projected!.data) + + const snapshot = yield* events.checkpoint({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(1), + expectedLatest: EventV2.Cursor.make(1), + codec: "session-projection", + schemaVersion: 1, + }) + const snapshotRows = yield* events.snapshotRows!({ snapshotID: snapshot.snapshotID }) + const snapshotTarget = yield* Layer.buildWithScope(Layer.fresh(projectedRuntime()), yield* Effect.scope) + const snapshotDb = Context.get(snapshotTarget, Database.Service).db + const snapshotEvents = Context.get(snapshotTarget, EventV2.Service) + yield* snapshotDb + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/project"), sandboxes: [] }) + .run() + .pipe(Effect.orDie) + yield* Effect.forEach( + artifactChunks, + (chunk, index) => + FilePartArtifact.importChunk({ + metadata, + index, + hash: metadata.chunkHashes[index]!, + data: chunk, + }).pipe(Effect.provide(snapshotTarget)), + { discard: true }, + ) + yield* FilePartArtifact.stageImport(metadata).pipe(Effect.provide(snapshotTarget)) + yield* snapshotEvents.stageSnapshotRows!(snapshot, snapshotRows) + yield* Effect.forEach( + snapshotRows, + (row) => + events.snapshotChunks!({ rowHash: row.rowHash }).pipe( + Effect.flatMap((chunks) => snapshotEvents.stageSnapshotChunks!(snapshot, row, chunks)), + ), + { discard: true }, + ) + yield* snapshotEvents.importSnapshot(snapshot) + expect((yield* snapshotDb.select().from(PartTable).where(eq(PartTable.id, partID)).get())?.data).toEqual(projected!.data) }), ) diff --git a/packages/deepagent-code/script/bug-407-010-release-gate.ts b/packages/deepagent-code/script/bug-407-010-release-gate.ts index aec46471d..cc9366b03 100644 --- a/packages/deepagent-code/script/bug-407-010-release-gate.ts +++ b/packages/deepagent-code/script/bug-407-010-release-gate.ts @@ -7,6 +7,8 @@ import { EventCompactionReceiptTable, EventSequenceTable, EventSnapshotTable, + EventSnapshotChunkTable, + EventSnapshotRowTable, EventSyncBackfillTable, EventSyncIndexTable, EventTable, @@ -14,9 +16,10 @@ import { import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" import { FilePartArtifactBindingTable } from "@deepagent-code/core/file-part-artifact.sql" import { SessionProjector } from "@deepagent-code/core/session/projector" -import { SessionTable } from "@deepagent-code/core/session/sql" -import { Cause, Effect, Exit, Layer } from "effect" -import { eq, sql } from "drizzle-orm" +import { MessageTable, SessionTable } from "@deepagent-code/core/session/sql" +import { Hash } from "@deepagent-code/core/util/hash" +import { Cause, Effect, Exit, Layer, Option, Schema } from "effect" +import { and, asc, eq, sql } from "drizzle-orm" import path from "node:path" import { parseArgs } from "node:util" import { SessionDiffArtifact } from "@/session/diff-artifact" @@ -147,7 +150,79 @@ const program = Effect.gen(function* () { if (diffMigration.failed > 0) return yield* Effect.die(new Error(`Legacy Session diff migration failed: ${JSON.stringify(diffMigration)}`)) - yield* stage("checkpoint-and-compact", Effect.forEach(sessions, (sessionID) => Effect.gen(function* () { + const checkpointSessions = [...new Set([ + ...sessions, + ...artifactSessions.map((row) => SessionID.make(row.id)), + ])] + + const validateSnapshotDiffAuthority = Effect.fn("Bug407010ReleaseGate.validateSnapshotDiffAuthority")( + function* (input: { sessionID: SessionID; snapshotID: string }) { + const receipts = yield* db + .select({ + messageID: SessionDiffMigrationReceiptTable.message_id, + artifactID: SessionDiffMigrationReceiptTable.artifact_id, + messageData: MessageTable.data, + }) + .from(SessionDiffMigrationReceiptTable) + .innerJoin(MessageTable, eq(MessageTable.id, SessionDiffMigrationReceiptTable.message_id)) + .where(and( + eq(SessionDiffMigrationReceiptTable.session_id, input.sessionID), + eq(SessionDiffMigrationReceiptTable.state, "committed"), + )) + .all() + .pipe(Effect.orDie) + for (const receipt of receipts) { + const row = yield* db + .select() + .from(EventSnapshotRowTable) + .where(and( + eq(EventSnapshotRowTable.snapshot_id, input.snapshotID), + eq(EventSnapshotRowTable.table_name, "message"), + eq(EventSnapshotRowTable.row_key, receipt.messageID), + )) + .get() + .pipe(Effect.orDie) + if (!row) + return yield* Effect.die(new Error(`Snapshot ${input.snapshotID} is missing Message ${receipt.messageID}`)) + const chunks = yield* db + .select() + .from(EventSnapshotChunkTable) + .where(eq(EventSnapshotChunkTable.row_hash, row.row_hash)) + .orderBy(asc(EventSnapshotChunkTable.chunk_index)) + .all() + .pipe(Effect.orDie) + if ( + chunks.length !== row.chunk_count || + chunks.some((chunk, index) => chunk.chunk_index !== index || Hash.sha256(chunk.data) !== chunk.chunk_hash) + ) + return yield* Effect.die(new Error(`Snapshot ${input.snapshotID} has invalid chunks for Message ${receipt.messageID}`)) + const body = Buffer.concat(chunks.map((chunk) => chunk.data)) + if (body.length !== row.row_bytes || Hash.sha256(body) !== row.row_hash) + return yield* Effect.die(new Error(`Snapshot ${input.snapshotID} has an invalid row hash for Message ${receipt.messageID}`)) + const decoded = Option.getOrUndefined(Schema.decodeUnknownOption(Schema.UnknownFromJsonString)(body.toString())) + const snapshotData = record(decoded) && record(decoded.data) ? decoded.data : undefined + const physicalDescriptor = diffArtifactDescriptor(receipt.messageData) + const snapshotDescriptor = diffArtifactDescriptor(snapshotData) + if ( + !physicalDescriptor || + !snapshotDescriptor || + physicalDescriptor.id !== receipt.artifactID || + snapshotDescriptor.id !== receipt.artifactID || + snapshotDescriptor.fileCount !== physicalDescriptor.fileCount || + snapshotDescriptor.previewFileCount !== physicalDescriptor.previewFileCount || + snapshotDescriptor.previewTruncated !== physicalDescriptor.previewTruncated + ) + return yield* Effect.die( + new Error(`Snapshot ${input.snapshotID} does not match committed diff authority for Message ${receipt.messageID}`), + ) + if (JSON.stringify(snapshotData).includes('"patch"')) + return yield* Effect.die(new Error(`Snapshot ${input.snapshotID} retained an inline patch for Message ${receipt.messageID}`)) + } + return { receipts: receipts.length } + }, + ) + + yield* stage("checkpoint-and-compact", Effect.forEach(checkpointSessions, (sessionID) => Effect.gen(function* () { const session = yield* db .select({ id: SessionTable.id }) .from(SessionTable) @@ -183,6 +258,7 @@ const program = Effect.gen(function* () { } return yield* events.finalizeCheckpoint!({ snapshotID: attempt.snapshotID }) }) + const authority = yield* validateSnapshotDiffAuthority({ sessionID, snapshotID: snapshot.snapshotID }) let compacted = { deleted: 0, complete: false } let deleted = 0 let batches = 0 @@ -197,7 +273,7 @@ const program = Effect.gen(function* () { } yield* db.run("PRAGMA shrink_memory").pipe(Effect.orDie) yield* Effect.sync(() => Bun.gc(true)) - return { sessionID, snapshotID: snapshot.snapshotID, throughSeq: snapshot.throughSeq, deleted, batches } + return { sessionID, snapshotID: snapshot.snapshotID, throughSeq: snapshot.throughSeq, deleted, batches, authority } }), { concurrency: 1 })) const after = yield* db.all<{ @@ -264,3 +340,26 @@ if (Exit.isFailure(exit)) { for (const error of Cause.prettyErrors(exit.cause)) console.error(error) process.exit(1) } + +function record(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value) +} + +function diffArtifactDescriptor(value: unknown) { + if (!record(value) || !record(value.summary) || !record(value.summary.diffArtifact)) return + const descriptor = value.summary.diffArtifact + if ( + typeof descriptor.id !== "string" || + typeof descriptor.fileCount !== "number" || + typeof descriptor.previewFileCount !== "number" || + typeof descriptor.previewTruncated !== "boolean" || + descriptor.previewTruncated !== (descriptor.previewFileCount < descriptor.fileCount) + ) + return + return { + id: descriptor.id, + fileCount: descriptor.fileCount, + previewFileCount: descriptor.previewFileCount, + previewTruncated: descriptor.previewTruncated, + } +} diff --git a/packages/deepagent-code/src/control-plane/workspace.ts b/packages/deepagent-code/src/control-plane/workspace.ts index 9701fb481..bef62279e 100644 --- a/packages/deepagent-code/src/control-plane/workspace.ts +++ b/packages/deepagent-code/src/control-plane/workspace.ts @@ -1,4 +1,5 @@ import { Context, Effect, FiberMap, Layer, Option, Schema, Stream } from "effect" +import { isDeepStrictEqual } from "node:util" import { serviceUse } from "@deepagent-code/core/effect/service-use" import { FetchHttpClient, HttpBody, HttpClient, HttpClientError, HttpClientRequest } from "effect/unstable/http" import { Database } from "@deepagent-code/core/database/database" @@ -438,7 +439,8 @@ export const layer = Layer.effect( metadata.eventID !== item.id || metadata.aggregateID !== item.aggregate_id || metadata.seq !== item.seq || - metadata.canonicalDataHash !== Hash.sha256(JSON.stringify(item.data)) || + !FilePartArtifact.matchesDataHash(metadata.canonicalDataHash, metadata.canonicalData) || + !isDeepStrictEqual(metadata.canonicalData, item.data) || JSON.stringify(metadata.descriptor) !== JSON.stringify(descriptor) ) return yield* new SyncHttpError({ message: "Workspace artifact metadata diverged from history", status: 409 }) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts index b39b33b93..8f43cf77c 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/groups/session.ts @@ -389,7 +389,7 @@ export const SessionApi = HttpApi.make("session") params: { sessionID: SessionID }, query: WorkspaceRoutingQuery, success: described(Session.Info, "Get session"), - error: [HttpApiError.BadRequest, ApiNotFoundError], + error: [HttpApiError.BadRequest, ApiNotFoundError, ServiceUnavailableError], }).annotateMerge( OpenApi.annotations({ identifier: "session.get", @@ -401,7 +401,7 @@ export const SessionApi = HttpApi.make("session") params: { sessionID: SessionID }, query: WorkspaceRoutingQuery, success: described(Schema.Array(Session.Info), "List of children"), - error: [HttpApiError.BadRequest, ApiNotFoundError], + error: [HttpApiError.BadRequest, ApiNotFoundError, ServiceUnavailableError], }).annotateMerge( OpenApi.annotations({ identifier: "session.children", @@ -456,7 +456,7 @@ export const SessionApi = HttpApi.make("session") }), "Legacy Session diff migration batch result", ), - error: [HttpApiError.BadRequest, ApiNotFoundError], + error: [HttpApiError.BadRequest, ApiNotFoundError, ServiceUnavailableError], }).annotateMerge( OpenApi.annotations({ identifier: "session.diffArtifactMaintenance", diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts index d137cf231..9292cdcdc 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/session.ts @@ -49,7 +49,7 @@ import { SummarizePayload, UpdatePayload, } from "../groups/session" -import { ConflictError, PermissionNotFoundError, notFound } from "../errors" +import { ConflictError, PermissionNotFoundError, ServiceUnavailableError, notFound } from "../errors" import * as SessionError from "./session-errors" import { randomUUID } from "node:crypto" import { getWorkspaceContext } from "../utils/workspace-context" @@ -158,8 +158,10 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", return yield* summary.diff({ sessionID: ctx.params.sessionID, messageID: ctx.query.messageID }) }) - const mapDiffArtifactError = (error: SessionDiffArtifact.Invalid | SessionDiffArtifact.NotFound) => - error instanceof SessionDiffArtifact.NotFound ? notFound(error.message) : new HttpApiError.BadRequest({}) + const mapDiffArtifactError = (error: SessionDiffArtifact.Invalid | SessionDiffArtifact.NotFound) => { + if (error instanceof SessionDiffArtifact.NotFound) return notFound(error.message) + return new HttpApiError.BadRequest({}) + } const diffArtifactMaintenance = Effect.fn("SessionHttpApi.diffArtifactMaintenance")(function* (ctx: { params: { sessionID: SessionID } @@ -169,7 +171,9 @@ export const sessionHandlers = HttpApiBuilder.group(InstanceHttpApi, "session", return yield* SessionDiffArtifact.migrate({ sessionID: ctx.params.sessionID, ...(ctx.payload.limit ? { limit: ctx.payload.limit } : {}), - }).pipe(Effect.mapError(mapDiffArtifactError)) + }).pipe(Effect.mapError((error) => + new ServiceUnavailableError({ service: "session.diff-artifact", message: error.message }), + )) }) const diffArtifactManifest = Effect.fn("SessionHttpApi.diffArtifactManifest")(function* (ctx: { diff --git a/packages/deepagent-code/src/session/diff-artifact.ts b/packages/deepagent-code/src/session/diff-artifact.ts index 012a606b3..d4c8983bf 100644 --- a/packages/deepagent-code/src/session/diff-artifact.ts +++ b/packages/deepagent-code/src/session/diff-artifact.ts @@ -15,7 +15,7 @@ import { } from "./diff-artifact.sql" import { and, asc, eq, gt, sql } from "drizzle-orm" import { createHash } from "node:crypto" -import { Data, Effect, Schema } from "effect" +import { Data, Effect, Schedule, Schema } from "effect" import { Descriptor, File, Limits, Manifest, ManifestFile } from "./diff-artifact-schema" export { Descriptor, File, Limits, Manifest, ManifestFile } from "./diff-artifact-schema" @@ -24,6 +24,12 @@ const FILE_CHUNK_BYTES = 256 * 1024 const MAX_MIGRATION_FILES = 10_000 const MAX_MIGRATION_BODY_BYTES = 64 * 1024 * 1024 const MAX_PATH_BYTES = 4096 +const MAX_MIGRATION_MESSAGES = 1_000 +const MAX_MIGRATION_PARTS = 5_000 +const MAX_MIGRATION_EPOCHS = 64 +const MAX_MIGRATION_EPOCH_MEMBERS = 100_000 +const MAX_MIGRATION_SESSION_JSON_BYTES = 32 * 1024 * 1024 +const MAX_MIGRATION_ROW_JSON_BYTES = 16 * 1024 * 1024 const NULL_SUMMARY_HASH = Hash.sha256("null") export class Invalid extends Data.TaggedError("SessionDiffArtifact.Invalid")<{ @@ -34,6 +40,10 @@ export class NotFound extends Data.TaggedError("SessionDiffArtifact.NotFound")<{ readonly message: string }> {} +export class Retryable extends Data.TaggedError("SessionDiffArtifact.Retryable")<{ + readonly message: string +}> {} + class RewriteCasLost extends Data.TaggedError("SessionDiffArtifact.RewriteCasLost")<{ readonly message: string readonly expectedMessageDataHash: string @@ -45,6 +55,8 @@ type CanonicalArtifact = { hash: string codec: "legacy-message-diff.v1" | "legacy-message-diff.v2" fileCount: number + previewFileCount: number + previewTruncated: boolean } type Candidate = { @@ -134,6 +146,35 @@ export const migrate = Effect.fn("SessionDiffArtifact.migrate")(function* (input }) function migrateCandidate(db: Database.Interface["db"], candidate: Candidate, now: number) { + return Effect.gen(function* () { + const budgetFailure = yield* migrationBudget(db, candidate) + if (budgetFailure) return yield* recordValidationFailure(db, candidate, now, budgetFailure) + return yield* migrateBoundedCandidate(db, candidate, now).pipe( + Effect.retry({ + while: isBusySnapshot, + schedule: Schedule.spaced("25 millis").pipe(Schedule.both(Schedule.recurs(2))), + }), + Effect.catchIf(isBusySnapshot, () => + Effect.fail(new Retryable({ message: `Session diff migration is contended: ${candidate.messageID}` })), + ), + ) + }) +} + +function isBusySnapshot(error: unknown) { + const seen = new Set() + const inspect = (value: unknown): boolean => { + if (typeof value === "string") return value.includes("SQLITE_BUSY") || value.includes("database is locked") + if (!value || typeof value !== "object" || seen.has(value)) return false + seen.add(value) + if (value instanceof Error && (value.message.includes("SQLITE_BUSY") || value.message.includes("database is locked"))) + return true + return "cause" in value && inspect((value as { cause?: unknown }).cause) + } + return inspect(error) +} + +function migrateBoundedCandidate(db: Database.Interface["db"], candidate: Candidate, now: number) { return db .transaction( (tx) => @@ -192,6 +233,8 @@ function migrateCandidate(db: Database.Interface["db"], candidate: Candidate, no const artifact = canonicalArtifact(candidate) if (!artifact) return yield* failure("canonical artifact descriptor is invalid") + const budgetFailure = yield* migrationBudget(tx as unknown as Database.Interface["db"], candidate) + if (budgetFailure) return yield* failure(budgetFailure) if (candidate.bodyBytes > MAX_MIGRATION_BODY_BYTES) return yield* failure("artifact body exceeds the migration byte budget") if (Hash.sha256(candidate.canonicalDataText) !== candidate.canonicalDataHash) @@ -344,21 +387,7 @@ function migrateCandidate(db: Database.Interface["db"], candidate: Candidate, no pathKeys.add(pathKey) const chunkCount = Math.max(1, Math.ceil(file.patch_bytes / FILE_CHUNK_BYTES)) const digest = createHash("sha256") - yield* tx - .insert(SessionDiffArtifactFileTable) - .values({ - artifact_id: candidate.artifactID, - file_index: file.file_index, - path, - path_key: pathKey, - additions: file.additions, - deletions: file.deletions, - status: file.status, - patch_hash: Hash.sha256("pending"), - patch_bytes: file.patch_bytes, - patch_chunk_count: chunkCount, - }) - .run() + const chunks = [] as Buffer[] for (const index of Array.from({ length: chunkCount }, (_, chunkIndex) => chunkIndex)) { const patchChunk = yield* tx.get<{ data: Buffer | null }>(sql` SELECT substr( @@ -376,27 +405,38 @@ function migrateCandidate(db: Database.Interface["db"], candidate: Candidate, no // JSON null patch is the canonical empty patch, represented by one empty chunk. const data = patchChunk.data ?? Buffer.alloc(0) digest.update(data) - yield* tx - .insert(SessionDiffArtifactFileChunkTable) - .values({ - artifact_id: candidate.artifactID, - file_index: file.file_index, - chunk_index: index, - data, - chunk_hash: Hash.sha256(data), - }) - .run() + chunks.push(data) } yield* tx - .update(SessionDiffArtifactFileTable) - .set({ patch_hash: digest.digest("hex") }) - .where( - and( - eq(SessionDiffArtifactFileTable.artifact_id, candidate.artifactID), - eq(SessionDiffArtifactFileTable.file_index, file.file_index), - ), - ) + .insert(SessionDiffArtifactFileTable) + .values({ + artifact_id: candidate.artifactID, + file_index: file.file_index, + path, + path_key: pathKey, + additions: file.additions, + deletions: file.deletions, + status: file.status, + patch_hash: digest.digest("hex"), + patch_bytes: file.patch_bytes, + patch_chunk_count: chunkCount, + }) .run() + yield* Effect.forEach( + chunks, + (data, index) => + tx + .insert(SessionDiffArtifactFileChunkTable) + .values({ + artifact_id: candidate.artifactID, + file_index: file.file_index, + chunk_index: index, + data, + chunk_hash: Hash.sha256(data), + }) + .run(), + { discard: true }, + ) } const updated = yield* tx @@ -460,7 +500,9 @@ function migrateCandidate(db: Database.Interface["db"], candidate: Candidate, no .run() return { state: "committed" as const } }), - { behavior: "immediate" }, + // WAL keeps the validation reads on one snapshot without reserving the writer. + // A concurrent commit makes the later write upgrade fail instead of committing stale authority. + { behavior: "deferred" }, ) .pipe( Effect.catchTag("SessionDiffArtifact.RewriteCasLost", (error) => @@ -501,6 +543,92 @@ function migrateCandidate(db: Database.Interface["db"], candidate: Candidate, no ) } +function migrationBudget(db: Database.Interface["db"], candidate: Candidate) { + return db + .get<{ + messages: number + parts: number + epochs: number + members: number + jsonBytes: number + maxJsonBytes: number + }>(sql` + WITH bounded_messages AS ( + SELECT length(CAST(data AS BLOB)) AS bytes + FROM message + WHERE session_id = ${candidate.sessionID} + LIMIT ${MAX_MIGRATION_MESSAGES + 1} + ), bounded_parts AS ( + SELECT length(CAST(data AS BLOB)) AS bytes + FROM part + WHERE session_id = ${candidate.sessionID} + LIMIT ${MAX_MIGRATION_PARTS + 1} + ), bounded_epochs AS ( + SELECT 1 FROM session_prompt_epoch + WHERE session_id = ${candidate.sessionID} + LIMIT ${MAX_MIGRATION_EPOCHS + 1} + ), bounded_members AS ( + SELECT 1 FROM session_prompt_epoch_message + WHERE session_id = ${candidate.sessionID} + LIMIT ${MAX_MIGRATION_EPOCH_MEMBERS + 1} + ) + SELECT + (SELECT count(*) FROM bounded_messages) AS messages, + (SELECT count(*) FROM bounded_parts) AS parts, + (SELECT count(*) FROM bounded_epochs) AS epochs, + (SELECT count(*) FROM bounded_members) AS members, + COALESCE((SELECT sum(bytes) FROM bounded_messages), 0) + + COALESCE((SELECT sum(bytes) FROM bounded_parts), 0) AS jsonBytes, + max( + COALESCE((SELECT max(bytes) FROM bounded_messages), 0), + COALESCE((SELECT max(bytes) FROM bounded_parts), 0) + ) AS maxJsonBytes + `) + .pipe( + Effect.orDie, + Effect.map((budget) => { + if (!budget) return "candidate Session is missing" + if (budget.messages > MAX_MIGRATION_MESSAGES) return `Session message count exceeds ${MAX_MIGRATION_MESSAGES}` + if (budget.parts > MAX_MIGRATION_PARTS) return `Session part count exceeds ${MAX_MIGRATION_PARTS}` + if (budget.epochs > MAX_MIGRATION_EPOCHS) return `PromptEpoch count exceeds ${MAX_MIGRATION_EPOCHS}` + if (budget.members > MAX_MIGRATION_EPOCH_MEMBERS) + return `PromptEpoch membership count exceeds ${MAX_MIGRATION_EPOCH_MEMBERS}` + if (budget.maxJsonBytes > MAX_MIGRATION_ROW_JSON_BYTES) + return `Session history row exceeds ${MAX_MIGRATION_ROW_JSON_BYTES} bytes` + if (budget.jsonBytes > MAX_MIGRATION_SESSION_JSON_BYTES) + return `Session history JSON exceeds ${MAX_MIGRATION_SESSION_JSON_BYTES} bytes` + }), + ) +} + +function recordValidationFailure(db: Database.Interface["db"], candidate: Candidate, now: number, reason: string) { + return db + .transaction( + (tx) => + tx + .insert(SessionDiffMigrationReceiptTable) + .values({ + message_id: candidate.messageID, + session_id: candidate.sessionID, + artifact_id: candidate.artifactID, + source_event_id: candidate.sourceEventID, + expected_message_data_hash: Hash.sha256("budget-rejected"), + expected_session_summary_hash: Hash.sha256("unknown"), + canonicalizer_version: candidate.codecVersion, + canonicalization_version: HistoryAuthority.CANONICALIZATION_VERSION, + epoch_hashes: [], + state: "migration_validation_failed" as const, + failure_reason: reason, + created_at: now, + updated_at: now, + }) + .onConflictDoNothing() + .run(), + { behavior: "immediate" }, + ) + .pipe(Effect.orDie, Effect.as({ state: "migration_validation_failed" as const })) +} + function canonicalArtifact(candidate: Candidate): CanonicalArtifact | undefined { const info = record(candidate.canonicalData.info) ? candidate.canonicalData.info : undefined const summary = info && record(info.summary) ? info.summary : undefined @@ -516,6 +644,20 @@ function canonicalArtifact(candidate: Candidate): CanonicalArtifact | undefined descriptor.fileCount < 0 ) return + if (descriptor.codec === "legacy-message-diff.v1") + return { + ...(descriptor as Omit), + previewFileCount: descriptor.fileCount, + previewTruncated: false, + } + if ( + typeof descriptor.previewFileCount !== "number" || + !Number.isSafeInteger(descriptor.previewFileCount) || + descriptor.previewFileCount < 0 || + descriptor.previewFileCount > descriptor.fileCount || + descriptor.previewTruncated !== (descriptor.previewFileCount < descriptor.fileCount) + ) + return return descriptor as CanonicalArtifact } @@ -541,6 +683,25 @@ function validateEpochHashes( .orderBy(asc(MessageTable.time_created), asc(MessageTable.id)) .all() ).map((row) => row.id) + const messages = yield* MessageV2.messagesInTransaction(tx, sessionID, physicalIDs) + if (!messages) return { failure: "physical history is incomplete" } + const messagesByID = new Map(messages.map((message) => [message.info.id, message])) + const before = HistoryAuthority.hash(messages) + const after = HistoryAuthority.hash( + messages.map((message) => + message.info.id === messageID + ? { ...message, info: { ...afterInfo, id: messageID, sessionID } as SessionV1.User } + : message, + ), + ) + const isolatedBefore = HistoryAuthority.hash([ + { info: { ...beforeInfo, id: messageID, sessionID } as SessionV1.User, parts: [] }, + ]) + const isolatedAfter = HistoryAuthority.hash([ + { info: { ...afterInfo, id: messageID, sessionID } as SessionV1.User, parts: [] }, + ]) + if (before !== after || isolatedBefore !== isolatedAfter) + return { failure: "history hash changed during user summary rewrite" } const values = [] as Array<{ epoch: number; before: string; after: string }> for (const epoch of epochs) { if ( @@ -561,31 +722,17 @@ function validateEpochHashes( .orderBy(asc(SessionPromptEpochMessageTable.ordinal)) .all() ).map((row) => row.message_id) - const messages = yield* MessageV2.messagesInTransaction(tx, sessionID, physicalIDs) - if (!messages) return { failure: `PromptEpoch ${epoch.epoch} physical history is incomplete` } const baseIDs = memberIDs.length > 0 ? memberIDs : physicalIDs.slice(0, epoch.base_message_count ?? 0) - const base = yield* MessageV2.messagesInTransaction(tx, sessionID, baseIDs) - if (!base) return { failure: `PromptEpoch ${epoch.epoch} references a missing base message` } + const base = baseIDs.flatMap((id) => { + const message = messagesByID.get(id) + return message ? [message] : [] + }) + if (base.length !== baseIDs.length) + return { failure: `PromptEpoch ${epoch.epoch} references a missing base message` } if (epoch.base_message_count !== base.length || !epoch.effective_history_hash) return { failure: `PromptEpoch ${epoch.epoch} immutable base authority is incomplete` } if (HistoryAuthority.hash(base) !== epoch.effective_history_hash) return { failure: `PromptEpoch ${epoch.epoch} immutable base hash is already inconsistent` } - const before = HistoryAuthority.hash(messages) - const after = HistoryAuthority.hash( - messages.map((message) => - message.info.id === messageID - ? { ...message, info: { ...afterInfo, id: messageID, sessionID } as SessionV1.User } - : message, - ), - ) - const isolatedBefore = HistoryAuthority.hash([ - { info: { ...beforeInfo, id: messageID, sessionID } as SessionV1.User, parts: [] }, - ]) - const isolatedAfter = HistoryAuthority.hash([ - { info: { ...afterInfo, id: messageID, sessionID } as SessionV1.User, parts: [] }, - ]) - if (before !== after || isolatedBefore !== isolatedAfter) - return { failure: `PromptEpoch ${epoch.epoch} history hash changed during user summary rewrite` } values.push({ epoch: epoch.epoch, before, after }) } return { values } diff --git a/packages/deepagent-code/test/control-plane/workspace.test.ts b/packages/deepagent-code/test/control-plane/workspace.test.ts index aef5a5e82..2509d1b76 100644 --- a/packages/deepagent-code/test/control-plane/workspace.test.ts +++ b/packages/deepagent-code/test/control-plane/workspace.test.ts @@ -1923,8 +1923,8 @@ describe("workspace sync state", () => { eventID, aggregateID: session.id, seq: nextSeq, - originalDataHash: Hash.sha256(JSON.stringify(prepared.data)), - canonicalDataHash: Hash.sha256(JSON.stringify(prepared.data)), + originalDataHash: FilePartArtifact.dataHash(prepared.data), + canonicalDataHash: FilePartArtifact.dataHash(prepared.data), canonicalData: prepared.data, descriptor, chunkHashes: prepared.artifacts[0]!.chunks.map((chunk) => Hash.sha256(chunk)), diff --git a/packages/deepagent-code/test/session/diff-artifact.test.ts b/packages/deepagent-code/test/session/diff-artifact.test.ts index c35cc627b..81b8fc90f 100644 --- a/packages/deepagent-code/test/session/diff-artifact.test.ts +++ b/packages/deepagent-code/test/session/diff-artifact.test.ts @@ -283,6 +283,12 @@ describe("Session legacy diff physical migration", () => { expect(receipt?.expected_session_summary_hash).not.toBe(receipt?.committed_session_summary_hash) expect(receipt?.committed_session_summary_hash).toBe(Hash.sha256("null")) expect(receipt?.epoch_hashes).toEqual([{ epoch: 0, before, after: before }]) + const receiptDelete = yield* db + .delete(SessionDiffMigrationReceiptTable) + .where(eq(SessionDiffMigrationReceiptTable.message_id, input.messageID)) + .run() + .pipe(Effect.exit) + expect(String(receiptDelete)).toContain("session_diff_migration_receipt_delete_immutable") const manifestResponse = yield* request( `${pathFor(SessionPaths.diffArtifactManifest, input.session.id)}?${new URLSearchParams({ messageID: input.messageID, @@ -317,6 +323,31 @@ describe("Session legacy diff physical migration", () => { })}`, ) expect(unauthorized.status).toBe(404) + + const fileDelete = yield* db + .delete(SessionDiffArtifactFileChunkTable) + .where(eq(SessionDiffArtifactFileChunkTable.artifact_id, input.artifact.artifact_id)) + .run() + .pipe(Effect.exit) + expect(String(fileDelete)).toContain("session_diff_artifact_file_chunk_delete_immutable") + const sessions = yield* Session.Service + yield* sessions.remove(input.session.id) + expect( + yield* db + .select() + .from(SessionDiffMigrationReceiptTable) + .where(eq(SessionDiffMigrationReceiptTable.session_id, input.session.id)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) + expect( + yield* db + .select() + .from(SessionDiffArtifactFileChunkTable) + .where(eq(SessionDiffArtifactFileChunkTable.artifact_id, input.artifact.artifact_id)) + .all() + .pipe(Effect.orDie), + ).toEqual([]) }), { git: true }, 30_000, @@ -365,6 +396,56 @@ describe("Session legacy diff physical migration", () => { 30_000, ) + it.instance( + "rejects an oversized Session history before rewriting durable message authority", + () => + Effect.gen(function* () { + const input = yield* seed() + const { db } = yield* Database.Service + yield* db.run(sql` + WITH RECURSIVE counter(value) AS ( + SELECT 1 + UNION ALL + SELECT value + 1 FROM counter WHERE value < 1000 + ) + INSERT INTO message(id, session_id, time_created, time_updated, data) + SELECT 'msg_budget_' || printf('%04d', value), ${input.session.id}, value, value, + json_object('role', 'user', 'time', json_object('created', value)) + FROM counter + `) + + expect(yield* SessionDiffArtifact.migrate({ sessionID: input.session.id, now: 225 })).toEqual({ + processed: 1, + committed: 0, + failed: 1, + }) + const receipt = yield* db + .select() + .from(SessionDiffMigrationReceiptTable) + .where(eq(SessionDiffMigrationReceiptTable.message_id, input.messageID)) + .get() + .pipe(Effect.orDie) + expect(receipt).toMatchObject({ + state: "migration_validation_failed", + failure_reason: "Session message count exceeds 1000", + }) + const message = yield* db + .select({ data: MessageTable.data }) + .from(MessageTable) + .where(eq(MessageTable.id, input.messageID)) + .get() + .pipe(Effect.orDie) + expect( + message?.data.role === "user" && message.data.summary && typeof message.data.summary === "object" + ? message.data.summary.diffs[0]?.patch + : undefined, + ).toBe(input.patch) + expect(yield* db.select().from(SessionDiffArtifactFileChunkTable).all().pipe(Effect.orDie)).toEqual([]) + }), + { git: true }, + 30_000, + ) + it.instance( "preserves a legacy null patch as one verified empty chunk", () => @@ -448,7 +529,7 @@ describe("Session legacy diff physical migration", () => { ) it.instance( - "fails closed when a committed artifact chunk is corrupted", + "rejects mutation of a committed artifact chunk before migration can observe corrupted authority", () => Effect.gen(function* () { const input = yield* seed() @@ -460,7 +541,7 @@ describe("Session legacy diff physical migration", () => { .orderBy(asc(EventArtifactChunkTable.chunk_index)) .get() .pipe(Effect.orDie) - yield* db + const corrupted = yield* db .update(EventArtifactChunkTable) .set({ data: Buffer.from("corrupt") }) .where( @@ -470,8 +551,13 @@ describe("Session legacy diff physical migration", () => { ), ) .run() - .pipe(Effect.orDie) - expect((yield* SessionDiffArtifact.migrate({ sessionID: input.session.id })).failed).toBe(1) + .pipe(Effect.exit) + expect(String(corrupted)).toContain("event_artifact_chunk_update_immutable") + expect(yield* SessionDiffArtifact.migrate({ sessionID: input.session.id })).toEqual({ + processed: 1, + committed: 1, + failed: 0, + }) const row = yield* db .select({ data: MessageTable.data }) .from(MessageTable) @@ -480,9 +566,11 @@ describe("Session legacy diff physical migration", () => { .pipe(Effect.orDie) expect( row?.data.role === "user" && row.data.summary && typeof row.data.summary === "object" - ? row.data.summary.diffs[0]?.patch + ? row.data.summary.diffArtifact?.id : undefined, - ).toBe(input.patch) + ).toBe( + input.artifact.artifact_id, + ) }), { git: true }, 30_000, diff --git a/packages/sdk/js/src/gen/types.gen.ts b/packages/sdk/js/src/gen/types.gen.ts index 1dfc35757..9de2292ac 100644 --- a/packages/sdk/js/src/gen/types.gen.ts +++ b/packages/sdk/js/src/gen/types.gen.ts @@ -277,12 +277,21 @@ export type OutputFormatJsonSchema = { export type OutputFormat = OutputFormatText | OutputFormatJsonSchema -export type SnapshotDiffArtifactDescriptor = { - id: string - hash: string - codec: "legacy-message-diff.v1" | "legacy-message-diff.v2" - fileCount: number -} +export type SnapshotDiffArtifactDescriptor = + | { + id: string + hash: string + codec: "legacy-message-diff.v1" + fileCount: number + } + | { + id: string + hash: string + codec: "legacy-message-diff.v2" + fileCount: number + previewFileCount: number + previewTruncated: boolean + } export type UserMessage = { id: string @@ -3621,6 +3630,12 @@ export type NotFoundError = { } } +export type ServiceUnavailableError = { + _tag: "ServiceUnavailableError" + message: string + service?: string +} + export type Session3 = { id: string slug: string @@ -4125,12 +4140,6 @@ export type Session8 = { preview?: string } -export type ServiceUnavailableError = { - _tag: "ServiceUnavailableError" - message: string - service?: string -} - export type Session9 = { id: string slug: string @@ -13395,6 +13404,10 @@ export type SessionGetErrors = { * NotFoundError */ 404: NotFoundError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError } export type SessionGetError = SessionGetErrors[keyof SessionGetErrors] @@ -13472,6 +13485,10 @@ export type SessionChildrenErrors = { * NotFoundError */ 404: NotFoundError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError } export type SessionChildrenError = SessionChildrenErrors[keyof SessionChildrenErrors] diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index 1dfc35757..9de2292ac 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -277,12 +277,21 @@ export type OutputFormatJsonSchema = { export type OutputFormat = OutputFormatText | OutputFormatJsonSchema -export type SnapshotDiffArtifactDescriptor = { - id: string - hash: string - codec: "legacy-message-diff.v1" | "legacy-message-diff.v2" - fileCount: number -} +export type SnapshotDiffArtifactDescriptor = + | { + id: string + hash: string + codec: "legacy-message-diff.v1" + fileCount: number + } + | { + id: string + hash: string + codec: "legacy-message-diff.v2" + fileCount: number + previewFileCount: number + previewTruncated: boolean + } export type UserMessage = { id: string @@ -3621,6 +3630,12 @@ export type NotFoundError = { } } +export type ServiceUnavailableError = { + _tag: "ServiceUnavailableError" + message: string + service?: string +} + export type Session3 = { id: string slug: string @@ -4125,12 +4140,6 @@ export type Session8 = { preview?: string } -export type ServiceUnavailableError = { - _tag: "ServiceUnavailableError" - message: string - service?: string -} - export type Session9 = { id: string slug: string @@ -13395,6 +13404,10 @@ export type SessionGetErrors = { * NotFoundError */ 404: NotFoundError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError } export type SessionGetError = SessionGetErrors[keyof SessionGetErrors] @@ -13472,6 +13485,10 @@ export type SessionChildrenErrors = { * NotFoundError */ 404: NotFoundError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError } export type SessionChildrenError = SessionChildrenErrors[keyof SessionChildrenErrors] From 8c0ae19de4211124a9584513f407db14b41228b6 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 18:51:56 +0800 Subject: [PATCH 27/44] fix: compact file artifact source events --- .../deepagent-code/script/bug-407-010-release-gate.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/deepagent-code/script/bug-407-010-release-gate.ts b/packages/deepagent-code/script/bug-407-010-release-gate.ts index cc9366b03..377aae2bd 100644 --- a/packages/deepagent-code/script/bug-407-010-release-gate.ts +++ b/packages/deepagent-code/script/bug-407-010-release-gate.ts @@ -130,6 +130,11 @@ const program = Effect.gen(function* () { .where(eq(EventArtifactTable.kind, "legacy_message_diff")) .all() .pipe(Effect.orDie) + const fileArtifactSessions = yield* db + .selectDistinct({ id: FilePartArtifactBindingTable.aggregate_id }) + .from(FilePartArtifactBindingTable) + .all() + .pipe(Effect.orDie) const diffMigration = yield* stage("legacy-session-diffs", Effect.gen(function* () { let processed = 0 let committed = 0 @@ -153,6 +158,7 @@ const program = Effect.gen(function* () { const checkpointSessions = [...new Set([ ...sessions, ...artifactSessions.map((row) => SessionID.make(row.id)), + ...fileArtifactSessions.map((row) => SessionID.make(row.id)), ])] const validateSnapshotDiffAuthority = Effect.fn("Bug407010ReleaseGate.validateSnapshotDiffAuthority")( @@ -280,6 +286,7 @@ const program = Effect.gen(function* () { events: number eventBytes: number maximumEventBytes: number + oversizedEvents: number syncIndex: number snapshots: number eventArtifacts: number @@ -291,6 +298,7 @@ const program = Effect.gen(function* () { (SELECT count(*) FROM event) AS events, (SELECT coalesce(sum(length(data)), 0) FROM event) AS eventBytes, (SELECT coalesce(max(length(data)), 0) FROM event) AS maximumEventBytes, + (SELECT count(*) FROM event WHERE length(CAST(data AS BLOB)) > ${EventV2.MAX_ENCODED_PAYLOAD_BYTES}) AS oversizedEvents, (SELECT count(*) FROM event_sync_index) AS syncIndex, (SELECT count(*) FROM event_snapshot) AS snapshots, (SELECT count(*) FROM event_artifact) AS eventArtifacts, @@ -329,6 +337,7 @@ const program = Effect.gen(function* () { backfill?.state !== "complete" || quickCheck.some((row) => row.quick_check !== "ok") || foreignKeys.length > 0 || + result.oversizedEvents > 0 || result.failedDiffMigrationReceipts > 0 ) return yield* Effect.die(new Error(`Release gate failed: ${JSON.stringify(result)}`)) From 05a12dcdd1a3c94cfdbafed27669c0d2c5bad968 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 19:05:52 +0800 Subject: [PATCH 28/44] fix(core): bound event compaction memory --- packages/core/src/event.ts | 157 +++++++++++++++++----- packages/core/test/event-snapshot.test.ts | 23 ++++ 2 files changed, 149 insertions(+), 31 deletions(-) diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index 10a5d114b..b089afba7 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -287,6 +287,23 @@ function isLegacyFileArtifactRetry( return FilePartArtifact.isLegacySyntheticCanonical(binding.canonicalData, encoded) } +function isCompactedLegacyFileArtifactRetry( + type: string, + original: Record, + encoded: Record, + binding: { readonly originalDataHash: string; readonly canonicalData: Record }, +) { + if (!FilePartArtifact.matchesDataHash(binding.originalDataHash, original)) return false + const prepared = FilePartArtifact.prepare( + type, + original, + Buffer.byteLength(JSON.stringify(original)), + MAX_ENCODED_PAYLOAD_BYTES, + ) + if (prepared.artifacts.length !== 1 || !isDeepStrictEqual(prepared.data, encoded)) return false + return FilePartArtifact.isLegacySyntheticCanonical(binding.canonicalData, encoded) +} + // Synchronized events cross a JSON boundary, so their data schemas must encode and decode without services. const syncCodec = (definition: Definition) => definition.data as Schema.Codec @@ -584,7 +601,16 @@ export const layerWith = (layerOptions?: LayerOptions) => .where(eq(EventArtifactTable.event_id, stored.id)) .get() .pipe(Effect.orDie) - : undefined + : yield* db + .select({ + originalDataHash: EventArtifactTable.original_data_hash, + canonicalData: EventArtifactTable.canonical_data, + codecVersion: EventArtifactTable.codec_version, + }) + .from(EventArtifactTable) + .where(eq(EventArtifactTable.event_id, event.id)) + .get() + .pipe(Effect.orDie) const fileArtifact = stored ? yield* db .select({ @@ -595,7 +621,15 @@ export const layerWith = (layerOptions?: LayerOptions) => .where(eq(FilePartArtifactBindingTable.event_id, stored.id)) .get() .pipe(Effect.orDie) - : undefined + : yield* db + .select({ + originalDataHash: FilePartArtifactBindingTable.original_data_hash, + canonicalData: FilePartArtifactBindingTable.canonical_data, + }) + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, event.id)) + .get() + .pipe(Effect.orDie) const dedupe = !stored ? yield* db .select() @@ -630,7 +664,19 @@ export const layerWith = (layerOptions?: LayerOptions) => : hashJson(encoded))))) || (dedupe?.event_id === event.id && dedupe.type === versionedType(definition.type, sync.version) && - dedupe.data_hash === hashJson(encoded))) + (dedupe.data_hash === hashJson(encoded) || + (artifact && + artifact.originalDataHash === + (artifact.codecVersion >= 2 + ? Hash.sha256(JSON.stringify(encoded)) + : hashJson(encoded))) || + (fileArtifact && + isCompactedLegacyFileArtifactRetry( + definition.type, + original, + encoded, + fileArtifact, + ))))) ) { if (input.ownerID && row?.ownerID == null) { for (const guard of commitGuards) { @@ -1680,43 +1726,92 @@ export const layerWith = (layerOptions?: LayerOptions) => receipt.codec !== active.codec || receipt.schema_version !== active.schema_version)) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Compaction receipt conflicts with active snapshot ${active.snapshot_id}` })) const cursor = receipt?.cursor_seq ?? -1 - const candidates = yield* db - .select({ - row: EventTable, - canonicalData: sql | null>`COALESCE(${FilePartArtifactBindingTable.canonical_data}, ${EventArtifactTable.canonical_data})`, - }) - .from(EventTable) - .leftJoin(EventArtifactTable, eq(EventArtifactTable.event_id, EventTable.id)) - .leftJoin(FilePartArtifactBindingTable, eq(FilePartArtifactBindingTable.event_id, EventTable.id)) - .where(and(eq(EventTable.aggregate_id, input.aggregateID), gt(EventTable.seq, cursor), lte(EventTable.seq, input.throughSeq))) - .orderBy(asc(EventTable.seq)) - .limit(Math.min(Math.max(input.limit ?? 100, 1), 100) + 1) - .all() - .pipe(Effect.orDie) - const rows = candidates.slice(0, Math.min(Math.max(input.limit ?? 100, 1), 100)) - const compactable = rows.filter((item) => codec.rebuildEventTypes.has(item.row.type)) + const limit = Math.min(Math.max(input.limit ?? 100, 1), 100) + const candidates = yield* db.all<{ + id: ID + aggregateID: string + seq: number + type: string + effectiveBytes: number + cumulativeBytes: number + canonicalData: string | null + }>(sql` + WITH candidate AS ( + SELECT + event.id, + event.aggregate_id AS aggregateID, + event.seq, + event.type, + COALESCE(file_binding.canonical_data, artifact.canonical_data) AS canonicalData, + length(CAST(COALESCE(file_binding.canonical_data, artifact.canonical_data, event.data) AS BLOB)) AS effectiveBytes + FROM event + LEFT JOIN event_artifact artifact ON artifact.event_id = event.id + LEFT JOIN file_part_artifact_binding file_binding ON file_binding.event_id = event.id + WHERE event.aggregate_id = ${input.aggregateID} + AND event.seq > ${cursor} + AND event.seq <= ${input.throughSeq} + ORDER BY event.seq ASC + LIMIT ${limit + 1} + ) + SELECT *, sum(effectiveBytes) OVER ( + ORDER BY seq ASC ROWS UNBOUNDED PRECEDING + ) AS cumulativeBytes + FROM candidate + ORDER BY seq ASC + `).pipe(Effect.orDie) + const rows = candidates + .slice(0, limit) + .filter((item, index) => index === 0 || item.cumulativeBytes <= AGGREGATE_READ_BATCH_BYTES) + if (rows[0] && rows[0].effectiveBytes > AGGREGATE_READ_BATCH_BYTES) + return yield* Effect.die(new EncodedPayloadTooLargeError({ + type: rows[0].type, + encodedBytes: rows[0].effectiveBytes, + limitBytes: AGGREGATE_READ_BATCH_BYTES, + message: `Compaction source ${rows[0].id} has no bounded canonical projection`, + })) + const compactable = rows.filter((item) => codec.rebuildEventTypes.has(item.type)) + const rawIDs = compactable.filter((item) => item.canonicalData === null).map((item) => item.id) + const rawRows = rawIDs.length > 0 + ? yield* db + .select({ id: EventTable.id, data: EventTable.data }) + .from(EventTable) + .where(inArray(EventTable.id, rawIDs)) + .all() + .pipe(Effect.orDie) + : [] + const rawByID = new Map(rawRows.map((row) => [row.id, row.data])) yield* Effect.forEach(compactable, (item) => Effect.gen(function* () { - const dataHash = hashJson(item.canonicalData ?? item.row.data) + const dataHash = item.canonicalData + ? hashJson(Schema.decodeUnknownSync(Schema.UnknownFromJsonString)(item.canonicalData)) + : rawByID.has(item.id) + ? hashJson(rawByID.get(item.id)) + : yield* Effect.die(new InvalidSyncEventError({ + type: "snapshot", + message: `Compaction source ${item.id} disappeared`, + })) const existing = yield* db.select().from(EventDedupeTable).where(or( - and(eq(EventDedupeTable.aggregate_id, item.row.aggregate_id), eq(EventDedupeTable.seq, item.row.seq)), - eq(EventDedupeTable.event_id, item.row.id), + and(eq(EventDedupeTable.aggregate_id, item.aggregateID), eq(EventDedupeTable.seq, item.seq)), + eq(EventDedupeTable.event_id, item.id), )).get().pipe(Effect.orDie) - if (existing && (existing.aggregate_id !== item.row.aggregate_id || existing.seq !== item.row.seq || - existing.event_id !== item.row.id || existing.type !== item.row.type || existing.data_hash !== dataHash)) - return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Compaction dedupe conflicts at ${item.row.aggregate_id}:${item.row.seq}` })) - if (!existing) yield* db.insert(EventDedupeTable).values({ - aggregate_id: item.row.aggregate_id, seq: item.row.seq, event_id: item.row.id, type: item.row.type, - data_hash: dataHash, source_data: item.row.data, compacted_at: input.now ?? Date.now(), - }).run().pipe(Effect.orDie) + if (existing && (existing.aggregate_id !== item.aggregateID || existing.seq !== item.seq || + existing.event_id !== item.id || existing.type !== item.type || existing.data_hash !== dataHash)) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Compaction dedupe conflicts at ${item.aggregateID}:${item.seq}` })) + if (!existing) yield* db.run(sql` + INSERT INTO event_dedupe ( + aggregate_id, seq, event_id, type, data_hash, source_data, compacted_at + ) + SELECT aggregate_id, seq, id, type, ${dataHash}, data, ${input.now ?? Date.now()} + FROM event WHERE id = ${item.id} + `).pipe(Effect.orDie) }), { discard: true }) if (compactable.length > 0) yield* db.delete(EventTable) - .where(inArray(EventTable.id, compactable.map((item) => item.row.id))).run().pipe(Effect.orDie) + .where(inArray(EventTable.id, compactable.map((item) => item.id))).run().pipe(Effect.orDie) if (compactable.length > 0) yield* db.update(EventDedupeTable) .set({ source_data: null }) - .where(inArray(EventDedupeTable.event_id, compactable.map((item) => item.row.id))) + .where(inArray(EventDedupeTable.event_id, compactable.map((item) => item.id))) .run() .pipe(Effect.orDie) - const nextCursor = rows.at(-1)?.row.seq ?? cursor + const nextCursor = rows.at(-1)?.seq ?? cursor const complete = rows.length === 0 || (nextCursor >= input.throughSeq && candidates.length <= rows.length) yield* db.insert(EventCompactionReceiptTable).values({ aggregate_id: input.aggregateID, snapshot_id: active.snapshot_id, through_seq: input.throughSeq, codec: active.codec, diff --git a/packages/core/test/event-snapshot.test.ts b/packages/core/test/event-snapshot.test.ts index 2c2528d70..665655d1f 100644 --- a/packages/core/test/event-snapshot.test.ts +++ b/packages/core/test/event-snapshot.test.ts @@ -281,6 +281,7 @@ describe("EventV2 canonical projection snapshots", () => { messageID, label: "latest", }) + latest.original_data_hash = Hash.sha256(JSON.stringify(sourceEvents[1]!.data)) yield* db.insert(EventArtifactTable).values([old, latest]).run().pipe(Effect.orDie) yield* db .insert(EventArtifactChunkTable) @@ -365,6 +366,28 @@ describe("EventV2 canonical projection snapshots", () => { ).toBe(latest.artifact_id) expect(JSON.stringify(imported)).not.toContain('"patch"') }).pipe(Effect.provide(Layer.fresh(Layer.mergeAll(targetDatabase, targetEvents, targetProjector)))) + + let compacted = { deleted: 0, complete: false } + while (!compacted.complete) + compacted = yield* events.compact({ aggregateID: sessionID, throughSeq: EventV2.Cursor.make(1), limit: 1 }) + yield* events.replay({ + id: sourceEvents[1]!.id, + aggregateID: sessionID, + seq: sourceEvents[1]!.seq, + type: sourceEvents[1]!.type, + data: sourceEvents[1]!.data, + }) + const divergent = yield* events.replay({ + id: sourceEvents[1]!.id, + aggregateID: sessionID, + seq: sourceEvents[1]!.seq, + type: sourceEvents[1]!.type, + data: { + ...sourceEvents[1]!.data, + info: { ...(sourceEvents[1]!.data.info as Record), system: "divergent" }, + }, + }).pipe(Effect.catchDefect(Effect.succeed)) + expect(divergent).toBeInstanceOf(EventV2.InvalidSyncEventError) }), ) }) From d2990de69ba12fe7cb00294b8590197a3688eb0e Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 22:03:28 +0800 Subject: [PATCH 29/44] fix(core): close bounded event maintenance gates --- packages/core/src/database/migration.gen.ts | 4 + ...20260813100000_event_snapshot_authority.ts | 4 +- .../20260813130000_file_part_artifact.ts | 21 + .../20260813131000_event_snapshot_chunks.ts | 29 +- ...20260813140000_event_sidecar_compaction.ts | 512 ++++++++++++++++++ ...813141000_bug_407_010_sidecar_lifecycle.ts | 10 + ...60813142000_bug_407_010_sidecar_indexes.ts | 10 + ...813143000_bug_407_010_aggregate_indexes.ts | 10 + .../core/src/deepagent/deepagent-event-bus.ts | 15 +- .../core/src/deepagent/retention-sweeper.ts | 197 ++++--- packages/core/src/event.ts | 433 ++++++++++++--- packages/core/src/event/sql.ts | 11 +- packages/core/src/file-part-artifact.sql.ts | 23 +- packages/core/src/file-part-artifact.ts | 92 ++++ packages/core/src/session/projector.ts | 11 +- packages/core/test/database-migration.test.ts | 350 ++++++++++++ packages/core/test/event-snapshot.test.ts | 147 ++++- packages/core/test/file-part-artifact.test.ts | 340 +++++++++++- packages/core/test/retention-sweeper.test.ts | 234 +++++++- .../script/bug-407-010-release-gate.ts | 129 ++++- .../src/control-plane/workspace.ts | 82 ++- .../routes/instance/httpapi/handlers/sync.ts | 24 +- .../src/session/v4-event-runtime.ts | 42 +- .../test/server/httpapi-sync.test.ts | 69 ++- 24 files changed, 2579 insertions(+), 220 deletions(-) create mode 100644 packages/core/src/database/migration/20260813140000_event_sidecar_compaction.ts create mode 100644 packages/core/src/database/migration/20260813141000_bug_407_010_sidecar_lifecycle.ts create mode 100644 packages/core/src/database/migration/20260813142000_bug_407_010_sidecar_indexes.ts create mode 100644 packages/core/src/database/migration/20260813143000_bug_407_010_aggregate_indexes.ts diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index 4e6cc97d3..7d9533195 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -94,5 +94,9 @@ export const migrations = ( import("./migration/20260813133000_session_transfer_authority"), import("./migration/20260813134000_database_capability"), import("./migration/20260813135000_event_immutable"), + import("./migration/20260813140000_event_sidecar_compaction"), + import("./migration/20260813141000_bug_407_010_sidecar_lifecycle"), + import("./migration/20260813142000_bug_407_010_sidecar_indexes"), + import("./migration/20260813143000_bug_407_010_aggregate_indexes"), ]) ).map((module) => module.default) satisfies DatabaseMigration.Migration[] diff --git a/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts b/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts index 31735f9d7..7496b3462 100644 --- a/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts +++ b/packages/core/src/database/migration/20260813100000_event_snapshot_authority.ts @@ -79,13 +79,13 @@ export default { snapshot_hash TEXT NOT NULL CHECK (length(snapshot_hash) = 64), body TEXT NOT NULL CHECK (json_valid(body)), owner_id TEXT, - created_at INTEGER NOT NULL, - UNIQUE (aggregate_id, through_seq) + created_at INTEGER NOT NULL ) `) yield* tx.run( "CREATE INDEX event_snapshot_aggregate_created_idx ON event_snapshot(aggregate_id, created_at)", ) + yield* tx.run("CREATE INDEX event_snapshot_aggregate_seq_idx ON event_snapshot(aggregate_id, through_seq)") yield* tx.run(` CREATE TABLE event_dedupe ( aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, diff --git a/packages/core/src/database/migration/20260813130000_file_part_artifact.ts b/packages/core/src/database/migration/20260813130000_file_part_artifact.ts index 8f71c0425..89900e8b4 100644 --- a/packages/core/src/database/migration/20260813130000_file_part_artifact.ts +++ b/packages/core/src/database/migration/20260813130000_file_part_artifact.ts @@ -44,6 +44,10 @@ export default { CREATE INDEX file_part_artifact_binding_part_idx ON file_part_artifact_binding(aggregate_id, part_id, seq) `) + yield* tx.run(` + CREATE INDEX file_part_artifact_binding_artifact_idx + ON file_part_artifact_binding(artifact_id) + `) yield* tx.run(` CREATE TABLE file_part_artifact_import ( event_id TEXT NOT NULL PRIMARY KEY, @@ -57,6 +61,23 @@ export default { UNIQUE (aggregate_id, seq) ) `) + yield* tx.run(` + CREATE INDEX file_part_artifact_import_artifact_idx + ON file_part_artifact_import(artifact_id) + `) + yield* tx.run(` + CREATE TABLE file_part_artifact_discard ( + event_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL, + seq INTEGER NOT NULL CHECK (seq >= 0), + artifact_id TEXT NOT NULL REFERENCES file_part_artifact(artifact_id) ON DELETE CASCADE, + original_data_hash TEXT NOT NULL CHECK (length(original_data_hash) = 64), + canonical_data_hash TEXT NOT NULL CHECK (length(canonical_data_hash) = 64), + canonical_data TEXT NOT NULL CHECK (json_valid(canonical_data)), + created_at INTEGER NOT NULL, + UNIQUE (aggregate_id, seq) + ) + `) yield* tx.run(` CREATE TRIGGER file_part_artifact_binding_source_guard BEFORE INSERT ON file_part_artifact_binding diff --git a/packages/core/src/database/migration/20260813131000_event_snapshot_chunks.ts b/packages/core/src/database/migration/20260813131000_event_snapshot_chunks.ts index c96b3d955..3b73192b9 100644 --- a/packages/core/src/database/migration/20260813131000_event_snapshot_chunks.ts +++ b/packages/core/src/database/migration/20260813131000_event_snapshot_chunks.ts @@ -8,6 +8,7 @@ export default { yield* tx.run(` CREATE TABLE event_snapshot_row ( snapshot_id TEXT NOT NULL, + aggregate_id TEXT NOT NULL, row_index INTEGER NOT NULL CHECK (row_index >= 0), table_name TEXT NOT NULL CHECK (length(table_name) > 0), row_key TEXT NOT NULL CHECK (length(row_key) > 0), @@ -19,6 +20,7 @@ export default { UNIQUE (snapshot_id, table_name, row_key) ) `) + yield* tx.run("CREATE INDEX event_snapshot_row_aggregate_idx ON event_snapshot_row(aggregate_id)") yield* tx.run(` CREATE TABLE event_snapshot_chunk ( row_hash TEXT NOT NULL CHECK (length(row_hash) = 64), @@ -31,7 +33,7 @@ export default { yield* tx.run(` CREATE TABLE event_snapshot_attempt ( snapshot_id TEXT NOT NULL PRIMARY KEY, - aggregate_id TEXT NOT NULL, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, through_seq INTEGER NOT NULL CHECK (through_seq >= 0), expected_latest INTEGER NOT NULL CHECK (expected_latest >= 0), owner_id TEXT, @@ -48,6 +50,7 @@ export default { updated_at INTEGER NOT NULL ) `) + yield* tx.run("CREATE INDEX event_snapshot_attempt_aggregate_idx ON event_snapshot_attempt(aggregate_id)") yield* tx.run(` CREATE TRIGGER event_snapshot_row_immutable BEFORE UPDATE ON event_snapshot_row @@ -55,6 +58,7 @@ export default { SELECT RAISE(ABORT, 'event_snapshot_row_immutable'); END `) + yield* tx.run("CREATE INDEX event_snapshot_row_hash_idx ON event_snapshot_row(row_hash)") yield* tx.run(` CREATE TRIGGER event_snapshot_row_delete_guard BEFORE DELETE ON event_snapshot_row @@ -63,11 +67,34 @@ export default { SELECT 1 FROM event_snapshot_attempt attempt WHERE attempt.snapshot_id = OLD.snapshot_id AND attempt.state != 'prepared' + AND EXISTS ( + SELECT 1 FROM event_sequence sequence + WHERE sequence.aggregate_id = attempt.aggregate_id + ) ) BEGIN SELECT RAISE(ABORT, 'event_snapshot_row_active'); END `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_row_chunk_cleanup + AFTER DELETE ON event_snapshot_row + BEGIN + DELETE FROM event_snapshot_chunk + WHERE row_hash = OLD.row_hash + AND NOT EXISTS ( + SELECT 1 FROM event_snapshot_row row + WHERE row.row_hash = OLD.row_hash + ); + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_aggregate_cleanup + AFTER DELETE ON event_sequence + BEGIN + DELETE FROM event_snapshot_row WHERE aggregate_id = OLD.aggregate_id; + END + `) yield* tx.run(` CREATE TRIGGER event_snapshot_chunk_immutable BEFORE UPDATE ON event_snapshot_chunk diff --git a/packages/core/src/database/migration/20260813140000_event_sidecar_compaction.ts b/packages/core/src/database/migration/20260813140000_event_sidecar_compaction.ts new file mode 100644 index 000000000..5e6c07326 --- /dev/null +++ b/packages/core/src/database/migration/20260813140000_event_sidecar_compaction.ts @@ -0,0 +1,512 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813140000_event_sidecar_compaction", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE IF NOT EXISTS file_part_artifact_discard ( + event_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL, + seq INTEGER NOT NULL CHECK (seq >= 0), + artifact_id TEXT NOT NULL REFERENCES file_part_artifact(artifact_id) ON DELETE CASCADE, + original_data_hash TEXT NOT NULL CHECK (length(original_data_hash) = 64), + canonical_data_hash TEXT NOT NULL CHECK (length(canonical_data_hash) = 64), + canonical_data TEXT NOT NULL CHECK (json_valid(canonical_data)), + created_at INTEGER NOT NULL, + UNIQUE (aggregate_id, seq) + ) + `) + + const snapshotRowColumns = (yield* tx.all<{ name: string }>("PRAGMA table_info('event_snapshot_row')")) + .map((column) => column.name) + const sequenceColumns = (yield* tx.all<{ name: string }>("PRAGMA table_info('event_sequence')")) + .map((column) => column.name) + const hasSnapshotAuthority = sequenceColumns.includes("snapshot_id") && sequenceColumns.includes("retention_floor_seq") + const legacySnapshotRowColumns = [ + "snapshot_id", "row_index", "table_name", "row_key", "row_hash", "row_bytes", "chunk_count", "chain_hash", + ] + const currentSnapshotRowColumns = [ + "snapshot_id", "aggregate_id", "row_index", "table_name", "row_key", "row_hash", "row_bytes", "chunk_count", "chain_hash", + ] + const hasSnapshotRows = snapshotRowColumns.length > 0 + const legacySnapshotRows = snapshotRowColumns.join("\0") === legacySnapshotRowColumns.join("\0") + if (hasSnapshotRows && !legacySnapshotRows && snapshotRowColumns.join("\0") !== currentSnapshotRowColumns.join("\0")) + return yield* Effect.fail(new Error(`unsupported event_snapshot_row schema: ${snapshotRowColumns.join(",")}`)) + const snapshotColumns = (yield* tx.all<{ name: string }>("PRAGMA table_info('event_snapshot')")).map( + (column) => column.name, + ) + const expectedSnapshotColumns = [ + "snapshot_id", "aggregate_id", "through_seq", "sync_seq", "codec", "schema_version", + "snapshot_hash", "body", "owner_id", "created_at", + ] + if (hasSnapshotRows && snapshotColumns.join("\0") !== expectedSnapshotColumns.join("\0")) + return yield* Effect.fail(new Error(`unsupported event_snapshot schema: ${snapshotColumns.join(",")}`)) + + const uniqueSnapshotIndexColumns = yield* Effect.forEach( + (yield* tx.all<{ name: string; unique: number }>("PRAGMA index_list('event_snapshot')")) + .filter((index) => index.unique === 1), + (index) => { + if (!/^[A-Za-z0-9_]+$/.test(index.name)) return Effect.fail(new Error("invalid event_snapshot index name")) + return tx.all<{ name: string }>(`PRAGMA index_info('${index.name}')`).pipe( + Effect.map((columns) => columns.map((column) => column.name).join(",")), + ) + }, + ) + const snapshotUniqueShapes = [...uniqueSnapshotIndexColumns].sort() + const legacySnapshot = snapshotUniqueShapes.join("\0") === ["aggregate_id,through_seq", "snapshot_id", "sync_seq"].sort().join("\0") + const currentSnapshot = snapshotUniqueShapes.join("\0") === ["snapshot_id", "sync_seq"].sort().join("\0") + if (hasSnapshotRows && !legacySnapshot && !currentSnapshot) + return yield* Effect.fail(new Error(`unsupported event_snapshot uniqueness: ${snapshotUniqueShapes.join(";")}`)) + + if (hasSnapshotAuthority) { + yield* tx.run("DROP TRIGGER IF EXISTS event_sequence_snapshot_validate_update") + yield* tx.run("DROP TRIGGER IF EXISTS event_snapshot_delete_active_guard") + } + yield* tx.run("DROP TRIGGER IF EXISTS event_snapshot_row_immutable") + yield* tx.run("DROP TRIGGER IF EXISTS event_snapshot_row_delete_guard") + yield* tx.run("DROP TRIGGER IF EXISTS event_snapshot_row_chunk_cleanup") + yield* tx.run("DROP TRIGGER IF EXISTS event_snapshot_chunk_delete_guard") + yield* tx.run("DROP TRIGGER IF EXISTS event_snapshot_aggregate_cleanup") + + if (hasSnapshotRows && legacySnapshot) { + yield* tx.run("ALTER TABLE event_snapshot RENAME TO event_snapshot_legacy") + yield* tx.run(` + CREATE TABLE event_snapshot ( + snapshot_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + through_seq INTEGER NOT NULL CHECK (through_seq >= 0), + sync_seq INTEGER NOT NULL UNIQUE CHECK (sync_seq >= 0), + codec TEXT NOT NULL CHECK (length(codec) > 0), + schema_version INTEGER NOT NULL CHECK (schema_version >= 1), + snapshot_hash TEXT NOT NULL CHECK (length(snapshot_hash) = 64), + body TEXT NOT NULL CHECK (json_valid(body)), + owner_id TEXT, + created_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + INSERT INTO event_snapshot( + snapshot_id, aggregate_id, through_seq, sync_seq, codec, schema_version, + snapshot_hash, body, owner_id, created_at + ) + SELECT snapshot_id, aggregate_id, through_seq, sync_seq, codec, schema_version, + snapshot_hash, body, owner_id, created_at + FROM event_snapshot_legacy + `) + yield* tx.run("DROP TABLE event_snapshot_legacy") + yield* tx.run("CREATE INDEX event_snapshot_aggregate_created_idx ON event_snapshot(aggregate_id, created_at)") + yield* tx.run("CREATE INDEX event_snapshot_aggregate_seq_idx ON event_snapshot(aggregate_id, through_seq)") + } + + if (hasSnapshotRows && legacySnapshotRows) { + const conflictingAuthorities = yield* tx.get<{ count: number }>(` + SELECT COUNT(*) AS count + FROM event_snapshot_row row + JOIN event_snapshot_attempt attempt ON attempt.snapshot_id = row.snapshot_id + JOIN event_snapshot snapshot ON snapshot.snapshot_id = row.snapshot_id + WHERE attempt.aggregate_id != snapshot.aggregate_id + `) + if ((conflictingAuthorities?.count ?? 0) !== 0) + return yield* Effect.fail(new Error("legacy event_snapshot_row has conflicting aggregate authorities")) + yield* tx.run(` + CREATE TEMP TABLE event_snapshot_orphan_row_hash ( + row_hash TEXT NOT NULL PRIMARY KEY + ) WITHOUT ROWID + `) + yield* tx.run(` + INSERT OR IGNORE INTO event_snapshot_orphan_row_hash(row_hash) + SELECT row.row_hash + FROM event_snapshot_row row + LEFT JOIN event_snapshot_attempt attempt ON attempt.snapshot_id = row.snapshot_id + LEFT JOIN event_snapshot snapshot ON snapshot.snapshot_id = row.snapshot_id + WHERE (attempt.aggregate_id IS NULL AND snapshot.aggregate_id IS NULL) + OR (attempt.aggregate_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM event_sequence sequence WHERE sequence.aggregate_id = attempt.aggregate_id + )) + `) + yield* tx.run(` + DELETE FROM event_snapshot_row + WHERE snapshot_id IN ( + SELECT row.snapshot_id + FROM event_snapshot_row row + LEFT JOIN event_snapshot_attempt attempt ON attempt.snapshot_id = row.snapshot_id + LEFT JOIN event_snapshot snapshot ON snapshot.snapshot_id = row.snapshot_id + WHERE (attempt.aggregate_id IS NULL AND snapshot.aggregate_id IS NULL) + OR (attempt.aggregate_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM event_sequence sequence WHERE sequence.aggregate_id = attempt.aggregate_id + )) + ) + `) + yield* tx.run(` + DELETE FROM event_snapshot_chunk + WHERE row_hash IN (SELECT row_hash FROM event_snapshot_orphan_row_hash) + AND NOT EXISTS ( + SELECT 1 FROM event_snapshot_row row + WHERE row.row_hash = event_snapshot_chunk.row_hash + ) + `) + yield* tx.run("DROP TABLE event_snapshot_orphan_row_hash") + yield* tx.run("ALTER TABLE event_snapshot_row RENAME TO event_snapshot_row_legacy") + yield* tx.run(` + CREATE TABLE event_snapshot_row ( + snapshot_id TEXT NOT NULL, + aggregate_id TEXT NOT NULL, + row_index INTEGER NOT NULL CHECK (row_index >= 0), + table_name TEXT NOT NULL CHECK (length(table_name) > 0), + row_key TEXT NOT NULL CHECK (length(row_key) > 0), + row_hash TEXT NOT NULL CHECK (length(row_hash) = 64), + row_bytes INTEGER NOT NULL CHECK (row_bytes > 0), + chunk_count INTEGER NOT NULL CHECK (chunk_count > 0), + chain_hash TEXT NOT NULL CHECK (length(chain_hash) = 64), + PRIMARY KEY (snapshot_id, row_index), + UNIQUE (snapshot_id, table_name, row_key) + ) + `) + yield* tx.run(` + INSERT INTO event_snapshot_row( + snapshot_id, aggregate_id, row_index, table_name, row_key, + row_hash, row_bytes, chunk_count, chain_hash + ) + SELECT row.snapshot_id, COALESCE(attempt.aggregate_id, snapshot.aggregate_id), + row.row_index, row.table_name, row.row_key, + row.row_hash, row.row_bytes, row.chunk_count, row.chain_hash + FROM event_snapshot_row_legacy row + LEFT JOIN event_snapshot_attempt attempt ON attempt.snapshot_id = row.snapshot_id + LEFT JOIN event_snapshot snapshot ON snapshot.snapshot_id = row.snapshot_id + WHERE COALESCE(attempt.aggregate_id, snapshot.aggregate_id) IS NOT NULL + `) + yield* tx.run("DROP TABLE event_snapshot_row_legacy") + } + + const attemptColumns = yield* tx.all<{ name: string }>("PRAGMA table_info('event_snapshot_attempt')") + const expectedAttemptColumns = [ + "snapshot_id", "aggregate_id", "through_seq", "expected_latest", "owner_id", "codec", + "schema_version", "projection_revision", "cursor", "row_count", "encoded_bytes", "content_hash", + "tables", "state", "created_at", "updated_at", + ] + if (hasSnapshotRows && attemptColumns.length > 0 && attemptColumns.map((column) => column.name).join("\0") !== expectedAttemptColumns.join("\0")) + return yield* Effect.fail(new Error(`unsupported event_snapshot_attempt schema: ${attemptColumns.map((column) => column.name).join(",")}`)) + const attemptForeignKeys = attemptColumns.length === 0 + ? [] + : yield* tx.all<{ from: string; table: string; to: string; on_delete: string }>( + "PRAGMA foreign_key_list('event_snapshot_attempt')", + ) + if (attemptColumns.length > 0 && !attemptForeignKeys.some((foreignKey) => + foreignKey.from === "aggregate_id" && foreignKey.table === "event_sequence" && + foreignKey.to === "aggregate_id" && foreignKey.on_delete.toUpperCase() === "CASCADE")) { + yield* tx.run(` + CREATE TEMP TABLE event_snapshot_orphan_attempt ( + snapshot_id TEXT NOT NULL PRIMARY KEY + ) WITHOUT ROWID + `) + yield* tx.run(` + INSERT INTO event_snapshot_orphan_attempt(snapshot_id) + SELECT attempt.snapshot_id + FROM event_snapshot_attempt attempt + LEFT JOIN event_sequence sequence ON sequence.aggregate_id = attempt.aggregate_id + WHERE sequence.aggregate_id IS NULL + `) + yield* tx.run(` + DELETE FROM event_snapshot_row + WHERE snapshot_id IN (SELECT snapshot_id FROM event_snapshot_orphan_attempt) + AND NOT EXISTS ( + SELECT 1 FROM event_snapshot snapshot + WHERE snapshot.snapshot_id = event_snapshot_row.snapshot_id + ) + `) + yield* tx.run(` + DELETE FROM event_snapshot_chunk + WHERE NOT EXISTS ( + SELECT 1 FROM event_snapshot_row row + WHERE row.row_hash = event_snapshot_chunk.row_hash + ) + `) + yield* tx.run(` + DELETE FROM event_snapshot_attempt + WHERE snapshot_id IN (SELECT snapshot_id FROM event_snapshot_orphan_attempt) + `) + yield* tx.run("DROP TABLE event_snapshot_orphan_attempt") + yield* tx.run("ALTER TABLE event_snapshot_attempt RENAME TO event_snapshot_attempt_legacy") + yield* tx.run(` + CREATE TABLE event_snapshot_attempt ( + snapshot_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + through_seq INTEGER NOT NULL CHECK (through_seq >= 0), + expected_latest INTEGER NOT NULL CHECK (expected_latest >= 0), + owner_id TEXT, + codec TEXT NOT NULL CHECK (length(codec) > 0), + schema_version INTEGER NOT NULL CHECK (schema_version > 0), + projection_revision TEXT NOT NULL, + cursor TEXT, + row_count INTEGER NOT NULL CHECK (row_count >= 0), + encoded_bytes INTEGER NOT NULL CHECK (encoded_bytes >= 0), + content_hash TEXT NOT NULL CHECK (length(content_hash) = 64), + tables TEXT NOT NULL CHECK (json_valid(tables)), + state TEXT NOT NULL CHECK (state IN ('prepared', 'staged', 'complete')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + INSERT INTO event_snapshot_attempt( + snapshot_id, aggregate_id, through_seq, expected_latest, owner_id, codec, + schema_version, projection_revision, cursor, row_count, encoded_bytes, + content_hash, tables, state, created_at, updated_at + ) + SELECT snapshot_id, aggregate_id, through_seq, expected_latest, owner_id, codec, + schema_version, projection_revision, cursor, row_count, encoded_bytes, + content_hash, tables, state, created_at, updated_at + FROM event_snapshot_attempt_legacy + `) + yield* tx.run("DROP TABLE event_snapshot_attempt_legacy") + } + + if (hasSnapshotAuthority) { + yield* tx.run(` + CREATE TRIGGER event_sequence_snapshot_validate_update + BEFORE UPDATE OF retention_floor_seq, snapshot_id ON event_sequence + BEGIN + SELECT CASE + WHEN NEW.retention_floor_seq IS NULL OR NEW.snapshot_id IS NULL + THEN RAISE(ABORT, 'event_sequence_snapshot_pointer_incomplete') + WHEN NEW.retention_floor_seq < 0 OR NEW.retention_floor_seq > NEW.seq + THEN RAISE(ABORT, 'event_sequence_snapshot_floor_invalid') + WHEN OLD.retention_floor_seq IS NOT NULL AND NEW.retention_floor_seq < OLD.retention_floor_seq + THEN RAISE(ABORT, 'event_sequence_snapshot_floor_regressed') + WHEN NOT EXISTS ( + SELECT 1 FROM event_snapshot snapshot + WHERE snapshot.snapshot_id = NEW.snapshot_id + AND snapshot.aggregate_id = NEW.aggregate_id + AND snapshot.through_seq = NEW.retention_floor_seq + ) THEN RAISE(ABORT, 'event_sequence_snapshot_missing') + END; + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_delete_active_guard + BEFORE DELETE ON event_snapshot + WHEN EXISTS (SELECT 1 FROM event_sequence sequence WHERE sequence.snapshot_id = OLD.snapshot_id) + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_is_active'); + END + `) + } + if (hasSnapshotRows) { + yield* tx.run("CREATE INDEX IF NOT EXISTS event_snapshot_row_hash_idx ON event_snapshot_row(row_hash)") + yield* tx.run("CREATE INDEX IF NOT EXISTS event_snapshot_row_aggregate_idx ON event_snapshot_row(aggregate_id)") + yield* tx.run("CREATE INDEX IF NOT EXISTS event_snapshot_attempt_aggregate_idx ON event_snapshot_attempt(aggregate_id)") + yield* tx.run(` + CREATE TRIGGER event_snapshot_row_immutable + BEFORE UPDATE ON event_snapshot_row + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_row_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_row_delete_guard + BEFORE DELETE ON event_snapshot_row + WHEN EXISTS (SELECT 1 FROM event_snapshot snapshot WHERE snapshot.snapshot_id = OLD.snapshot_id) + OR EXISTS ( + SELECT 1 FROM event_snapshot_attempt attempt + WHERE attempt.snapshot_id = OLD.snapshot_id + AND attempt.state != 'prepared' + AND EXISTS ( + SELECT 1 FROM event_sequence sequence + WHERE sequence.aggregate_id = attempt.aggregate_id + ) + ) + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_row_active'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_row_chunk_cleanup + AFTER DELETE ON event_snapshot_row + BEGIN + DELETE FROM event_snapshot_chunk + WHERE row_hash = OLD.row_hash + AND NOT EXISTS ( + SELECT 1 FROM event_snapshot_row row + WHERE row.row_hash = OLD.row_hash + ); + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_aggregate_cleanup + AFTER DELETE ON event_sequence + BEGIN + DELETE FROM event_snapshot_row WHERE aggregate_id = OLD.aggregate_id; + DELETE FROM event_snapshot_attempt WHERE aggregate_id = OLD.aggregate_id; + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_chunk_delete_guard + BEFORE DELETE ON event_snapshot_chunk + WHEN EXISTS (SELECT 1 FROM event_snapshot_row row WHERE row.row_hash = OLD.row_hash) + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_chunk_referenced'); + END + `) + } + + yield* tx.run("DROP TRIGGER IF EXISTS event_delete_requires_dedupe") + yield* tx.run(` + CREATE TRIGGER event_delete_requires_dedupe + BEFORE DELETE ON event + WHEN EXISTS ( + SELECT 1 FROM event_sequence sequence + WHERE sequence.aggregate_id = OLD.aggregate_id + AND sequence.retention_floor_seq IS NOT NULL + AND OLD.seq <= sequence.retention_floor_seq + ) AND NOT EXISTS ( + SELECT 1 FROM event_dedupe dedupe + WHERE dedupe.aggregate_id = OLD.aggregate_id + AND dedupe.seq = OLD.seq + AND dedupe.event_id = OLD.id + AND dedupe.type = OLD.type + AND length(dedupe.data_hash) = 64 + AND ( + dedupe.source_data = OLD.data + OR ( + dedupe.source_data IS NULL + AND ( + EXISTS ( + SELECT 1 FROM event_artifact artifact + WHERE artifact.event_id = OLD.id + AND artifact.aggregate_id = OLD.aggregate_id + AND artifact.seq = OLD.seq + AND artifact.original_data_hash = dedupe.data_hash + ) + OR EXISTS ( + SELECT 1 FROM file_part_artifact_binding binding + WHERE binding.event_id = OLD.id + AND binding.aggregate_id = OLD.aggregate_id + AND binding.seq = OLD.seq + AND binding.original_data_hash = dedupe.data_hash + ) + ) + ) + ) + ) + BEGIN + SELECT RAISE(ABORT, 'event_compaction_dedupe_missing'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_dedupe_update_guard + BEFORE UPDATE ON event_dedupe + WHEN NOT ( + NEW.aggregate_id = OLD.aggregate_id + AND NEW.seq = OLD.seq + AND NEW.event_id = OLD.event_id + AND NEW.type = OLD.type + AND NEW.data_hash = OLD.data_hash + AND NEW.compacted_at = OLD.compacted_at + AND OLD.source_data IS NOT NULL + AND NEW.source_data IS NULL + ) + BEGIN + SELECT RAISE(ABORT, 'event_dedupe_update_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_dedupe_delete_guard + BEFORE DELETE ON event_dedupe + WHEN EXISTS ( + SELECT 1 FROM event_sequence sequence + WHERE sequence.aggregate_id = OLD.aggregate_id + ) + BEGIN + SELECT RAISE(ABORT, 'event_dedupe_delete_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER event_snapshot_update_immutable + BEFORE UPDATE ON event_snapshot + BEGIN + SELECT RAISE(ABORT, 'event_snapshot_update_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER file_part_artifact_import_update_immutable + BEFORE UPDATE ON file_part_artifact_import + BEGIN + SELECT RAISE(ABORT, 'file_part_artifact_import_update_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER file_part_artifact_import_delete_guard + BEFORE DELETE ON file_part_artifact_import + WHEN EXISTS ( + SELECT 1 FROM file_part_artifact artifact + WHERE artifact.artifact_id = OLD.artifact_id + ) AND NOT EXISTS ( + SELECT 1 FROM file_part_artifact_binding binding + WHERE binding.event_id = OLD.event_id + AND binding.aggregate_id = OLD.aggregate_id + AND binding.seq = OLD.seq + AND binding.artifact_id = OLD.artifact_id + AND binding.original_data_hash = OLD.original_data_hash + AND binding.canonical_data_hash = OLD.canonical_data_hash + AND binding.canonical_data = OLD.canonical_data + ) AND NOT EXISTS ( + SELECT 1 FROM file_part_artifact_discard discard + WHERE discard.event_id = OLD.event_id + AND discard.aggregate_id = OLD.aggregate_id + AND discard.seq = OLD.seq + AND discard.artifact_id = OLD.artifact_id + AND discard.original_data_hash = OLD.original_data_hash + AND discard.canonical_data_hash = OLD.canonical_data_hash + AND discard.canonical_data = OLD.canonical_data + ) + BEGIN + SELECT RAISE(ABORT, 'file_part_artifact_import_not_consumed'); + END + `) + yield* tx.run(` + CREATE TRIGGER file_part_artifact_discard_update_immutable + BEFORE UPDATE ON file_part_artifact_discard + BEGIN + SELECT RAISE(ABORT, 'file_part_artifact_discard_update_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER file_part_artifact_discard_delete_guard + BEFORE DELETE ON file_part_artifact_discard + WHEN EXISTS ( + SELECT 1 FROM file_part_artifact artifact + WHERE artifact.artifact_id = OLD.artifact_id + ) + BEGIN + SELECT RAISE(ABORT, 'file_part_artifact_discard_delete_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER file_part_artifact_binding_cleanup + AFTER DELETE ON file_part_artifact_binding + BEGIN + DELETE FROM file_part_artifact + WHERE artifact_id = OLD.artifact_id + AND NOT EXISTS ( + SELECT 1 FROM file_part_artifact_binding binding + WHERE binding.artifact_id = OLD.artifact_id + ) + AND NOT EXISTS ( + SELECT 1 FROM file_part_artifact_import imported + WHERE imported.artifact_id = OLD.artifact_id + ); + END + `) + yield* tx.run(` + CREATE INDEX IF NOT EXISTS file_part_artifact_binding_artifact_idx + ON file_part_artifact_binding(artifact_id) + `) + yield* tx.run(` + CREATE INDEX IF NOT EXISTS file_part_artifact_import_artifact_idx + ON file_part_artifact_import(artifact_id) + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813141000_bug_407_010_sidecar_lifecycle.ts b/packages/core/src/database/migration/20260813141000_bug_407_010_sidecar_lifecycle.ts new file mode 100644 index 000000000..544ef5d53 --- /dev/null +++ b/packages/core/src/database/migration/20260813141000_bug_407_010_sidecar_lifecycle.ts @@ -0,0 +1,10 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813141000_bug_407_010_sidecar_lifecycle", + up() { + // Drizzle schema baseline only. The staged runtime migrations below this ID own the online DDL. + return Effect.void + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813142000_bug_407_010_sidecar_indexes.ts b/packages/core/src/database/migration/20260813142000_bug_407_010_sidecar_indexes.ts new file mode 100644 index 000000000..e642b98d6 --- /dev/null +++ b/packages/core/src/database/migration/20260813142000_bug_407_010_sidecar_indexes.ts @@ -0,0 +1,10 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813142000_bug_407_010_sidecar_indexes", + up() { + // Drizzle schema baseline only. Runtime migrations 131300/131310/131400 own the online index DDL. + return Effect.void + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813143000_bug_407_010_aggregate_indexes.ts b/packages/core/src/database/migration/20260813143000_bug_407_010_aggregate_indexes.ts new file mode 100644 index 000000000..1b4356f44 --- /dev/null +++ b/packages/core/src/database/migration/20260813143000_bug_407_010_aggregate_indexes.ts @@ -0,0 +1,10 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813143000_bug_407_010_aggregate_indexes", + up() { + // Drizzle schema baseline only. Runtime migrations own the online index DDL. + return Effect.void + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/deepagent/deepagent-event-bus.ts b/packages/core/src/deepagent/deepagent-event-bus.ts index f76d8eb9b..1791352b3 100644 --- a/packages/core/src/deepagent/deepagent-event-bus.ts +++ b/packages/core/src/deepagent/deepagent-event-bus.ts @@ -160,6 +160,7 @@ export interface Interface { readonly sweep: (input: { readonly workspaceID: string readonly olderThan: number + readonly limit?: number }) => Effect.Effect<{ readonly deletedEvents: number }> /** * §E2 — prune the publish rate-limiter's per-workspace buckets whose fixed window has elapsed as of @@ -839,6 +840,13 @@ export const layerWith = (options?: LayerOptions) => noPendingApproval, noLiveDlqAlert, ) + const limit = Math.min(Math.max(input.limit ?? 100, 1), 100) + const candidateIDs = db + .select({ id: DeepAgentEventTable.id }) + .from(DeepAgentEventTable) + .where(deletable) + .orderBy(asc(DeepAgentEventTable.id)) + .limit(limit) // Delete the terminal (delivered/dead) delivery rows of the doomed events FIRST. The FK // cascade already removes them, but doing it explicitly keeps the sweep correct even if a @@ -847,17 +855,14 @@ export const layerWith = (options?: LayerOptions) => yield* db .delete(DeepAgentEventDeliveryTable) .where( - sql`${DeepAgentEventDeliveryTable.event_id} in (${db - .select({ id: DeepAgentEventTable.id }) - .from(DeepAgentEventTable) - .where(deletable)})`, + sql`${DeepAgentEventDeliveryTable.event_id} in (${candidateIDs})`, ) .run() .pipe(Effect.orDie) const deleted = yield* db .delete(DeepAgentEventTable) - .where(deletable) + .where(sql`${DeepAgentEventTable.id} in (${candidateIDs})`) .returning({ id: DeepAgentEventTable.id }) .all() .pipe(Effect.orDie) diff --git a/packages/core/src/deepagent/retention-sweeper.ts b/packages/core/src/deepagent/retention-sweeper.ts index 5effac61e..235605b00 100644 --- a/packages/core/src/deepagent/retention-sweeper.ts +++ b/packages/core/src/deepagent/retention-sweeper.ts @@ -1,14 +1,20 @@ export * as RetentionSweeper from "./retention-sweeper" import { Cause, Context, Duration, Effect, Layer, Schedule } from "effect" -import { and, eq, isNotNull, lt } from "drizzle-orm" +import { and, asc, eq, inArray, isNotNull, lt, sql } from "drizzle-orm" import { Database } from "../database/database" import { DeepAgentEventBus } from "./deepagent-event-bus" import { DeepAgentEventTable } from "./deepagent-event-sql" import { ApprovalQueueTable } from "./approval-queue-sql" import { WorkspaceConfig } from "./workspace-config" import { AgentPushLogTable } from "../im/push-log-sql" -import { EventSequenceTable } from "../event/sql" +import { EventArtifactTable, EventCompactionReceiptTable, EventSequenceTable } from "../event/sql" +import { + FilePartArtifactBindingTable, + FilePartArtifactDiscardTable, + FilePartArtifactImportTable, + FilePartArtifactTable, +} from "../file-part-artifact.sql" import { SessionTable } from "../session/sql" import type { WorkspaceV2 } from "../workspace" import * as Log from "../util/log" @@ -29,6 +35,7 @@ import * as Log from "../util/log" const log = Log.create({ service: "retention-sweeper" }) const DAY_MS = 86_400_000 +const SWEEP_BATCH_ROWS = 100 // default sweep cadence — hourly (retention is a slow reclaim; a missed hour is harmless). export const DEFAULT_SWEEP_INTERVAL_MS = Duration.toMillis(Duration.hours(1)) @@ -58,6 +65,7 @@ export interface LayerOptions { readonly intervalMs?: number // start the background sweep daemon (scoped fork). Default true; tests pass false and call sweepOnce. readonly runLoop?: boolean + readonly compactSession?: (sessionID: string) => Effect.Effect } export const layerWith = (options?: LayerOptions) => @@ -75,91 +83,153 @@ export const layerWith = (options?: LayerOptions) => Effect.gen(function* () { const at = nowArg ?? now() - // enumerate the workspaces that actually have events — the only ones worth a retention pass. - // (push-log / approval pruning is scoped to these same workspaces; a workspace with no events - // but stray audit rows is swept the next time it publishes — acceptable for slow reclaim.) - const workspaceRows = yield* db + const eventWorkspaceRows = yield* db .selectDistinct({ workspaceID: DeepAgentEventTable.workspace_id }) .from(DeepAgentEventTable) + .orderBy(asc(DeepAgentEventTable.workspace_id)) + .limit(SWEEP_BATCH_ROWS) .all() .pipe(Effect.orDie) + const sessionWorkspaceRows = yield* db + .selectDistinct({ workspaceID: SessionTable.workspace_id }) + .from(SessionTable) + .where(isNotNull(SessionTable.workspace_id)) + .orderBy(asc(SessionTable.workspace_id)) + .limit(SWEEP_BATCH_ROWS) + .all() + .pipe(Effect.orDie) + const workspaceRows = [...new Set([...eventWorkspaceRows, ...sessionWorkspaceRows].map((row) => row.workspaceID))] + .filter((workspaceID): workspaceID is WorkspaceV2.ID => workspaceID !== null) + .map((workspaceID) => ({ workspaceID })) let deletedEvents = 0 let deletedPushLogs = 0 let deletedApprovals = 0 let deletedEventV2Sequences = 0 + const abandonedImports = yield* db + .select({ + eventID: FilePartArtifactImportTable.event_id, + artifactID: FilePartArtifactImportTable.artifact_id, + aggregateID: FilePartArtifactImportTable.aggregate_id, + seq: FilePartArtifactImportTable.seq, + originalDataHash: FilePartArtifactImportTable.original_data_hash, + canonicalDataHash: FilePartArtifactImportTable.canonical_data_hash, + canonicalData: FilePartArtifactImportTable.canonical_data, + }) + .from(FilePartArtifactImportTable) + .where(lt(FilePartArtifactImportTable.created_at, at - DAY_MS)) + .limit(SWEEP_BATCH_ROWS) + .all() + .pipe(Effect.orDie) + yield* db.transaction( + () => Effect.forEach( + abandonedImports, + (imported) => Effect.gen(function* () { + const binding = yield* db.select({ id: FilePartArtifactBindingTable.event_id }) + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, imported.eventID)).get().pipe(Effect.orDie) + if (binding) return + yield* db.insert(FilePartArtifactDiscardTable).values({ + event_id: imported.eventID, + aggregate_id: imported.aggregateID, + seq: imported.seq, + artifact_id: imported.artifactID, + original_data_hash: imported.originalDataHash, + canonical_data_hash: imported.canonicalDataHash, + canonical_data: imported.canonicalData, + created_at: at, + }).onConflictDoNothing().run().pipe(Effect.orDie) + yield* db.delete(FilePartArtifactImportTable) + .where(eq(FilePartArtifactImportTable.event_id, imported.eventID)).run().pipe(Effect.orDie) + yield* db.delete(FilePartArtifactTable).where(and( + eq(FilePartArtifactTable.artifact_id, imported.artifactID), + sql`NOT EXISTS ( + SELECT 1 FROM ${FilePartArtifactBindingTable} binding + WHERE binding.artifact_id = ${imported.artifactID} + )`, + sql`NOT EXISTS ( + SELECT 1 FROM ${FilePartArtifactImportTable} imported + WHERE imported.artifact_id = ${imported.artifactID} + )`, + )).run().pipe(Effect.orDie) + }), + { discard: true }, + ), + { behavior: "immediate" }, + ).pipe(Effect.orDie) + for (const { workspaceID } of workspaceRows) { const resolved = yield* config.get(workspaceID) const olderThan = at - resolved.retentionDays * DAY_MS // §A3 events (referential-safe sweep on the bus). - const eventResult = yield* bus.sweep({ workspaceID, olderThan }) + const eventResult = yield* bus.sweep({ workspaceID, olderThan, limit: SWEEP_BATCH_ROWS }) deletedEvents += eventResult.deletedEvents // §B4 push audit log — prune this workspace's rows past retention. - const pushDeleted = yield* db - .delete(AgentPushLogTable) - .where( - and( - eq(AgentPushLogTable.workspace_id, workspaceID), - lt(AgentPushLogTable.created_at, olderThan), - ), - ) - .returning({ id: AgentPushLogTable.id }) - .all() - .pipe(Effect.orDie) - deletedPushLogs += pushDeleted.length + const pushIDs = yield* db.select({ id: AgentPushLogTable.id }).from(AgentPushLogTable).where(and( + eq(AgentPushLogTable.workspace_id, workspaceID), lt(AgentPushLogTable.created_at, olderThan), + )).orderBy(asc(AgentPushLogTable.id)).limit(SWEEP_BATCH_ROWS).all().pipe(Effect.orDie) + if (pushIDs.length > 0) { + const pushDeleted = yield* db.delete(AgentPushLogTable).where(inArray(AgentPushLogTable.id, pushIDs.map((row) => row.id))).returning({ id: AgentPushLogTable.id }).all().pipe(Effect.orDie) + deletedPushLogs += pushDeleted.length + } // §D2 approval queue — prune RESOLVED items past retention only. A pending item survives // regardless of age (a human still owes it a decision). - const approvalDeleted = yield* db - .delete(ApprovalQueueTable) - .where( - and( - eq(ApprovalQueueTable.workspace_id, workspaceID), - eq(ApprovalQueueTable.status, "resolved"), - lt(ApprovalQueueTable.created_at, olderThan), - ), - ) - .returning({ id: ApprovalQueueTable.id }) - .all() - .pipe(Effect.orDie) - deletedApprovals += approvalDeleted.length - - // PERF §EventV2-retention: prune EventV2 mirror events for sessions that have been - // archived for longer than retentionDays. The EventV2 `event` table records every - // streaming delta (message.part.updated, etc.) and has no built-in expiry — it grows - // unboundedly and can reach 500MB+ on long-running deployments. Once a session is - // archived its event history is cold and safe to discard: active sessions keep their - // full event log for live projector replay, but archived ones no longer need it. - // - // Implementation: delete rows from `event_sequence` where the corresponding session row - // (same workspace) has time_archived IS NOT NULL AND time_archived < olderThan. - // The EventTable has ON DELETE CASCADE from event_sequence, so this one DELETE removes - // both the sequence header and all its events in a single statement. - const archivedSessions = yield* db + const approvalIDs = yield* db.select({ id: ApprovalQueueTable.id }).from(ApprovalQueueTable).where(and( + eq(ApprovalQueueTable.workspace_id, workspaceID), eq(ApprovalQueueTable.status, "resolved"), lt(ApprovalQueueTable.created_at, olderThan), + )).orderBy(asc(ApprovalQueueTable.id)).limit(SWEEP_BATCH_ROWS).all().pipe(Effect.orDie) + if (approvalIDs.length > 0) { + const approvalDeleted = yield* db.delete(ApprovalQueueTable).where(inArray(ApprovalQueueTable.id, approvalIDs.map((row) => row.id))).returning({ id: ApprovalQueueTable.id }).all().pipe(Effect.orDie) + deletedApprovals += approvalDeleted.length + } + + // Archived sessions may be compacted through their active snapshot, but the aggregate + // root remains readable until the explicit Session/EventV2 remove authority runs. The + // sweeper must not delete EventSequence directly: doing so would bypass the root + // lifecycle and make replay/fork history disappear while the Session row remains. + const archivedFilter = and( + eq(SessionTable.workspace_id, workspaceID as WorkspaceV2.ID), + isNotNull(SessionTable.time_archived), + lt(SessionTable.time_archived, olderThan), + ) + + const compactableSessions = yield* db .select({ id: SessionTable.id }) .from(SessionTable) - .where( - and( - eq(SessionTable.workspace_id, workspaceID as WorkspaceV2.ID), - isNotNull(SessionTable.time_archived), - lt(SessionTable.time_archived, olderThan), - ), - ) + .where(and( + archivedFilter, + sql`EXISTS ( + SELECT 1 FROM ${EventSequenceTable} sequence + WHERE sequence.aggregate_id = ${SessionTable.id} + AND NOT EXISTS ( + SELECT 1 FROM ${EventCompactionReceiptTable} receipt + WHERE receipt.aggregate_id = sequence.aggregate_id + AND receipt.state = 'complete' + AND receipt.through_seq >= sequence.seq + ) + )`, + sql`( + EXISTS ( + SELECT 1 FROM ${EventArtifactTable} artifact + WHERE artifact.aggregate_id = ${SessionTable.id} + ) OR EXISTS ( + SELECT 1 FROM ${FilePartArtifactBindingTable} binding + WHERE binding.aggregate_id = ${SessionTable.id} + ) + )`, + )) + .orderBy(SessionTable.id) + .limit(SWEEP_BATCH_ROWS) .all() .pipe(Effect.orDie) - - for (const { id } of archivedSessions) { - const seqDeleted = yield* db - .delete(EventSequenceTable) - .where(eq(EventSequenceTable.aggregate_id, id)) - .returning({ aggregate_id: EventSequenceTable.aggregate_id }) - .all() - .pipe(Effect.orDie) - deletedEventV2Sequences += seqDeleted.length - } + yield* Effect.forEach( + compactableSessions, + (session) => options?.compactSession?.(session.id) ?? Effect.void, + { discard: true }, + ) } return { @@ -174,8 +244,9 @@ export const layerWith = (options?: LayerOptions) => // Background daemon (scoped to the layer). A failure in a single pass is logged and swallowed so // the loop never dies on one bad sweep. Schedule.spaced waits between completions. if (runLoop) { - yield* sweepOnce() + yield* Effect.sleep(Duration.millis(intervalMs)) .pipe( + Effect.andThen(sweepOnce()), Effect.catchCause((cause) => Effect.sync(() => log.error("retention sweep failed", { cause: Cause.pretty(cause) })).pipe( Effect.as({ diff --git a/packages/core/src/event.ts b/packages/core/src/event.ts index b089afba7..c9e15da2f 100644 --- a/packages/core/src/event.ts +++ b/packages/core/src/event.ts @@ -268,25 +268,6 @@ function hashJson(value: unknown) { return Hash.sha256(stableJson(value)) } -function isLegacyFileArtifactRetry( - type: string, - stored: Record, - original: Record, - encoded: Record, - binding: { readonly originalDataHash: string; readonly canonicalData: Record }, -) { - if (!isDeepStrictEqual(stored, original) || !FilePartArtifact.matchesDataHash(binding.originalDataHash, original)) - return false - const prepared = FilePartArtifact.prepare( - type, - original, - Buffer.byteLength(JSON.stringify(original)), - MAX_ENCODED_PAYLOAD_BYTES, - ) - if (prepared.artifacts.length !== 1 || !isDeepStrictEqual(prepared.data, encoded)) return false - return FilePartArtifact.isLegacySyntheticCanonical(binding.canonicalData, encoded) -} - function isCompactedLegacyFileArtifactRetry( type: string, original: Record, @@ -301,7 +282,8 @@ function isCompactedLegacyFileArtifactRetry( MAX_ENCODED_PAYLOAD_BYTES, ) if (prepared.artifacts.length !== 1 || !isDeepStrictEqual(prepared.data, encoded)) return false - return FilePartArtifact.isLegacySyntheticCanonical(binding.canonicalData, encoded) + return isDeepStrictEqual(binding.canonicalData, encoded) || + FilePartArtifact.isLegacySyntheticCanonical(binding.canonicalData, encoded) } // Synchronized events cross a JSON boundary, so their data schemas must encode and decode without services. @@ -390,6 +372,10 @@ export interface Interface { options?: { readonly publish?: boolean; readonly ownerID?: string; readonly strictOwner?: boolean }, ) => Effect.Effect readonly snapshot: (aggregateID: string) => Effect.Effect + readonly aggregateState?: (aggregateID: string) => Effect.Effect<{ + readonly seq: Cursor + readonly ownerID?: string + } | undefined> readonly prepareCheckpoint?: (input: { readonly aggregateID: string readonly throughSeq: Cursor @@ -412,6 +398,11 @@ export interface Interface { readonly snapshotID: string readonly limit?: number }) => Effect.Effect<{ readonly deletedRows: number; readonly complete: boolean }> + readonly discardImportedSnapshot?: (input: { + readonly snapshotID: string + readonly aggregateID: string + readonly limit?: number + }) => Effect.Effect<{ readonly deletedRows: number; readonly complete: boolean }> readonly checkpoint: (input: { readonly aggregateID: string readonly throughSeq: Cursor @@ -646,37 +637,40 @@ export const layerWith = (layerOptions?: LayerOptions) => if ( ((stored?.id === event.id && stored.type === versionedType(definition.type, sync.version) && - (isDeepStrictEqual(stored.data, encoded) || - isDeepStrictEqual(artifact?.canonicalData, encoded) || - (fileArtifact && - (isDeepStrictEqual(fileArtifact.canonicalData, encoded) || - isLegacyFileArtifactRetry( - definition.type, - stored.data, - original, - encoded, - fileArtifact, - ))) || - (artifact && - artifact.originalDataHash === - (artifact.codecVersion >= 2 - ? Hash.sha256(JSON.stringify(encoded)) - : hashJson(encoded))))) || + (fileArtifact + ? (isDeepStrictEqual(original, encoded) && + isDeepStrictEqual(fileArtifact.canonicalData, encoded)) || + isCompactedLegacyFileArtifactRetry( + definition.type, + original, + encoded, + fileArtifact, + ) + : artifact + ? isDeepStrictEqual(artifact.canonicalData, encoded) || + artifact.originalDataHash === + (artifact.codecVersion >= 2 + ? Hash.sha256(JSON.stringify(encoded)) + : hashJson(encoded)) + : isDeepStrictEqual(stored.data, encoded))) || (dedupe?.event_id === event.id && dedupe.type === versionedType(definition.type, sync.version) && - (dedupe.data_hash === hashJson(encoded) || - (artifact && + (artifact + ? isDeepStrictEqual(artifact.canonicalData, encoded) || artifact.originalDataHash === (artifact.codecVersion >= 2 ? Hash.sha256(JSON.stringify(encoded)) - : hashJson(encoded))) || - (fileArtifact && - isCompactedLegacyFileArtifactRetry( - definition.type, - original, - encoded, - fileArtifact, - ))))) + : hashJson(encoded)) + : fileArtifact + ? (isDeepStrictEqual(original, encoded) && + isDeepStrictEqual(fileArtifact.canonicalData, encoded)) || + isCompactedLegacyFileArtifactRetry( + definition.type, + original, + encoded, + fileArtifact, + ) + : dedupe.data_hash === hashJson(encoded)))) ) { if (input.ownerID && row?.ownerID == null) { for (const guard of commitGuards) { @@ -728,12 +722,32 @@ export const layerWith = (layerOptions?: LayerOptions) => .where(eq(EventTable.id, event.id)) .get() .pipe(Effect.orDie) + const storedFileArtifact = stored + ? yield* db + .select({ + originalDataHash: FilePartArtifactBindingTable.original_data_hash, + canonicalData: FilePartArtifactBindingTable.canonical_data, + }) + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, event.id)) + .get() + .pipe(Effect.orDie) + : undefined if ( stored && idempotent && stored.aggregateID === aggregateID && stored.type === versionedType(definition.type, sync.version) && - isDeepStrictEqual(stored.data, encoded) + (storedFileArtifact + ? (isDeepStrictEqual(original, encoded) && + isDeepStrictEqual(storedFileArtifact.canonicalData, encoded)) || + isCompactedLegacyFileArtifactRetry( + definition.type, + original, + encoded, + storedFileArtifact, + ) + : isDeepStrictEqual(stored.data, encoded)) ) { if (commit) yield* commit(stored.seq) return { aggregateID, seq: stored.seq, inserted: false } @@ -1069,6 +1083,17 @@ export const layerWith = (layerOptions?: LayerOptions) => ) } + function aggregateState(aggregateID: string) { + return db.select({ seq: EventSequenceTable.seq, ownerID: EventSequenceTable.owner_id }) + .from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, aggregateID)).get().pipe( + Effect.orDie, + Effect.map((row) => row ? { + seq: Cursor.make(row.seq), + ...(row.ownerID ? { ownerID: row.ownerID } : {}), + } : undefined), + ) + } + function checkpoint(input: { readonly aggregateID: string readonly throughSeq: Cursor @@ -1217,7 +1242,8 @@ export const layerWith = (layerOptions?: LayerOptions) => const chunks = Math.ceil(data.length / SNAPSHOT_CHUNK_BYTES) const chainHash = Hash.sha256(`${attempt.content_hash}\0${row.tableName}\0${row.rowKey}\0${rowHash}\0${data.length}`) const insertedRow = yield* db.insert(EventSnapshotRowTable).values({ - snapshot_id: input.snapshotID, row_index: rowCount, table_name: row.tableName, row_key: row.rowKey, + snapshot_id: input.snapshotID, aggregate_id: attempt.aggregate_id, row_index: rowCount, + table_name: row.tableName, row_key: row.rowKey, row_hash: rowHash, row_bytes: data.length, chunk_count: chunks, chain_hash: chainHash, }).onConflictDoNothing().returning().get().pipe(Effect.orDie) if (!insertedRow) { @@ -1301,6 +1327,8 @@ export const layerWith = (layerOptions?: LayerOptions) => if (!syncSequence) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: "Sync sequence authority missing" })) const createdAt = input.now ?? Date.now() const snapshotHash = hashJson(body) + const previousSnapshot = yield* db.select({ snapshotID: EventSequenceTable.snapshot_id }) + .from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, attempt.aggregate_id)).get().pipe(Effect.orDie) yield* db.insert(EventSnapshotTable).values({ snapshot_id: input.snapshotID, aggregate_id: attempt.aggregate_id, through_seq: attempt.through_seq, sync_seq: syncSequence.seq, codec: attempt.codec, schema_version: attempt.schema_version, snapshot_hash: snapshotHash, body, owner_id: attempt.owner_id, created_at: createdAt }).run().pipe(Effect.orDie) @@ -1308,6 +1336,11 @@ export const layerWith = (layerOptions?: LayerOptions) => .where(and(eq(EventSequenceTable.aggregate_id, attempt.aggregate_id), eq(EventSequenceTable.seq, attempt.expected_latest))).run().pipe(Effect.orDie) yield* db.update(EventSnapshotAttemptTable).set({ state: "complete", content_hash: body.contentHash, updated_at: createdAt }) .where(eq(EventSnapshotAttemptTable.snapshot_id, input.snapshotID)).run().pipe(Effect.orDie) + if (previousSnapshot?.snapshotID && previousSnapshot.snapshotID !== input.snapshotID) { + yield* db.delete(EventSnapshotTable).where(eq(EventSnapshotTable.snapshot_id, previousSnapshot.snapshotID)).run().pipe(Effect.orDie) + yield* db.delete(EventSnapshotAttemptTable).where(eq(EventSnapshotAttemptTable.snapshot_id, previousSnapshot.snapshotID)).run().pipe(Effect.orDie) + yield* db.delete(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.snapshot_id, previousSnapshot.snapshotID)).run().pipe(Effect.orDie) + } return { snapshotID: input.snapshotID, aggregateID: attempt.aggregate_id, throughSeq: attempt.through_seq, syncSeq: syncSequence.seq, codec: attempt.codec, schemaVersion: attempt.schema_version, snapshotHash, body, ...(attempt.owner_id ? { ownerID: attempt.owner_id } : {}), createdAt } @@ -1342,6 +1375,71 @@ export const layerWith = (layerOptions?: LayerOptions) => ).pipe(Effect.orDie) } + function discardImportedSnapshot(input: { + readonly snapshotID: string + readonly aggregateID: string + readonly limit?: number + }) { + const limit = Math.min(Math.max(input.limit ?? SNAPSHOT_TRANSFER_ROWS, 1), SNAPSHOT_TRANSFER_ROWS) + return db.transaction( + () => Effect.gen(function* () { + const active = yield* db + .select({ aggregateID: EventSnapshotTable.aggregate_id }) + .from(EventSnapshotTable) + .where(eq(EventSnapshotTable.snapshot_id, input.snapshotID)) + .get() + .pipe(Effect.orDie) + if (active) { + if (active.aggregateID !== input.aggregateID) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: "Imported snapshot aggregate scope conflicts" })) + return { deletedRows: 0, complete: true } + } + const attempt = yield* db + .select({ aggregateID: EventSnapshotAttemptTable.aggregate_id }) + .from(EventSnapshotAttemptTable) + .where(eq(EventSnapshotAttemptTable.snapshot_id, input.snapshotID)) + .get() + .pipe(Effect.orDie) + if (attempt) { + if (attempt.aggregateID !== input.aggregateID) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: "Imported snapshot attempt scope conflicts" })) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: "Imported snapshot has a checkpoint authority" })) + } + const scopes = yield* db + .select({ aggregateID: EventSnapshotRowTable.aggregate_id }) + .from(EventSnapshotRowTable) + .where(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID)) + .all() + .pipe(Effect.orDie) + if (scopes.some((row) => row.aggregateID !== input.aggregateID)) + return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: "Imported snapshot row scope conflicts" })) + const rows = yield* db + .select({ rowIndex: EventSnapshotRowTable.row_index, rowHash: EventSnapshotRowTable.row_hash }) + .from(EventSnapshotRowTable) + .where(and(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID), eq(EventSnapshotRowTable.aggregate_id, input.aggregateID))) + .orderBy(asc(EventSnapshotRowTable.row_index)) + .limit(limit) + .all() + .pipe(Effect.orDie) + if (rows.length > 0) { + yield* db.delete(EventSnapshotRowTable).where(and( + eq(EventSnapshotRowTable.snapshot_id, input.snapshotID), + eq(EventSnapshotRowTable.aggregate_id, input.aggregateID), + inArray(EventSnapshotRowTable.row_index, rows.map((row) => row.rowIndex)), + )).run().pipe(Effect.orDie) + yield* Effect.forEach(rows, (row) => db.delete(EventSnapshotChunkTable).where(and( + eq(EventSnapshotChunkTable.row_hash, row.rowHash), + sql`NOT EXISTS (SELECT 1 FROM ${EventSnapshotRowTable} WHERE ${EventSnapshotRowTable.row_hash} = ${row.rowHash})`, + )).run().pipe(Effect.orDie), { discard: true }) + } + const remaining = yield* db.select({ rowIndex: EventSnapshotRowTable.row_index }) + .from(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID)).limit(1).get().pipe(Effect.orDie) + return { deletedRows: rows.length, complete: !remaining } + }), + { behavior: "immediate" }, + ).pipe(Effect.orDie) + } + function importSnapshot(input: SerializedSnapshot) { return Effect.gen(function* () { const codec = snapshotCodecs.get(`${input.codec}@${input.schemaVersion}`) @@ -1476,6 +1574,8 @@ export const layerWith = (layerOptions?: LayerOptions) => message: `Snapshot ${input.snapshotID} manifest does not match its staged rows`, }), ) + const previousSnapshot = yield* db.select({ snapshotID: EventSequenceTable.snapshot_id }) + .from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, input.aggregateID)).get().pipe(Effect.orDie) yield* db .insert(EventSequenceTable) .values({ @@ -1544,6 +1644,11 @@ export const layerWith = (layerOptions?: LayerOptions) => .where(eq(EventSequenceTable.aggregate_id, input.aggregateID)) .run() .pipe(Effect.orDie) + if (previousSnapshot?.snapshotID && previousSnapshot.snapshotID !== input.snapshotID) { + yield* db.delete(EventSnapshotTable).where(eq(EventSnapshotTable.snapshot_id, previousSnapshot.snapshotID)).run().pipe(Effect.orDie) + yield* db.delete(EventSnapshotAttemptTable).where(eq(EventSnapshotAttemptTable.snapshot_id, previousSnapshot.snapshotID)).run().pipe(Effect.orDie) + yield* db.delete(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.snapshot_id, previousSnapshot.snapshotID)).run().pipe(Effect.orDie) + } }), { behavior: "immediate" }, ) @@ -1601,10 +1706,35 @@ export const layerWith = (layerOptions?: LayerOptions) => return db.transaction(() => Effect.gen(function* () { if (rows.length > SNAPSHOT_TRANSFER_ROWS) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot row page exceeds ${SNAPSHOT_TRANSFER_ROWS}` })) - if (hashJson(Schema.decodeUnknownSync(SnapshotManifest)(input.body)) !== input.snapshotHash) + const manifest = Schema.decodeUnknownSync(SnapshotManifest)(input.body) + if (Buffer.byteLength(JSON.stringify(input.body)) > SNAPSHOT_MAX_ENCODED_BYTES || + manifest.rowCount > SNAPSHOT_MAX_ROWS || manifest.encodedBytes > SNAPSHOT_MAX_TOTAL_BYTES || + Object.values(manifest.tables).reduce((total, count) => total + count, 0) !== manifest.rowCount) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} manifest exceeds admission limits` }), + ) + if (hashJson(manifest) !== input.snapshotHash) return yield* Effect.die( new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} hash mismatch` }), ) + const previous = yield* db.select({ rowIndex: EventSnapshotRowTable.row_index }) + .from(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID)) + .orderBy(sql`${EventSnapshotRowTable.row_index} DESC`).limit(1).get().pipe(Effect.orDie) + const nextRowIndex = (previous?.rowIndex ?? -1) + 1 + if (rows.some((row, index) => + !Number.isSafeInteger(row.rowIndex) || row.rowIndex < 0 || row.rowIndex >= manifest.rowCount || + row.rowIndex !== (rows[0]?.rowIndex ?? 0) + index || (rows[0]?.rowIndex ?? 0) > nextRowIndex || + row.rowBytes <= 0 || row.rowBytes > SNAPSHOT_ROW_MAX_ENCODED_BYTES || + row.chunkCount !== Math.ceil(row.rowBytes / SNAPSHOT_CHUNK_BYTES) || + row.chunkCount > Math.ceil(SNAPSHOT_ROW_MAX_ENCODED_BYTES / SNAPSHOT_CHUNK_BYTES) || + row.tableName.length === 0 || Buffer.byteLength(row.tableName) > 128 || + row.rowKey.length === 0 || Buffer.byteLength(row.rowKey) > 4_096 || + row.rowHash.length !== 64 || row.chainHash.length !== 64 || + manifest.tables[row.tableName] === undefined + )) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} row page is not a bounded contiguous prefix` }), + ) yield* Effect.forEach( rows, (row) => Effect.gen(function* () { @@ -1616,6 +1746,7 @@ export const layerWith = (layerOptions?: LayerOptions) => .insert(EventSnapshotRowTable) .values({ snapshot_id: row.snapshotID, + aggregate_id: input.aggregateID, row_index: row.rowIndex, table_name: row.tableName, row_key: row.rowKey, @@ -1639,6 +1770,24 @@ export const layerWith = (layerOptions?: LayerOptions) => }), { discard: true }, ) + const totals = yield* db.select({ + count: sql`COUNT(*)`, + bytes: sql`COALESCE(SUM(${EventSnapshotRowTable.row_bytes}), 0)`, + }).from(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID)) + .get().pipe(Effect.orDie) + if ((totals?.count ?? 0) > manifest.rowCount || (totals?.bytes ?? 0) > manifest.encodedBytes) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} rows exceed the manifest authority` }), + ) + const tableCounts = yield* db.select({ + tableName: EventSnapshotRowTable.table_name, + count: sql`COUNT(*)`, + }).from(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.snapshot_id, input.snapshotID)) + .groupBy(EventSnapshotRowTable.table_name).all().pipe(Effect.orDie) + if (tableCounts.some((table) => table.count > (manifest.tables[table.tableName] ?? -1))) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} table counts exceed the manifest authority` }), + ) }), { behavior: "immediate" }).pipe(Effect.orDie) } @@ -1650,17 +1799,45 @@ export const layerWith = (layerOptions?: LayerOptions) => return db.transaction(() => Effect.gen(function* () { if (chunks.length > SNAPSHOT_TRANSFER_CHUNKS) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Snapshot chunk page exceeds ${SNAPSHOT_TRANSFER_CHUNKS}` })) - if (row.snapshotID !== input.snapshotID || chunks.some((chunk) => chunk.rowHash !== row.rowHash)) + const manifest = Schema.decodeUnknownSync(SnapshotManifest)(input.body) + if (Buffer.byteLength(JSON.stringify(input.body)) > SNAPSHOT_MAX_ENCODED_BYTES || + manifest.rowCount > SNAPSHOT_MAX_ROWS || manifest.encodedBytes > SNAPSHOT_MAX_TOTAL_BYTES || + hashJson(manifest) !== input.snapshotHash) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: `Snapshot ${input.snapshotID} manifest is invalid` }), + ) + const durableRow = yield* db.select().from(EventSnapshotRowTable).where(and( + eq(EventSnapshotRowTable.snapshot_id, input.snapshotID), + eq(EventSnapshotRowTable.row_index, row.rowIndex), + )).get().pipe(Effect.orDie) + if (!durableRow || durableRow.aggregate_id !== input.aggregateID || row.snapshotID !== input.snapshotID || + durableRow.table_name !== row.tableName || durableRow.row_key !== row.rowKey || + durableRow.row_hash !== row.rowHash || durableRow.row_bytes !== row.rowBytes || + durableRow.chunk_count !== row.chunkCount || durableRow.chain_hash !== row.chainHash || + chunks.some((chunk) => chunk.rowHash !== row.rowHash)) + return yield* Effect.die( + new InvalidSyncEventError({ type: "snapshot", message: "Snapshot chunk row authority mismatch" }), + ) + const previous = yield* db.select({ chunkIndex: EventSnapshotChunkTable.chunk_index }) + .from(EventSnapshotChunkTable).where(eq(EventSnapshotChunkTable.row_hash, row.rowHash)) + .orderBy(sql`${EventSnapshotChunkTable.chunk_index} DESC`).limit(1).get().pipe(Effect.orDie) + const nextChunkIndex = (previous?.chunkIndex ?? -1) + 1 + if (chunks.some((chunk, index) => { + const expectedIndex = (chunks[0]?.chunkIndex ?? 0) + index + const expectedBytes = chunk.chunkIndex === row.chunkCount - 1 + ? row.rowBytes - SNAPSHOT_CHUNK_BYTES * (row.chunkCount - 1) + : SNAPSHOT_CHUNK_BYTES + return !Number.isSafeInteger(chunk.chunkIndex) || chunk.chunkIndex < 0 || + chunk.chunkIndex >= row.chunkCount || chunk.chunkIndex !== expectedIndex || + (chunks[0]?.chunkIndex ?? 0) > nextChunkIndex || chunk.data.length !== expectedBytes || + chunk.chunkHash.length !== 64 || Hash.sha256(chunk.data) !== chunk.chunkHash + })) return yield* Effect.die( - new InvalidSyncEventError({ type: "snapshot", message: "Snapshot chunk identity mismatch" }), + new InvalidSyncEventError({ type: "snapshot", message: "Snapshot chunk page is not a bounded contiguous prefix" }), ) yield* Effect.forEach( chunks, (chunk) => Effect.gen(function* () { - if (chunk.data.length > SNAPSHOT_CHUNK_BYTES || Hash.sha256(chunk.data) !== chunk.chunkHash) - return yield* Effect.die( - new InvalidSyncEventError({ type: "snapshot", message: "Snapshot chunk hash mismatch" }), - ) const inserted = yield* db .insert(EventSnapshotChunkTable) .values({ @@ -1720,21 +1897,70 @@ export const layerWith = (layerOptions?: LayerOptions) => const codec = active ? snapshotCodecs.get(`${active.codec}@${active.schema_version}`) : undefined if (!active || !codec || active.through_seq < input.throughSeq) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Compaction requires an active importable snapshot for ${input.aggregateID}` })) - const receipt = yield* db.select().from(EventCompactionReceiptTable) + const existingReceipt = yield* db.select().from(EventCompactionReceiptTable) .where(eq(EventCompactionReceiptTable.aggregate_id, input.aggregateID)).get().pipe(Effect.orDie) - if (receipt && (receipt.snapshot_id !== active.snapshot_id || receipt.through_seq !== input.throughSeq || - receipt.codec !== active.codec || receipt.schema_version !== active.schema_version)) - return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Compaction receipt conflicts with active snapshot ${active.snapshot_id}` })) + const receipt = yield* Effect.gen(function* () { + if (!existingReceipt) return undefined + if (existingReceipt.snapshot_id === active.snapshot_id && + existingReceipt.through_seq === input.throughSeq && existingReceipt.codec === active.codec && + existingReceipt.schema_version === active.schema_version) return existingReceipt + if (existingReceipt.state === "complete" && existingReceipt.through_seq === input.throughSeq && + active.through_seq === input.throughSeq) { + const rebound = yield* db.update(EventCompactionReceiptTable).set({ + snapshot_id: active.snapshot_id, + codec: active.codec, + schema_version: active.schema_version, + cursor_seq: input.throughSeq, + state: "complete", + updated_at: input.now ?? Date.now(), + }).where(and( + eq(EventCompactionReceiptTable.aggregate_id, input.aggregateID), + eq(EventCompactionReceiptTable.snapshot_id, existingReceipt.snapshot_id), + eq(EventCompactionReceiptTable.through_seq, existingReceipt.through_seq), + eq(EventCompactionReceiptTable.state, "complete"), + )).returning().get().pipe(Effect.orDie) + if (!rebound) return yield* Effect.die(new InvalidSyncEventError({ + type: "snapshot", + message: `Compaction receipt changed before snapshot ${active.snapshot_id} could rebind`, + })) + return rebound + } + if (existingReceipt.state !== "complete" || existingReceipt.through_seq >= input.throughSeq) + return yield* Effect.die(new InvalidSyncEventError({ + type: "snapshot", + message: `Compaction receipt conflicts with active snapshot ${active.snapshot_id}`, + })) + const continued = yield* db.update(EventCompactionReceiptTable).set({ + snapshot_id: active.snapshot_id, + through_seq: input.throughSeq, + codec: active.codec, + schema_version: active.schema_version, + cursor_seq: existingReceipt.through_seq, + deleted_count: 0, + state: "running", + updated_at: input.now ?? Date.now(), + }).where(and( + eq(EventCompactionReceiptTable.aggregate_id, input.aggregateID), + eq(EventCompactionReceiptTable.snapshot_id, existingReceipt.snapshot_id), + eq(EventCompactionReceiptTable.through_seq, existingReceipt.through_seq), + eq(EventCompactionReceiptTable.state, "complete"), + )).returning().get().pipe(Effect.orDie) + if (!continued) return yield* Effect.die(new InvalidSyncEventError({ + type: "snapshot", + message: `Compaction receipt changed before snapshot ${active.snapshot_id} could continue`, + })) + return continued + }) const cursor = receipt?.cursor_seq ?? -1 const limit = Math.min(Math.max(input.limit ?? 100, 1), 100) - const candidates = yield* db.all<{ + const rows = yield* db.all<{ id: ID aggregateID: string seq: number type: string effectiveBytes: number cumulativeBytes: number - canonicalData: string | null + candidateCount: number }>(sql` WITH candidate AS ( SELECT @@ -1742,7 +1968,6 @@ export const layerWith = (layerOptions?: LayerOptions) => event.aggregate_id AS aggregateID, event.seq, event.type, - COALESCE(file_binding.canonical_data, artifact.canonical_data) AS canonicalData, length(CAST(COALESCE(file_binding.canonical_data, artifact.canonical_data, event.data) AS BLOB)) AS effectiveBytes FROM event LEFT JOIN event_artifact artifact ON artifact.event_id = event.id @@ -1752,16 +1977,20 @@ export const layerWith = (layerOptions?: LayerOptions) => AND event.seq <= ${input.throughSeq} ORDER BY event.seq ASC LIMIT ${limit + 1} + ), scored AS ( + SELECT *, + row_number() OVER (ORDER BY seq ASC) AS position, + sum(effectiveBytes) OVER ( + ORDER BY seq ASC ROWS UNBOUNDED PRECEDING + ) AS cumulativeBytes, + count(*) OVER () AS candidateCount + FROM candidate ) - SELECT *, sum(effectiveBytes) OVER ( - ORDER BY seq ASC ROWS UNBOUNDED PRECEDING - ) AS cumulativeBytes - FROM candidate + SELECT id, aggregateID, seq, type, effectiveBytes, cumulativeBytes, candidateCount + FROM scored + WHERE position = 1 OR (position <= ${limit} AND cumulativeBytes <= ${AGGREGATE_READ_BATCH_BYTES}) ORDER BY seq ASC `).pipe(Effect.orDie) - const rows = candidates - .slice(0, limit) - .filter((item, index) => index === 0 || item.cumulativeBytes <= AGGREGATE_READ_BATCH_BYTES) if (rows[0] && rows[0].effectiveBytes > AGGREGATE_READ_BATCH_BYTES) return yield* Effect.die(new EncodedPayloadTooLargeError({ type: rows[0].type, @@ -1770,7 +1999,24 @@ export const layerWith = (layerOptions?: LayerOptions) => message: `Compaction source ${rows[0].id} has no bounded canonical projection`, })) const compactable = rows.filter((item) => codec.rebuildEventTypes.has(item.type)) - const rawIDs = compactable.filter((item) => item.canonicalData === null).map((item) => item.id) + const canonicalRows = compactable.length > 0 + ? yield* db.all<{ + id: ID + data: string | null + originalDataHash: string | null + }>(sql` + SELECT event.id, + COALESCE(file_binding.canonical_data, artifact.canonical_data) AS data, + COALESCE(file_binding.original_data_hash, artifact.original_data_hash) AS originalDataHash + FROM event + LEFT JOIN event_artifact artifact ON artifact.event_id = event.id + LEFT JOIN file_part_artifact_binding file_binding ON file_binding.event_id = event.id + WHERE event.id IN ${compactable.map((item) => item.id)} + `).pipe(Effect.orDie) + : [] + const canonicalByID = new Map(canonicalRows.map((row) => [row.id, row.data])) + const originalDataHashByID = new Map(canonicalRows.map((row) => [row.id, row.originalDataHash])) + const rawIDs = compactable.filter((item) => canonicalByID.get(item.id) === null).map((item) => item.id) const rawRows = rawIDs.length > 0 ? yield* db .select({ id: EventTable.id, data: EventTable.data }) @@ -1781,8 +2027,10 @@ export const layerWith = (layerOptions?: LayerOptions) => : [] const rawByID = new Map(rawRows.map((row) => [row.id, row.data])) yield* Effect.forEach(compactable, (item) => Effect.gen(function* () { - const dataHash = item.canonicalData - ? hashJson(Schema.decodeUnknownSync(Schema.UnknownFromJsonString)(item.canonicalData)) + const canonicalData = canonicalByID.get(item.id) + const originalDataHash = originalDataHashByID.get(item.id) + const dataHash = originalDataHash + ? originalDataHash : rawByID.has(item.id) ? hashJson(rawByID.get(item.id)) : yield* Effect.die(new InvalidSyncEventError({ @@ -1796,7 +2044,16 @@ export const layerWith = (layerOptions?: LayerOptions) => if (existing && (existing.aggregate_id !== item.aggregateID || existing.seq !== item.seq || existing.event_id !== item.id || existing.type !== item.type || existing.data_hash !== dataHash)) return yield* Effect.die(new InvalidSyncEventError({ type: "snapshot", message: `Compaction dedupe conflicts at ${item.aggregateID}:${item.seq}` })) - if (!existing) yield* db.run(sql` + if (!existing && originalDataHash) yield* db.insert(EventDedupeTable).values({ + aggregate_id: item.aggregateID, + seq: item.seq, + event_id: item.id, + type: item.type, + data_hash: dataHash, + source_data: null, + compacted_at: input.now ?? Date.now(), + }).run().pipe(Effect.orDie) + if (!existing && !originalDataHash) yield* db.run(sql` INSERT INTO event_dedupe ( aggregate_id, seq, event_id, type, data_hash, source_data, compacted_at ) @@ -1806,13 +2063,14 @@ export const layerWith = (layerOptions?: LayerOptions) => }), { discard: true }) if (compactable.length > 0) yield* db.delete(EventTable) .where(inArray(EventTable.id, compactable.map((item) => item.id))).run().pipe(Effect.orDie) - if (compactable.length > 0) yield* db.update(EventDedupeTable) + if (rawIDs.length > 0) yield* db.update(EventDedupeTable) .set({ source_data: null }) - .where(inArray(EventDedupeTable.event_id, compactable.map((item) => item.id))) + .where(inArray(EventDedupeTable.event_id, rawIDs)) .run() .pipe(Effect.orDie) const nextCursor = rows.at(-1)?.seq ?? cursor - const complete = rows.length === 0 || (nextCursor >= input.throughSeq && candidates.length <= rows.length) + const complete = rows.length === 0 || + (nextCursor >= input.throughSeq && (rows[0]?.candidateCount ?? 0) <= rows.length) yield* db.insert(EventCompactionReceiptTable).values({ aggregate_id: input.aggregateID, snapshot_id: active.snapshot_id, through_seq: input.throughSeq, codec: active.codec, schema_version: active.schema_version, cursor_seq: nextCursor, @@ -1985,9 +2243,28 @@ export const layerWith = (layerOptions?: LayerOptions) => return db .transaction(() => Effect.gen(function* () { + const artifactIDs = yield* db + .selectDistinct({ id: FilePartArtifactBindingTable.artifact_id }) + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.aggregate_id, aggregateID)) + .all() + .pipe(Effect.orDie) yield* db.delete(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, aggregateID)).run() - yield* db.delete(EventTable).where(eq(EventTable.aggregate_id, aggregateID)).run() + if (artifactIDs.length > 0) + yield* db.run(sql` + DELETE FROM file_part_artifact + WHERE artifact_id IN ${artifactIDs.map((row) => row.id)} + AND NOT EXISTS ( + SELECT 1 FROM file_part_artifact_binding binding + WHERE binding.artifact_id = file_part_artifact.artifact_id + ) + AND NOT EXISTS ( + SELECT 1 FROM file_part_artifact_import imported + WHERE imported.artifact_id = file_part_artifact.artifact_id + ) + `).pipe(Effect.orDie) }), + { behavior: "immediate" }, ) .pipe(Effect.orDie) } @@ -2338,10 +2615,12 @@ export const layerWith = (layerOptions?: LayerOptions) => replay, replayAll, snapshot, + aggregateState, prepareCheckpoint, stageCheckpoint, finalizeCheckpoint, discardCheckpoint, + discardImportedSnapshot, checkpoint, importSnapshot, compact, diff --git a/packages/core/src/event/sql.ts b/packages/core/src/event/sql.ts index 0bda2c21f..e9836b7b5 100644 --- a/packages/core/src/event/sql.ts +++ b/packages/core/src/event/sql.ts @@ -45,7 +45,7 @@ export const EventSnapshotTable = sqliteTable( created_at: integer().notNull(), }, (table) => [ - uniqueIndex("event_snapshot_aggregate_seq_idx").on(table.aggregate_id, table.through_seq), + index("event_snapshot_aggregate_seq_idx").on(table.aggregate_id, table.through_seq), index("event_snapshot_aggregate_created_idx").on(table.aggregate_id, table.created_at), uniqueIndex("event_snapshot_sync_seq_idx").on(table.sync_seq), ], @@ -55,6 +55,7 @@ export const EventSnapshotRowTable = sqliteTable( "event_snapshot_row", { snapshot_id: text().notNull(), + aggregate_id: text().notNull(), row_index: integer().notNull(), table_name: text().notNull(), row_key: text().notNull(), @@ -66,6 +67,8 @@ export const EventSnapshotRowTable = sqliteTable( (table) => [ primaryKey({ columns: [table.snapshot_id, table.row_index] }), uniqueIndex("event_snapshot_row_identity_idx").on(table.snapshot_id, table.table_name, table.row_key), + index("event_snapshot_row_hash_idx").on(table.row_hash), + index("event_snapshot_row_aggregate_idx").on(table.aggregate_id), ], ) @@ -82,7 +85,9 @@ export const EventSnapshotChunkTable = sqliteTable( export const EventSnapshotAttemptTable = sqliteTable("event_snapshot_attempt", { snapshot_id: text().primaryKey(), - aggregate_id: text().notNull(), + aggregate_id: text() + .notNull() + .references(() => EventSequenceTable.aggregate_id, { onDelete: "cascade" }), through_seq: integer().notNull(), expected_latest: integer().notNull(), owner_id: text(), @@ -97,7 +102,7 @@ export const EventSnapshotAttemptTable = sqliteTable("event_snapshot_attempt", { state: text().$type<"prepared" | "staged" | "complete">().notNull(), created_at: integer().notNull(), updated_at: integer().notNull(), -}) +}, (table) => [index("event_snapshot_attempt_aggregate_idx").on(table.aggregate_id)]) export const EventSyncSequenceTable = sqliteTable("event_sync_sequence", { id: integer().primaryKey(), diff --git a/packages/core/src/file-part-artifact.sql.ts b/packages/core/src/file-part-artifact.sql.ts index 4baef7992..098905c94 100644 --- a/packages/core/src/file-part-artifact.sql.ts +++ b/packages/core/src/file-part-artifact.sql.ts @@ -46,6 +46,7 @@ export const FilePartArtifactBindingTable = sqliteTable( (table) => [ uniqueIndex("file_part_artifact_binding_aggregate_seq_idx").on(table.aggregate_id, table.seq), index("file_part_artifact_binding_part_idx").on(table.aggregate_id, table.part_id, table.seq), + index("file_part_artifact_binding_artifact_idx").on(table.artifact_id), ], ) @@ -63,5 +64,25 @@ export const FilePartArtifactImportTable = sqliteTable( canonical_data: text({ mode: "json" }).$type>().notNull(), created_at: integer().notNull(), }, - (table) => [uniqueIndex("file_part_artifact_import_aggregate_seq_idx").on(table.aggregate_id, table.seq)], + (table) => [ + uniqueIndex("file_part_artifact_import_aggregate_seq_idx").on(table.aggregate_id, table.seq), + index("file_part_artifact_import_artifact_idx").on(table.artifact_id), + ], +) + +export const FilePartArtifactDiscardTable = sqliteTable( + "file_part_artifact_discard", + { + event_id: text().$type().primaryKey(), + aggregate_id: text().notNull(), + seq: integer().notNull(), + artifact_id: text() + .notNull() + .references(() => FilePartArtifactTable.artifact_id, { onDelete: "cascade" }), + original_data_hash: text().notNull(), + canonical_data_hash: text().notNull(), + canonical_data: text({ mode: "json" }).$type>().notNull(), + created_at: integer().notNull(), + }, + (table) => [uniqueIndex("file_part_artifact_discard_aggregate_seq_idx").on(table.aggregate_id, table.seq)], ) diff --git a/packages/core/src/file-part-artifact.ts b/packages/core/src/file-part-artifact.ts index 49f9ff8d9..2083cf5d1 100644 --- a/packages/core/src/file-part-artifact.ts +++ b/packages/core/src/file-part-artifact.ts @@ -9,6 +9,7 @@ import type { EventV2 } from "./event" import { FilePartArtifactBindingTable, FilePartArtifactChunkTable, + FilePartArtifactDiscardTable, FilePartArtifactImportTable, FilePartArtifactTable, } from "./file-part-artifact.sql" @@ -488,6 +489,7 @@ export function importChunk(input: { }) .run() .pipe(Effect.orDie) + yield* stageReceipt(db, input.metadata) const storedChunk = yield* db .select({ hash: FilePartArtifactChunkTable.chunk_hash, data: FilePartArtifactChunkTable.data }) .from(FilePartArtifactChunkTable) @@ -537,6 +539,65 @@ export function stageImport(metadata: Metadata) { ) } +export function discardImport(input: { + readonly eventID: EventV2.ID + readonly aggregateID: string + readonly artifactID: ID +}) { + return Database.Service.use(({ db }) => + db.transaction( + () => + Effect.gen(function* () { + const imported = yield* db + .select() + .from(FilePartArtifactImportTable) + .where( + and( + eq(FilePartArtifactImportTable.event_id, input.eventID), + eq(FilePartArtifactImportTable.aggregate_id, input.aggregateID), + eq(FilePartArtifactImportTable.artifact_id, input.artifactID), + ), + ) + .get() + .pipe(Effect.orDie) + if (!imported) return false + const binding = yield* db + .select({ id: FilePartArtifactBindingTable.event_id }) + .from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.event_id, input.eventID)) + .get() + .pipe(Effect.orDie) + if (binding) return yield* fail(input.artifactID, "Bound artifact imports cannot be discarded") + yield* db.insert(FilePartArtifactDiscardTable).values({ + event_id: imported.event_id, + aggregate_id: imported.aggregate_id, + seq: imported.seq, + artifact_id: imported.artifact_id, + original_data_hash: imported.original_data_hash, + canonical_data_hash: imported.canonical_data_hash, + canonical_data: imported.canonical_data, + created_at: Date.now(), + }).onConflictDoNothing().run().pipe(Effect.orDie) + yield* db.delete(FilePartArtifactImportTable) + .where(eq(FilePartArtifactImportTable.event_id, input.eventID)).run().pipe(Effect.orDie) + yield* db.delete(FilePartArtifactTable).where(and( + eq(FilePartArtifactTable.artifact_id, input.artifactID), + sql`NOT EXISTS ( + SELECT 1 FROM ${FilePartArtifactBindingTable} binding + WHERE binding.artifact_id = ${input.artifactID} + )`, + sql`NOT EXISTS ( + SELECT 1 FROM ${FilePartArtifactImportTable} imported + WHERE imported.artifact_id = ${input.artifactID} + )`, + )).run().pipe(Effect.orDie) + return true + }), + { behavior: "immediate" }, + ).pipe(Effect.orDie), + ) +} + export function read(input: { readonly aggregateID: string; readonly descriptor: Descriptor }) { return Database.Service.use(({ db }) => Effect.gen(function* () { @@ -614,6 +675,28 @@ export function bindSnapshotRef(input: { readonly metadata: Metadata; readonly p Effect.gen(function* () { if (!input.partID.startsWith("prt")) return yield* fail(input.metadata.descriptor.id, "Snapshot artifact part identity is invalid") + const canonicalBinding = descriptorFromData("message.part.updated", input.metadata.canonicalData) + if ( + !canonicalBinding || + canonicalBinding.aggregateID !== input.metadata.aggregateID || + canonicalBinding.partID !== input.partID || + !isDeepStrictEqual(canonicalBinding.descriptor, input.metadata.descriptor) + ) + return yield* fail(input.metadata.descriptor.id, "Snapshot artifact canonical data does not match its descriptor") + const projected = yield* db + .select() + .from(PartTable) + .where(eq(PartTable.id, input.partID as typeof PartTable.$inferSelect.id)) + .get() + .pipe(Effect.orDie) + if (!projected || projected.session_id !== input.metadata.aggregateID || projected.message_id !== (input.metadata.canonicalData.part as Record).messageID) + return yield* fail(input.metadata.descriptor.id, "Snapshot artifact has no matching projected part") + const canonicalPart = input.metadata.canonicalData.part as Record + const projectedData = Object.fromEntries( + Object.entries(canonicalPart).filter(([key]) => key !== "id" && key !== "sessionID" && key !== "messageID"), + ) + if (!isDeepStrictEqual(projected.data, projectedData)) + return yield* fail(input.metadata.descriptor.id, "Snapshot artifact projected part diverges from its binding") yield* stage(db, input.metadata) const existing = yield* db .select() @@ -851,6 +934,15 @@ function stage(db: Database.Interface["db"], metadata: Metadata) { ) return yield* fail(metadata.descriptor.id, "Artifact import metadata does not match durable chunks") yield* validate(metadata.descriptor, chunks.map((chunk) => chunk.data)) + yield* stageReceipt(db, metadata) + }) +} + +function stageReceipt(db: Database.Interface["db"], metadata: Metadata) { + return Effect.gen(function* () { + if (!matchesDataHash(metadata.canonicalDataHash, metadata.canonicalData)) + return yield* fail(metadata.descriptor.id, "Artifact import metadata canonical hash is invalid") + const canonicalData = JSON.parse(CanonicalJson.stringify(metadata.canonicalData)) as Record const existing = yield* db .select() .from(FilePartArtifactImportTable) diff --git a/packages/core/src/session/projector.ts b/packages/core/src/session/projector.ts index dffa414a6..b42496d49 100644 --- a/packages/core/src/session/projector.ts +++ b/packages/core/src/session/projector.ts @@ -8,6 +8,7 @@ import { makeGlobalNode } from "../effect/app-node" import { EventV2 } from "../event" import { EventArtifactTable } from "../event/sql" import { FilePartArtifact } from "../file-part-artifact" +import { FilePartArtifactBindingTable } from "../file-part-artifact.sql" import { SessionEvent } from "./event" import { SessionV1 } from "../v1/session" import { WorkspaceTable } from "../control-plane/workspace.sql" @@ -318,7 +319,15 @@ export const layer = Layer.effectDiscard( .from(SessionTable).where(eq(SessionTable.id, SessionSchema.ID.make(aggregateID))).get().pipe(Effect.orDie) const context = yield* db.select({ revision: SessionContextEpochTable.revision }) .from(SessionContextEpochTable).where(eq(SessionContextEpochTable.session_id, SessionSchema.ID.make(aggregateID))).get().pipe(Effect.orDie) - return `${session?.mutationEpoch ?? -1}:${context?.revision ?? -1}` + const fileArtifacts = yield* db.select({ + eventID: FilePartArtifactBindingTable.event_id, + canonicalDataHash: FilePartArtifactBindingTable.canonical_data_hash, + }).from(FilePartArtifactBindingTable) + .where(eq(FilePartArtifactBindingTable.aggregate_id, aggregateID)) + .orderBy(asc(FilePartArtifactBindingTable.seq)).all().pipe(Effect.orDie) + return `${session?.mutationEpoch ?? -1}:${context?.revision ?? -1}:${fileArtifacts + .map((artifact) => `${artifact.eventID}:${artifact.canonicalDataHash}`) + .join(",")}` }), next: (aggregateID, cursor) => Effect.gen(function* () { diff --git a/packages/core/test/database-migration.test.ts b/packages/core/test/database-migration.test.ts index ef03d6d26..ebaed90d0 100644 --- a/packages/core/test/database-migration.test.ts +++ b/packages/core/test/database-migration.test.ts @@ -29,7 +29,10 @@ import partIntegrityBackfillMigration from "@deepagent-code/core/database/migrat import legacyActivityProgressMigration from "@deepagent-code/core/database/migration/20260811090000_legacy_activity_progress" import legacyActivityOwnerMigration from "@deepagent-code/core/database/migration/20260811100000_legacy_activity_owner" import eventSnapshotAuthorityMigration from "@deepagent-code/core/database/migration/20260813100000_event_snapshot_authority" +import filePartArtifactMigration from "@deepagent-code/core/database/migration/20260813130000_file_part_artifact" +import eventSnapshotChunksMigration from "@deepagent-code/core/database/migration/20260813131000_event_snapshot_chunks" import eventImmutableMigration from "@deepagent-code/core/database/migration/20260813135000_event_immutable" +import eventSidecarCompactionMigration from "@deepagent-code/core/database/migration/20260813140000_event_sidecar_compaction" import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" import { Database } from "@deepagent-code/core/database/database" import { tmpdir } from "./fixture/tmpdir" @@ -131,6 +134,353 @@ describe("DatabaseMigration", () => { }), ) }) + test("keeps compacted replay and snapshot authorities append-only", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`PRAGMA foreign_keys = ON`) + yield* db.run(sql`CREATE TABLE event_sequence (aggregate_id TEXT PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE event ( + id TEXT PRIMARY KEY, aggregate_id TEXT NOT NULL, seq INTEGER NOT NULL, + type TEXT NOT NULL, data TEXT NOT NULL + )`) + yield* db.run(sql`CREATE TABLE event_artifact ( + event_id TEXT PRIMARY KEY, aggregate_id TEXT NOT NULL, seq INTEGER NOT NULL, + original_data_hash TEXT NOT NULL + )`) + yield* db.run(sql`CREATE TABLE event_dedupe ( + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + seq INTEGER NOT NULL, event_id TEXT NOT NULL, type TEXT NOT NULL, + data_hash TEXT NOT NULL, source_data TEXT, compacted_at INTEGER NOT NULL + )`) + yield* db.run(sql`CREATE TABLE event_snapshot ( + snapshot_id TEXT PRIMARY KEY, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + snapshot_hash TEXT NOT NULL, body TEXT NOT NULL + )`) + yield* db.run(sql`CREATE TABLE file_part_artifact ( + artifact_id TEXT PRIMARY KEY, body_hash TEXT NOT NULL + )`) + yield* db.run(sql`CREATE TABLE file_part_artifact_binding ( + event_id TEXT PRIMARY KEY, aggregate_id TEXT NOT NULL, seq INTEGER NOT NULL, + artifact_id TEXT NOT NULL, original_data_hash TEXT NOT NULL, + canonical_data_hash TEXT NOT NULL, canonical_data TEXT NOT NULL + )`) + yield* db.run(sql`CREATE TABLE file_part_artifact_import ( + event_id TEXT PRIMARY KEY, aggregate_id TEXT NOT NULL, seq INTEGER NOT NULL, + artifact_id TEXT NOT NULL REFERENCES file_part_artifact(artifact_id) ON DELETE CASCADE, + original_data_hash TEXT NOT NULL, canonical_data_hash TEXT NOT NULL, + canonical_data TEXT NOT NULL, created_at INTEGER NOT NULL + )`) + yield* db.run(sql`CREATE TABLE file_part_artifact_discard ( + event_id TEXT PRIMARY KEY, aggregate_id TEXT NOT NULL, seq INTEGER NOT NULL, + artifact_id TEXT NOT NULL REFERENCES file_part_artifact(artifact_id) ON DELETE CASCADE, + original_data_hash TEXT NOT NULL, canonical_data_hash TEXT NOT NULL, + canonical_data TEXT NOT NULL, created_at INTEGER NOT NULL + )`) + yield* DatabaseMigration.applyOnly(db, [eventSidecarCompactionMigration]) + yield* db.run(sql`INSERT INTO event_sequence VALUES ('session-a')`) + yield* db.run(sql`INSERT INTO event_dedupe VALUES ( + 'session-a', 0, 'event-a', 'test.1', '${"a".repeat(64)}', '{}', 1 + )`) + yield* db.run(sql`INSERT INTO event_snapshot VALUES ('snapshot-a', 'session-a', '${"b".repeat(64)}', '{}')`) + + expect(String(yield* db.run(sql` + UPDATE event_dedupe SET data_hash = '${"c".repeat(64)}' WHERE event_id = 'event-a' + `).pipe(Effect.exit))).toContain("event_dedupe_update_immutable") + expect(String(yield* db.run(sql`DELETE FROM event_dedupe WHERE event_id = 'event-a'`).pipe(Effect.exit))) + .toContain("event_dedupe_delete_immutable") + yield* db.run(sql`UPDATE event_dedupe SET source_data = NULL WHERE event_id = 'event-a'`) + expect(String(yield* db.run(sql` + UPDATE event_snapshot SET snapshot_hash = '${"c".repeat(64)}' WHERE snapshot_id = 'snapshot-a' + `).pipe(Effect.exit))).toContain("event_snapshot_update_immutable") + + yield* db.run(sql`DELETE FROM event_sequence WHERE aggregate_id = 'session-a'`) + expect(yield* db.all(sql`SELECT * FROM event_dedupe`)).toEqual([]) + expect(yield* db.all(sql`SELECT * FROM event_snapshot`)).toEqual([]) + + yield* db.run(sql`INSERT INTO file_part_artifact VALUES ('file-a', '${"a".repeat(64)}')`) + yield* db.run(sql`INSERT INTO file_part_artifact_import VALUES ( + 'file-event-a', 'session-file', 1, 'file-a', '${"b".repeat(64)}', + '${"c".repeat(64)}', '{}', 1 + )`) + expect(String(yield* db.run(sql` + UPDATE file_part_artifact_import SET original_data_hash = '${"d".repeat(64)}' + `).pipe(Effect.exit))).toContain("file_part_artifact_import_update_immutable") + expect(String(yield* db.run(sql`DELETE FROM file_part_artifact_import`).pipe(Effect.exit))) + .toContain("file_part_artifact_import_not_consumed") + yield* db.run(sql`INSERT INTO file_part_artifact_binding VALUES ( + 'file-event-a', 'session-file', 1, 'file-a', '${"b".repeat(64)}', '${"c".repeat(64)}', '{}' + )`) + yield* db.run(sql`DELETE FROM file_part_artifact_import`) + yield* db.run(sql`INSERT INTO file_part_artifact_import VALUES ( + 'file-event-b', 'session-file', 2, 'file-a', '${"b".repeat(64)}', + '${"c".repeat(64)}', '{}', 1 + )`) + yield* db.run(sql`DELETE FROM file_part_artifact WHERE artifact_id = 'file-a'`) + expect(yield* db.all(sql`SELECT * FROM file_part_artifact_import`)).toEqual([]) + }), + ) + }) + test("upgrades the exact legacy snapshot candidate schema without losing rows or chunks", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`PRAGMA foreign_keys = ON`) + yield* db.run(sql`CREATE TABLE event_sequence ( + aggregate_id TEXT PRIMARY KEY, seq INTEGER NOT NULL, owner_id TEXT + )`) + yield* db.run(sql`CREATE TABLE event ( + id TEXT PRIMARY KEY, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + seq INTEGER NOT NULL, type TEXT NOT NULL, data TEXT NOT NULL + )`) + yield* DatabaseMigration.applyOnly(db, [ + eventSnapshotAuthorityMigration, + filePartArtifactMigration, + eventSnapshotChunksMigration, + ]) + yield* db.run(sql`DROP TABLE file_part_artifact_discard`) + yield* db.run(sql`INSERT INTO event_sequence(aggregate_id, seq) VALUES ('session-legacy-snapshot', 0)`) + yield* db.run(sql`INSERT INTO event_snapshot_attempt( + snapshot_id, aggregate_id, through_seq, expected_latest, owner_id, codec, schema_version, + projection_revision, cursor, row_count, encoded_bytes, content_hash, tables, state, created_at, updated_at + ) VALUES ( + 'snapshot-legacy', 'session-legacy-snapshot', 0, 0, NULL, 'session-projection', 1, + 'revision-legacy', NULL, 1, 2, ${"a".repeat(64)}, '{"session":1}', 'complete', 1, 1 + )`) + yield* db.run(sql`INSERT INTO event_snapshot( + snapshot_id, aggregate_id, through_seq, sync_seq, codec, schema_version, + snapshot_hash, body, owner_id, created_at + ) VALUES ( + 'snapshot-legacy', 'session-legacy-snapshot', 0, 1, 'session-projection', 1, + ${"b".repeat(64)}, '{}', NULL, 1 + )`) + yield* db.run(sql`INSERT INTO event_snapshot_chunk(row_hash, chunk_index, data, chunk_hash) + VALUES (${"c".repeat(64)}, 0, X'7B7D', ${"d".repeat(64)})`) + yield* db.run(sql`INSERT INTO event_snapshot_row( + snapshot_id, aggregate_id, row_index, table_name, row_key, + row_hash, row_bytes, chunk_count, chain_hash + ) VALUES ( + 'snapshot-legacy', 'session-legacy-snapshot', 0, 'session', 'session-legacy-snapshot', + ${"c".repeat(64)}, 2, 1, ${"e".repeat(64)} + )`) + yield* db.run(sql`UPDATE event_sequence SET retention_floor_seq = 0, snapshot_id = 'snapshot-legacy' + WHERE aggregate_id = 'session-legacy-snapshot'`) + + yield* db.run(sql`DROP TRIGGER event_sequence_snapshot_validate_update`) + yield* db.run(sql`DROP TRIGGER event_snapshot_delete_active_guard`) + yield* db.run(sql`ALTER TABLE event_snapshot RENAME TO event_snapshot_current`) + yield* db.run(sql`CREATE TABLE event_snapshot ( + snapshot_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + through_seq INTEGER NOT NULL CHECK (through_seq >= 0), + sync_seq INTEGER NOT NULL UNIQUE CHECK (sync_seq >= 0), + codec TEXT NOT NULL CHECK (length(codec) > 0), + schema_version INTEGER NOT NULL CHECK (schema_version >= 1), + snapshot_hash TEXT NOT NULL CHECK (length(snapshot_hash) = 64), + body TEXT NOT NULL CHECK (json_valid(body)), + owner_id TEXT, created_at INTEGER NOT NULL, + UNIQUE (aggregate_id, through_seq) + )`) + yield* db.run(sql`INSERT INTO event_snapshot SELECT * FROM event_snapshot_current`) + yield* db.run(sql`DROP TABLE event_snapshot_current`) + + yield* db.run(sql`DROP TRIGGER event_snapshot_row_immutable`) + yield* db.run(sql`DROP TRIGGER event_snapshot_row_delete_guard`) + yield* db.run(sql`DROP TRIGGER event_snapshot_row_chunk_cleanup`) + yield* db.run(sql`DROP TRIGGER event_snapshot_aggregate_cleanup`) + yield* db.run(sql`DROP TRIGGER event_snapshot_chunk_delete_guard`) + yield* db.run(sql`ALTER TABLE event_snapshot_row RENAME TO event_snapshot_row_current`) + yield* db.run(sql`CREATE TABLE event_snapshot_row ( + snapshot_id TEXT NOT NULL, + row_index INTEGER NOT NULL CHECK (row_index >= 0), + table_name TEXT NOT NULL CHECK (length(table_name) > 0), + row_key TEXT NOT NULL CHECK (length(row_key) > 0), + row_hash TEXT NOT NULL CHECK (length(row_hash) = 64), + row_bytes INTEGER NOT NULL CHECK (row_bytes > 0), + chunk_count INTEGER NOT NULL CHECK (chunk_count > 0), + chain_hash TEXT NOT NULL CHECK (length(chain_hash) = 64), + PRIMARY KEY (snapshot_id, row_index), + UNIQUE (snapshot_id, table_name, row_key) + )`) + yield* db.run(sql`INSERT INTO event_snapshot_row( + snapshot_id, row_index, table_name, row_key, row_hash, row_bytes, chunk_count, chain_hash + ) SELECT snapshot_id, row_index, table_name, row_key, row_hash, row_bytes, chunk_count, chain_hash + FROM event_snapshot_row_current`) + yield* db.run(sql`DROP TABLE event_snapshot_row_current`) + + yield* db.run(sql`ALTER TABLE event_snapshot_attempt RENAME TO event_snapshot_attempt_current`) + yield* db.run(sql`CREATE TABLE event_snapshot_attempt ( + snapshot_id TEXT NOT NULL PRIMARY KEY, + aggregate_id TEXT NOT NULL, + through_seq INTEGER NOT NULL CHECK (through_seq >= 0), + expected_latest INTEGER NOT NULL CHECK (expected_latest >= 0), + owner_id TEXT, + codec TEXT NOT NULL CHECK (length(codec) > 0), + schema_version INTEGER NOT NULL CHECK (schema_version > 0), + projection_revision TEXT NOT NULL, + cursor TEXT, + row_count INTEGER NOT NULL CHECK (row_count >= 0), + encoded_bytes INTEGER NOT NULL CHECK (encoded_bytes >= 0), + content_hash TEXT NOT NULL CHECK (length(content_hash) = 64), + tables TEXT NOT NULL CHECK (json_valid(tables)), + state TEXT NOT NULL CHECK (state IN ('prepared', 'staged', 'complete')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + )`) + yield* db.run(sql`INSERT INTO event_snapshot_attempt SELECT * FROM event_snapshot_attempt_current`) + yield* db.run(sql`DROP TABLE event_snapshot_attempt_current`) + yield* db.run(sql`INSERT INTO event_snapshot_attempt( + snapshot_id, aggregate_id, through_seq, expected_latest, codec, schema_version, + projection_revision, row_count, encoded_bytes, content_hash, tables, state, created_at, updated_at + ) VALUES ( + 'snapshot-orphan-attempt', 'session-missing', 0, 0, 'session-projection', 1, + 'revision-orphan', 1, 2, ${"f".repeat(64)}, '{}', 'staged', 1, 1 + )`) + yield* db.run(sql`INSERT INTO event_snapshot_chunk(row_hash, chunk_index, data, chunk_hash) + VALUES (${"4".repeat(64)}, 0, X'7B7D', ${"5".repeat(64)})`) + yield* db.run(sql`INSERT INTO event_snapshot_row( + snapshot_id, row_index, table_name, row_key, row_hash, row_bytes, chunk_count, chain_hash + ) VALUES ( + 'snapshot-orphan-attempt', 0, 'session', 'session-missing', + ${"4".repeat(64)}, 2, 1, ${"6".repeat(64)} + )`) + yield* db.run(sql`INSERT INTO event_snapshot_chunk(row_hash, chunk_index, data, chunk_hash) + VALUES (${"1".repeat(64)}, 0, X'7B7D', ${"2".repeat(64)})`) + yield* db.run(sql`INSERT INTO event_snapshot_row( + snapshot_id, row_index, table_name, row_key, row_hash, row_bytes, chunk_count, chain_hash + ) VALUES ( + 'snapshot-crash-staged', 0, 'session', 'session-crash-staged', + ${"1".repeat(64)}, 2, 1, ${"3".repeat(64)} + )`) + + yield* DatabaseMigration.applyOnly(db, [eventSidecarCompactionMigration]) + expect((yield* db.all<{ name: string }>(sql`PRAGMA table_info('event_snapshot_row')`)).map((row) => row.name)) + .toEqual([ + "snapshot_id", "aggregate_id", "row_index", "table_name", "row_key", + "row_hash", "row_bytes", "chunk_count", "chain_hash", + ]) + expect(yield* db.get(sql`SELECT aggregate_id, row_key, row_bytes FROM event_snapshot_row`)).toEqual({ + aggregate_id: "session-legacy-snapshot", + row_key: "session-legacy-snapshot", + row_bytes: 2, + }) + expect(yield* db.get(sql`SELECT hex(data) AS data, chunk_hash FROM event_snapshot_chunk`)).toEqual({ + data: "7B7D", + chunk_hash: "d".repeat(64), + }) + expect(yield* db.all(sql`SELECT * FROM event_snapshot_row WHERE snapshot_id = 'snapshot-crash-staged'`)) + .toEqual([]) + expect(yield* db.all(sql`SELECT * FROM event_snapshot_chunk WHERE row_hash = ${"1".repeat(64)}`)) + .toEqual([]) + expect(yield* db.all(sql`SELECT * FROM event_snapshot_attempt WHERE snapshot_id = 'snapshot-orphan-attempt'`)).toEqual([]) + expect(yield* db.all(sql`SELECT * FROM event_snapshot_row WHERE snapshot_id = 'snapshot-orphan-attempt'`)).toEqual([]) + expect(yield* db.all(sql`SELECT * FROM event_snapshot_chunk WHERE row_hash = ${"4".repeat(64)}`)).toEqual([]) + expect((yield* db.all<{ name: string; unique: number }>(sql`PRAGMA index_list('event_snapshot')`)) + .filter((index) => index.unique === 1)).toHaveLength(2) + expect(yield* db.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'file_part_artifact_discard'`)) + .toEqual({ name: "file_part_artifact_discard" }) + expect(yield* db.all(sql`PRAGMA foreign_key_list('event_snapshot_attempt')`)).toContainEqual( + expect.objectContaining({ table: "event_sequence", from: "aggregate_id", to: "aggregate_id", on_delete: "CASCADE" }), + ) + expect(yield* db.get(sql` + SELECT name FROM sqlite_master + WHERE type = 'index' AND name = 'event_snapshot_row_hash_idx' + `)).toEqual({ name: "event_snapshot_row_hash_idx" }) + expect(yield* db.get(sql` + SELECT name FROM sqlite_master + WHERE type = 'index' AND name = 'event_snapshot_row_aggregate_idx' + `)).toEqual({ name: "event_snapshot_row_aggregate_idx" }) + expect(yield* db.get(sql` + SELECT name FROM sqlite_master + WHERE type = 'index' AND name = 'event_snapshot_attempt_aggregate_idx' + `)).toEqual({ name: "event_snapshot_attempt_aggregate_idx" }) + expect((yield* db.run(sql`INSERT INTO event_snapshot_attempt( + snapshot_id, aggregate_id, through_seq, expected_latest, codec, schema_version, + projection_revision, row_count, encoded_bytes, content_hash, tables, state, created_at, updated_at + ) VALUES ( + 'snapshot-orphan', 'session-missing', 0, 0, 'session-projection', 1, + 'revision-orphan', 0, 0, ${"f".repeat(64)}, '{}', 'prepared', 1, 1 + )`).pipe(Effect.exit))._tag).toBe("Failure") + + yield* db.run(sql`DELETE FROM event_sequence WHERE aggregate_id = 'session-legacy-snapshot'`) + expect(yield* db.all(sql`SELECT * FROM event_snapshot`)).toEqual([]) + expect(yield* db.all(sql`SELECT * FROM event_snapshot_attempt`)).toEqual([]) + expect(yield* db.all(sql`SELECT * FROM event_snapshot_row`)).toEqual([]) + expect(yield* db.all(sql`SELECT * FROM event_snapshot_chunk`)).toEqual([]) + }), + ) + }) + test("rejects conflicting legacy snapshot authorities without mutating staged rows", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`PRAGMA foreign_keys = ON`) + yield* db.run(sql`CREATE TABLE event_sequence (aggregate_id TEXT PRIMARY KEY, seq INTEGER NOT NULL, owner_id TEXT)`) + yield* db.run(sql`CREATE TABLE event ( + id TEXT PRIMARY KEY, aggregate_id TEXT NOT NULL REFERENCES event_sequence(aggregate_id) ON DELETE CASCADE, + seq INTEGER NOT NULL, type TEXT NOT NULL, data TEXT NOT NULL + )`) + yield* DatabaseMigration.applyOnly(db, [ + eventSnapshotAuthorityMigration, + filePartArtifactMigration, + eventSnapshotChunksMigration, + ]) + yield* db.run(sql`DROP TABLE file_part_artifact_discard`) + yield* db.run(sql`INSERT INTO event_sequence(aggregate_id, seq) VALUES ('session-a', 0), ('session-b', 0)`) + yield* db.run(sql`INSERT INTO event_snapshot_attempt( + snapshot_id, aggregate_id, through_seq, expected_latest, codec, schema_version, + projection_revision, row_count, encoded_bytes, content_hash, tables, state, created_at, updated_at + ) VALUES ( + 'snapshot-conflict', 'session-a', 0, 0, 'session-projection', 1, + 'revision-conflict', 1, 2, ${"a".repeat(64)}, '{"session":1}', 'complete', 1, 1 + )`) + yield* db.run(sql`INSERT INTO event_snapshot( + snapshot_id, aggregate_id, through_seq, sync_seq, codec, schema_version, + snapshot_hash, body, created_at + ) VALUES ( + 'snapshot-conflict', 'session-b', 0, 1, 'session-projection', 1, + ${"b".repeat(64)}, '{}', 1 + )`) + yield* db.run(sql`INSERT INTO event_snapshot_chunk(row_hash, chunk_index, data, chunk_hash) + VALUES (${"c".repeat(64)}, 0, X'7B7D', ${"d".repeat(64)})`) + yield* db.run(sql`INSERT INTO event_snapshot_row( + snapshot_id, aggregate_id, row_index, table_name, row_key, + row_hash, row_bytes, chunk_count, chain_hash + ) VALUES ( + 'snapshot-conflict', 'session-a', 0, 'session', 'session-a', + ${"c".repeat(64)}, 2, 1, ${"e".repeat(64)} + )`) + yield* db.run(sql`DROP TRIGGER event_snapshot_row_immutable`) + yield* db.run(sql`DROP TRIGGER event_snapshot_row_delete_guard`) + yield* db.run(sql`DROP TRIGGER event_snapshot_row_chunk_cleanup`) + yield* db.run(sql`DROP TRIGGER event_snapshot_aggregate_cleanup`) + yield* db.run(sql`DROP TRIGGER event_snapshot_chunk_delete_guard`) + yield* db.run(sql`ALTER TABLE event_snapshot_row RENAME TO event_snapshot_row_current`) + yield* db.run(sql`CREATE TABLE event_snapshot_row ( + snapshot_id TEXT NOT NULL, row_index INTEGER NOT NULL, table_name TEXT NOT NULL, + row_key TEXT NOT NULL, row_hash TEXT NOT NULL, row_bytes INTEGER NOT NULL, + chunk_count INTEGER NOT NULL, chain_hash TEXT NOT NULL, + PRIMARY KEY (snapshot_id, row_index), UNIQUE (snapshot_id, table_name, row_key) + )`) + yield* db.run(sql`INSERT INTO event_snapshot_row + SELECT snapshot_id, row_index, table_name, row_key, row_hash, row_bytes, chunk_count, chain_hash + FROM event_snapshot_row_current`) + yield* db.run(sql`DROP TABLE event_snapshot_row_current`) + + const before = yield* db.get(sql`SELECT snapshot_id, row_key, row_hash FROM event_snapshot_row`) + const result = yield* DatabaseMigration.applyOnly(db, [eventSidecarCompactionMigration]).pipe(Effect.exit) + expect(result._tag).toBe("Failure") + expect(String(result)).toContain("conflicting aggregate authorities") + expect(yield* db.get(sql`SELECT snapshot_id, row_key, row_hash FROM event_snapshot_row`)).toEqual(before) + expect(yield* db.get(sql`SELECT hex(data) AS data FROM event_snapshot_chunk`)).toEqual({ data: "7B7D" }) + expect(yield* db.get(sql` + SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'file_part_artifact_discard' + `)).toBeUndefined() + }), + ) + }) test("serializes concurrent embedded initialization for one database path", async () => { await using tmp = await tmpdir() const filename = path.join(tmp.path, "embedded.sqlite") diff --git a/packages/core/test/event-snapshot.test.ts b/packages/core/test/event-snapshot.test.ts index 665655d1f..144f57a92 100644 --- a/packages/core/test/event-snapshot.test.ts +++ b/packages/core/test/event-snapshot.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import path from "node:path" -import { Effect, Layer } from "effect" +import { DateTime, Effect, Layer } from "effect" import { asc, eq } from "drizzle-orm" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" @@ -10,6 +10,9 @@ import { EventCompactionReceiptTable, EventDedupeTable, EventSequenceTable, + EventSnapshotAttemptTable, + EventSnapshotChunkTable, + EventSnapshotRowTable, EventSnapshotTable, EventTable, } from "@deepagent-code/core/event/sql" @@ -21,6 +24,7 @@ import { AbsolutePath } from "@deepagent-code/core/schema" import { SessionV2 } from "@deepagent-code/core/session" import { MessageTable, PartTable, SessionInputTable, SessionHistoryStateTable, SessionTable } from "@deepagent-code/core/session/sql" import { SessionMessage } from "@deepagent-code/core/session/message" +import { SessionEvent } from "@deepagent-code/core/session/event" import { SessionProjector } from "@deepagent-code/core/session/projector" import { SessionV1 } from "@deepagent-code/core/v1/session" import { Hash } from "@deepagent-code/core/util/hash" @@ -202,8 +206,29 @@ describe("EventV2 canonical projection snapshots", () => { yield* targetDb.insert(PartTable).values({ id: SessionV1.PartID.make("prt_target_stale"), message_id: SessionV1.MessageID.make("msg_target_stale"), session_id: sessionID, time_created: 1, data: { type: "text", text: "stale" } as typeof PartTable.$inferInsert["data"] }).run().pipe(Effect.orDie) + expect((yield* target.stageSnapshotChunks!(active, rows[0]!, chunks.get(rows[0]!.rowHash)!).pipe(Effect.exit))._tag) + .toBe("Failure") + expect(yield* targetDb.select().from(EventSnapshotChunkTable).all()).toEqual([]) + expect((yield* target.stageSnapshotRows!(active, [{ + ...rows[0]!, + rowIndex: rows.length, + }]).pipe(Effect.exit))._tag).toBe("Failure") + expect(yield* targetDb.select().from(EventSnapshotRowTable).all()).toEqual([]) + yield* target.stageSnapshotRows!(active, [rows[0]!]) + expect(yield* target.discardImportedSnapshot!({ + snapshotID: active.snapshotID, + aggregateID: active.aggregateID, + limit: 1, + })).toEqual({ deletedRows: 1, complete: true }) + expect(yield* targetDb.select().from(EventSnapshotRowTable).all()).toEqual([]) + expect(yield* targetDb.select().from(EventSnapshotChunkTable).all()).toEqual([]) yield* target.stageSnapshotRows!(active, rows) yield* target.stageSnapshotRows!(active, rows) + expect((yield* target.stageSnapshotChunks!(active, rows[0]!, [{ + ...chunks.get(rows[0]!.rowHash)![0]!, + chunkIndex: rows[0]!.chunkCount, + }]).pipe(Effect.exit))._tag).toBe("Failure") + expect(yield* targetDb.select().from(EventSnapshotChunkTable).all()).toEqual([]) for (const row of rows) { yield* target.stageSnapshotChunks!(active, row, chunks.get(row.rowHash)!) yield* target.stageSnapshotChunks!(active, row, chunks.get(row.rowHash)!) @@ -390,4 +415,124 @@ describe("EventV2 canonical projection snapshots", () => { expect(divergent).toBeInstanceOf(EventV2.InvalidSyncEventError) }), ) + + it.effect("advances compaction across retained audit events without deleting them", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const db = (yield* Database.Service).db + const auditSessionID = SessionV2.ID.make("ses_snapshot_retained_audit") + yield* db.insert(ProjectTable).values({ + id: projectID, + worktree: AbsolutePath.make("/project"), + sandboxes: [], + }).run().pipe(Effect.orDie) + const info = SessionV1.SessionInfo.make({ + id: auditSessionID, + slug: "snapshot-retained-audit", + version: "test", + projectID, + directory: "/project", + title: "snapshot-retained-audit", + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 1, updated: 1 }, + }) + const updated = (id: string, text: string) => ({ + sessionID: auditSessionID, + info: { + id: SessionV1.MessageID.make(id), + sessionID: auditSessionID, + role: "user" as const, + time: { created: 1 }, + agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("test") }, + system: text, + }, + }) + yield* events.publish(SessionV1.Event.Created, { sessionID: auditSessionID, info }) + yield* events.publish(SessionV1.Event.MessageUpdated, updated("msg_audit_before", "before")) + yield* events.publish(SessionEvent.Execution.Started, { + sessionID: auditSessionID, + timestamp: DateTime.makeUnsafe(2), + }) + yield* events.publish(SessionV1.Event.MessageUpdated, updated("msg_audit_after", "after")) + const snapshot = yield* events.checkpoint({ + aggregateID: auditSessionID, + throughSeq: EventV2.Cursor.make(3), + expectedLatest: EventV2.Cursor.make(3), + codec: "session-projection", + schemaVersion: 1, + }) + + const cursors: number[] = [] + let result = { deleted: 0, complete: false } + let deleted = 0 + while (!result.complete) { + result = yield* events.compact({ + aggregateID: auditSessionID, + throughSeq: EventV2.Cursor.make(snapshot.throughSeq), + limit: 1, + }) + deleted += result.deleted + cursors.push((yield* db.select().from(EventCompactionReceiptTable) + .where(eq(EventCompactionReceiptTable.aggregate_id, auditSessionID)).get().pipe(Effect.orDie))!.cursor_seq) + } + + expect(deleted).toBe(3) + expect(cursors).toEqual([0, 1, 2, 3]) + expect(yield* db.select({ type: EventTable.type }).from(EventTable) + .where(eq(EventTable.aggregate_id, auditSessionID)).all().pipe(Effect.orDie)).toEqual([ + { type: EventV2.versionedType(SessionEvent.Execution.Started.type, 1) }, + ]) + expect(yield* db.select().from(EventDedupeTable) + .where(eq(EventDedupeTable.aggregate_id, auditSessionID)).all().pipe(Effect.orDie)).toHaveLength(3) + expect(yield* db.select().from(EventCompactionReceiptTable) + .where(eq(EventCompactionReceiptTable.aggregate_id, auditSessionID)).get().pipe(Effect.orDie)) + .toMatchObject({ cursor_seq: 3, state: "complete" }) + expect(yield* events.compact({ + aggregateID: auditSessionID, + throughSeq: EventV2.Cursor.make(snapshot.throughSeq), + limit: 1, + })).toEqual({ deleted: 0, complete: true }) + }), + ) + + it.effect("replaces a same-sequence snapshot after projection authority changes and cascades all snapshot sidecars", () => + Effect.gen(function* () { + const events = yield* EventV2.Service + const db = (yield* Database.Service).db + const localSessionID = SessionV2.ID.make("ses_snapshot_revision_replace") + yield* db.insert(ProjectTable).values({ id: projectID, worktree: AbsolutePath.make("/project"), sandboxes: [] }).run().pipe(Effect.orDie) + yield* events.publish(SessionV1.Event.Created, { + sessionID: localSessionID, + info: SessionV1.SessionInfo.make({ id: localSessionID, slug: "revision", version: "test", projectID, + directory: "/project", title: "revision", cost: 0, tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 1, updated: 1 } }), + }) + const first = yield* events.checkpoint({ aggregateID: localSessionID, throughSeq: EventV2.Cursor.make(0), + expectedLatest: EventV2.Cursor.make(0), codec: "session-projection", schemaVersion: 1 }) + let firstCompaction = { deleted: 0, complete: false } + while (!firstCompaction.complete) + firstCompaction = yield* events.compact({ aggregateID: localSessionID, throughSeq: EventV2.Cursor.make(0), limit: 1 }) + expect(yield* db.select().from(EventCompactionReceiptTable).where(eq(EventCompactionReceiptTable.aggregate_id, localSessionID)).get().pipe(Effect.orDie)) + .toMatchObject({ snapshot_id: first.snapshotID, state: "complete", cursor_seq: 0 }) + yield* db.update(SessionTable).set({ mutation_epoch: 1 }).where(eq(SessionTable.id, localSessionID)).run().pipe(Effect.orDie) + const second = yield* events.checkpoint({ aggregateID: localSessionID, throughSeq: EventV2.Cursor.make(0), + expectedLatest: EventV2.Cursor.make(0), codec: "session-projection", schemaVersion: 1 }) + expect(second.snapshotID).not.toBe(first.snapshotID) + expect((yield* events.snapshot(localSessionID))?.snapshotID).toBe(second.snapshotID) + expect(yield* events.compact({ aggregateID: localSessionID, throughSeq: EventV2.Cursor.make(0), limit: 1 })) + .toEqual({ deleted: 0, complete: true }) + expect(yield* db.select().from(EventCompactionReceiptTable).where(eq(EventCompactionReceiptTable.aggregate_id, localSessionID)).get().pipe(Effect.orDie)) + .toMatchObject({ snapshot_id: second.snapshotID, state: "complete", cursor_seq: 0 }) + expect(yield* db.select().from(EventSnapshotTable).where(eq(EventSnapshotTable.aggregate_id, localSessionID)).all()).toHaveLength(1) + expect(yield* db.select().from(EventSnapshotAttemptTable).where(eq(EventSnapshotAttemptTable.aggregate_id, localSessionID)).all()).toHaveLength(1) + expect(yield* db.select().from(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.aggregate_id, localSessionID)).all()).not.toHaveLength(0) + yield* events.remove(localSessionID) + expect(yield* db.select().from(EventSnapshotTable).where(eq(EventSnapshotTable.aggregate_id, localSessionID)).all()).toEqual([]) + expect(yield* db.select().from(EventSnapshotAttemptTable).where(eq(EventSnapshotAttemptTable.aggregate_id, localSessionID)).all()).toEqual([]) + expect(yield* db.select().from(EventSnapshotRowTable).where(eq(EventSnapshotRowTable.aggregate_id, localSessionID)).all()).toEqual([]) + expect(yield* db.select().from(EventSnapshotChunkTable).all()).toEqual([]) + }), + ) }) diff --git a/packages/core/test/file-part-artifact.test.ts b/packages/core/test/file-part-artifact.test.ts index 60e0b2229..16900d5bb 100644 --- a/packages/core/test/file-part-artifact.test.ts +++ b/packages/core/test/file-part-artifact.test.ts @@ -3,11 +3,18 @@ import { Context, Effect, Layer } from "effect" import { eq, sql } from "drizzle-orm" import { Database } from "@deepagent-code/core/database/database" import { EventV2 } from "@deepagent-code/core/event" -import { EventSequenceTable, EventSyncSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { + EventSequenceTable, + EventSnapshotRowTable, + EventSnapshotTable, + EventSyncSequenceTable, + EventTable, +} from "@deepagent-code/core/event/sql" import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" import { FilePartArtifactBindingTable, FilePartArtifactChunkTable, + FilePartArtifactDiscardTable, FilePartArtifactImportTable, FilePartArtifactTable, } from "@deepagent-code/core/file-part-artifact.sql" @@ -16,6 +23,8 @@ import { SessionV1 } from "@deepagent-code/core/v1/session" import { Hash } from "@deepagent-code/core/util/hash" import { AbsolutePath } from "@deepagent-code/core/schema" import { Project } from "@deepagent-code/core/project" +import { ProviderV2 } from "@deepagent-code/core/provider" +import { ModelV2 } from "@deepagent-code/core/model" import { ProjectTable } from "@deepagent-code/core/project/sql" import { MessageTable, PartTable, SessionTable } from "@deepagent-code/core/session/sql" import { SessionProjector } from "@deepagent-code/core/session/projector" @@ -131,6 +140,18 @@ describe("FilePartArtifact", () => { .run() .pipe(Effect.orDie) + const inlineSnapshot = yield* events.checkpoint({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(1), + expectedLatest: EventV2.Cursor.make(1), + codec: "session-projection", + schemaVersion: 1, + }) + const inlineRows = yield* events.snapshotRows!({ snapshotID: inlineSnapshot.snapshotID }) + const inlinePartRow = inlineRows.find((row) => row.tableName === "part" && row.rowKey === partID)! + expect(Buffer.concat((yield* events.snapshotChunks!({ rowHash: inlinePartRow.rowHash })).map((chunk) => chunk.data)).toString()) + .toContain("data:application/octet-stream;base64,") + const first = yield* FilePartArtifact.canonicalizeLegacy(db, { limit: 1, now: 7 }) expect(first).toEqual({ processed: 1, next: eventID }) expect(yield* FilePartArtifact.canonicalizeLegacy(db, { afterID: eventID, limit: 1, now: 8 })).toEqual({ @@ -164,6 +185,16 @@ describe("FilePartArtifact", () => { expect(yield* db.select().from(FilePartArtifactBindingTable).all()).toHaveLength(1) expect(yield* db.select().from(FilePartArtifactChunkTable).all()).toHaveLength(descriptor.chunks) + yield* db.run(` + CREATE TRIGGER test_file_part_dedupe_never_copies_source + BEFORE INSERT ON event_dedupe + WHEN NEW.source_data IS NOT NULL + AND EXISTS (SELECT 1 FROM file_part_artifact_binding WHERE event_id = NEW.event_id) + BEGIN + SELECT RAISE(ABORT, 'file_part_dedupe_copied_source'); + END + `).pipe(Effect.orDie) + yield* events.replay({ id: eventID, aggregateID: sessionID, @@ -252,6 +283,11 @@ describe("FilePartArtifact", () => { codec: "session-projection", schemaVersion: 1, }) + expect(snapshot.snapshotID).not.toBe(inlineSnapshot.snapshotID) + expect(yield* db.select({ id: EventSnapshotTable.snapshot_id }).from(EventSnapshotTable).all()) + .toEqual([{ id: snapshot.snapshotID }]) + expect(yield* db.select().from(EventSnapshotRowTable) + .where(eq(EventSnapshotRowTable.snapshot_id, inlineSnapshot.snapshotID)).all()).toEqual([]) const snapshotRows = yield* events.snapshotRows!({ snapshotID: snapshot.snapshotID }) const snapshotTarget = yield* Layer.buildWithScope(Layer.fresh(projectedRuntime()), yield* Effect.scope) const snapshotDb = Context.get(snapshotTarget, Database.Service).db @@ -284,6 +320,103 @@ describe("FilePartArtifact", () => { ) yield* snapshotEvents.importSnapshot(snapshot) expect((yield* snapshotDb.select().from(PartTable).where(eq(PartTable.id, partID)).get())?.data).toEqual(projected!.data) + + let compacted = { deleted: 0, complete: false } + while (!compacted.complete) + compacted = yield* events.compact({ aggregateID: sessionID, throughSeq: EventV2.Cursor.make(1), limit: 1 }) + expect(yield* db.select().from(EventTable).where(eq(EventTable.id, eventID)).all()).toEqual([]) + expect(yield* db.all<{ data_hash: string; source_data: string | null }>(sql` + SELECT data_hash, source_data FROM event_dedupe WHERE event_id = ${eventID} + `).pipe(Effect.orDie)).toEqual([{ data_hash: binding!.original_data_hash, source_data: null }]) + yield* events.replay({ + id: eventID, + aggregateID: sessionID, + seq: 1, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + data, + }) + const compactedDivergent = yield* events + .replay({ + id: eventID, + aggregateID: sessionID, + seq: 1, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + data: { + ...data, + part: { ...data.part, url: `data:text/plain;base64,${body.toString("base64")}` }, + }, + }) + .pipe(Effect.catchDefect(Effect.succeed)) + expect(compactedDivergent).toBeInstanceOf(EventV2.InvalidSyncEventError) + + const canonicalSource = { + ...data, + part: { + ...data.part, + id: SessionV1.PartID.make("prt_legacy_file_part_canonical"), + url: `data:application/octet-stream;base64,${body.toString("base64")}`, + }, + } + const canonicalEventID = EventV2.ID.make("evt_legacy_file_part_canonical") + const canonicalPrepared = FilePartArtifact.prepare( + SessionV1.Event.PartUpdated.type, + canonicalSource, + Buffer.byteLength(JSON.stringify(canonicalSource)), + EventV2.MAX_ENCODED_PAYLOAD_BYTES, + ) + yield* db.insert(EventTable).values({ + id: canonicalEventID, + aggregate_id: sessionID, + seq: 2, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + data: canonicalSource, + sync_seq: 3, + }).run().pipe(Effect.orDie) + yield* FilePartArtifact.put(db, canonicalPrepared.artifacts[0]!, 9) + yield* FilePartArtifact.bind(db, { + eventID: canonicalEventID, + aggregateID: sessionID, + seq: 2, + type: SessionV1.Event.PartUpdated.type, + data: canonicalPrepared.data, + originalData: canonicalSource, + now: 9, + }) + yield* db.update(EventSequenceTable).set({ seq: 2 }) + .where(eq(EventSequenceTable.aggregate_id, sessionID)).run().pipe(Effect.orDie) + const canonicalSnapshot = yield* events.checkpoint({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(2), + expectedLatest: EventV2.Cursor.make(2), + codec: "session-projection", + schemaVersion: 1, + }) + let canonicalCompacted = { deleted: 0, complete: false } + while (!canonicalCompacted.complete) + canonicalCompacted = yield* events.compact({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(2), + limit: 1, + }) + expect(canonicalSnapshot.throughSeq).toBe(2) + yield* events.replay({ + id: canonicalEventID, + aggregateID: sessionID, + seq: 2, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + data: canonicalSource, + }) + const sameDescriptorDifferentSource = yield* events.replay({ + id: canonicalEventID, + aggregateID: sessionID, + seq: 2, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + data: { + ...canonicalSource, + part: { ...canonicalSource.part, url: `data:text/plain;base64,${body.toString("base64")}` }, + }, + }).pipe(Effect.catchDefect(Effect.succeed)) + expect(sameDescriptorDifferentSource).toBeInstanceOf(EventV2.InvalidSyncEventError) }), ) @@ -300,19 +433,29 @@ describe("FilePartArtifact", () => { const sessionID = SessionSchema.ID.make("ses_file_part_two_db") const body = Buffer.alloc(FilePartArtifact.CHUNK_BYTES * 2 + 17, 0x5a) const prelude = yield* sourceEvents.publish(Prelude, { sessionID }) - const published = yield* sourceEvents.publish(SessionV1.Event.PartUpdated, { + const sourceData = { sessionID, time: 1, part: { id: SessionV1.PartID.make("prt_file_part_two_db"), sessionID, messageID: SessionV1.MessageID.make("msg_file_part_two_db"), - type: "file", + type: "file" as const, mime: "application/octet-stream", filename: "large.bin", url: `data:application/octet-stream;base64,${body.toString("base64")}`, }, + } + const published = yield* sourceEvents.publish(SessionV1.Event.PartUpdated, sourceData) + yield* sourceEvents.publish(SessionV1.Event.PartUpdated, sourceData, { + id: published.id, + idempotent: true, }) + const divergentPublish = yield* sourceEvents.publish(SessionV1.Event.PartUpdated, { + ...sourceData, + part: { ...sourceData.part, url: `data:text/plain;base64,${body.toString("base64")}` }, + }, { id: published.id, idempotent: true }).pipe(Effect.catchDefect(Effect.succeed)) + expect(divergentPublish).toBeInstanceOf(EventV2.InvalidSyncEventError) const descriptor = FilePartArtifact.descriptor(published.data) expect(descriptor).toBeDefined() if (!descriptor) return @@ -381,7 +524,7 @@ describe("FilePartArtifact", () => { ).toMatchObject({ seq: 0 }) expect(yield* targetDb.select().from(EventTable).where(eq(EventTable.id, published.id)).get()).toBeUndefined() expect(yield* targetDb.select().from(FilePartArtifactBindingTable).all()).toEqual([]) - expect(yield* targetDb.select().from(FilePartArtifactImportTable).all()).toEqual([]) + expect(yield* targetDb.select().from(FilePartArtifactImportTable).all()).toHaveLength(1) yield* Effect.forEach( chunks.slice(1), @@ -417,6 +560,41 @@ describe("FilePartArtifact", () => { expect(yield* targetDb.select().from(FilePartArtifactImportTable).all()).toEqual([]) expect(yield* targetDb.select().from(EventTable).where(eq(EventTable.id, published.id)).all()).toHaveLength(1) + yield* sourceEvents.replay({ + id: published.id, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + aggregateID: sessionID, + seq: 1, + data: published.data, + }) + yield* sourceEvents.replay({ + id: published.id, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + aggregateID: sessionID, + seq: 1, + data: sourceData, + }) + const differentRawSource = yield* sourceEvents.replay({ + id: published.id, + type: EventV2.versionedType(SessionV1.Event.PartUpdated.type, 1), + aggregateID: sessionID, + seq: 1, + data: { + sessionID, + time: 1, + part: { + id: SessionV1.PartID.make("prt_file_part_two_db"), + sessionID, + messageID: SessionV1.MessageID.make("msg_file_part_two_db"), + type: "file", + mime: "application/octet-stream", + filename: "large.bin", + url: `data:text/plain;base64,${body.toString("base64")}`, + }, + }, + }).pipe(Effect.catchDefect(Effect.succeed)) + expect(differentRawSource).toBeInstanceOf(EventV2.InvalidSyncEventError) + const snapshotRef = yield* FilePartArtifact.snapshotRef(sourceDb, { aggregateID: sessionID, partID: SessionV1.PartID.make("prt_file_part_two_db"), @@ -441,6 +619,71 @@ describe("FilePartArtifact", () => { expect(yield* snapshotDb.select().from(FilePartArtifactBindingTable).all()).toEqual([]) expect(yield* snapshotDb.select().from(FilePartArtifactImportTable).all()).toHaveLength(1) + yield* snapshotDb + .insert(ProjectTable) + .values({ id: Project.ID.global, worktree: AbsolutePath.make("/snapshot"), sandboxes: [] }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* snapshotDb + .insert(SessionTable) + .values({ + id: sessionID, + project_id: Project.ID.global, + slug: "snapshot-file-part", + directory: AbsolutePath.make("/snapshot"), + title: "snapshot-file-part", + version: "test", + }) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + const canonicalPart = snapshotRef.canonicalData.part as Record + yield* snapshotDb + .insert(MessageTable) + .values({ + id: canonicalPart.messageID as SessionV1.MessageID, + session_id: sessionID, + time_created: 1, + data: { role: "user", time: { created: 1 } }, + } as typeof MessageTable.$inferInsert) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + const projectedPartData = Object.fromEntries( + Object.entries(canonicalPart).filter(([key]) => key !== "id" && key !== "sessionID" && key !== "messageID"), + ) + yield* snapshotDb + .insert(PartTable) + .values({ + id: SessionV1.PartID.make("prt_file_part_two_db"), + message_id: canonicalPart.messageID as SessionV1.MessageID, + session_id: sessionID, + time_created: 1, + data: projectedPartData, + } as typeof PartTable.$inferInsert) + .onConflictDoNothing() + .run() + .pipe(Effect.orDie) + yield* snapshotDb + .update(PartTable) + .set({ data: { ...projectedPartData, filename: "tampered.bin" } as unknown as typeof PartTable.$inferInsert["data"] }) + .where(eq(PartTable.id, SessionV1.PartID.make("prt_file_part_two_db"))) + .run() + .pipe(Effect.orDie) + const mismatchedProjection = yield* FilePartArtifact.bindSnapshotRef({ + metadata: snapshotRef, + partID: SessionV1.PartID.make("prt_file_part_two_db"), + }).pipe(Effect.provide(snapshotTarget), Effect.exit) + expect(mismatchedProjection._tag).toBe("Failure") + expect(yield* snapshotDb.select().from(FilePartArtifactBindingTable).all()).toEqual([]) + expect(yield* snapshotDb.select().from(FilePartArtifactImportTable).all()).toHaveLength(1) + yield* snapshotDb + .update(PartTable) + .set({ data: projectedPartData as typeof PartTable.$inferInsert["data"] }) + .where(eq(PartTable.id, SessionV1.PartID.make("prt_file_part_two_db"))) + .run() + .pipe(Effect.orDie) yield* snapshotDb .insert(EventSequenceTable) .values({ aggregate_id: sessionID, seq: 1 }) @@ -456,4 +699,93 @@ describe("FilePartArtifact", () => { expect(yield* snapshotDb.select().from(FilePartArtifactImportTable).all()).toEqual([]) }), ) + + it.effect("discards abandoned imports and reclaims artifact bodies after the last aggregate binding", () => + Effect.gen(function* () { + const runtimeLayer = yield* Layer.buildWithScope(Layer.fresh(runtime()), yield* Effect.scope) + const events = Context.get(runtimeLayer, EventV2.Service) + const db = Context.get(runtimeLayer, Database.Service).db + const sessionID = SessionSchema.ID.make("ses_file_part_cleanup") + const body = Buffer.alloc(FilePartArtifact.CHUNK_BYTES + 1, 0x41) + const descriptor = FilePartArtifact.Descriptor.make({ + codec: FilePartArtifact.CODEC, + id: FilePartArtifact.ID.make(`fpart_${Hash.sha256(body)}`), + hash: Hash.sha256(body), + bytes: body.length, + chunkBytes: FilePartArtifact.CHUNK_BYTES, + chunks: 2, + }) + const canonicalData = { + sessionID, + time: 1, + part: { + id: SessionV1.PartID.make("prt_file_part_cleanup"), + messageID: SessionV1.MessageID.make("msg_file_part_cleanup"), + sessionID, + type: "file", + mime: "application/octet-stream", + url: `event-artifact:${descriptor.id}`, + artifact: descriptor, + }, + } + const metadata = FilePartArtifact.Metadata.make({ + eventID: EventV2.ID.create(), + aggregateID: sessionID, + seq: 0, + originalDataHash: Hash.sha256("original"), + canonicalDataHash: FilePartArtifact.dataHash(canonicalData), + canonicalData, + descriptor, + chunkHashes: [Hash.sha256(body.subarray(0, FilePartArtifact.CHUNK_BYTES)), Hash.sha256(body.subarray(FilePartArtifact.CHUNK_BYTES))], + }) + yield* FilePartArtifact.importChunk({ + metadata, + index: 0, + hash: metadata.chunkHashes[0]!, + data: body.subarray(0, FilePartArtifact.CHUNK_BYTES), + }).pipe(Effect.provide(runtimeLayer)) + yield* FilePartArtifact.importChunk({ + metadata, + index: 1, + hash: metadata.chunkHashes[1]!, + data: body.subarray(FilePartArtifact.CHUNK_BYTES), + }).pipe(Effect.provide(runtimeLayer)) + expect(yield* db.select().from(FilePartArtifactImportTable).all()).toHaveLength(1) + expect(yield* FilePartArtifact.discardImport({ + eventID: metadata.eventID as EventV2.ID, + aggregateID: sessionID, + artifactID: descriptor.id, + }).pipe(Effect.provide(runtimeLayer))).toBe(true) + expect(yield* db.select().from(FilePartArtifactImportTable).all()).toEqual([]) + expect(yield* db.select().from(FilePartArtifactDiscardTable).all()).toEqual([]) + expect(yield* db.select().from(FilePartArtifactTable).all()).toEqual([]) + expect(yield* db.select().from(FilePartArtifactChunkTable).all()).toEqual([]) + + const projected = yield* Layer.buildWithScope(Layer.fresh(projectedRuntime()), yield* Effect.scope) + const projectedEvents = Context.get(projected, EventV2.Service) + const projectedDb = Context.get(projected, Database.Service).db + yield* projectedDb.insert(ProjectTable).values({ id: Project.ID.global, worktree: AbsolutePath.make("/cleanup"), sandboxes: [] }).run().pipe(Effect.orDie) + yield* projectedEvents.publish(SessionV1.Event.Created, { + sessionID, + info: SessionV1.SessionInfo.make({ id: sessionID, slug: "cleanup", version: "test", projectID: Project.ID.global, + directory: "/cleanup", title: "cleanup", cost: 0, tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 1, updated: 1 } }), + }) + yield* projectedEvents.publish(SessionV1.Event.MessageUpdated, { sessionID, info: { id: SessionV1.MessageID.make("msg_file_part_cleanup"), sessionID, + role: "user", time: { created: 1 }, agent: "build", + model: { providerID: ProviderV2.ID.make("test"), modelID: ModelV2.ID.make("test") } } }) + yield* projectedEvents.publish(SessionV1.Event.PartUpdated, { + sessionID, + time: 1, + part: { id: SessionV1.PartID.make("prt_file_part_cleanup"), messageID: SessionV1.MessageID.make("msg_file_part_cleanup"), sessionID, + type: "file", mime: "application/octet-stream", url: `data:application/octet-stream;base64,${body.toString("base64")}` }, + }) + expect(yield* projectedDb.select().from(FilePartArtifactBindingTable).all()).toHaveLength(1) + expect(yield* projectedDb.select().from(FilePartArtifactTable).all()).toHaveLength(1) + yield* projectedEvents.remove(sessionID) + expect(yield* projectedDb.select().from(FilePartArtifactBindingTable).all()).toEqual([]) + expect(yield* projectedDb.select().from(FilePartArtifactTable).all()).toEqual([]) + expect(yield* projectedDb.select().from(FilePartArtifactChunkTable).all()).toEqual([]) + }), + ) }) diff --git a/packages/core/test/retention-sweeper.test.ts b/packages/core/test/retention-sweeper.test.ts index 986bb4e0d..cb960114b 100644 --- a/packages/core/test/retention-sweeper.test.ts +++ b/packages/core/test/retention-sweeper.test.ts @@ -1,5 +1,6 @@ import { describe, expect } from "bun:test" import { Effect, Layer } from "effect" +import { eq, sql } from "drizzle-orm" import { RetentionSweeper } from "@deepagent-code/core/deepagent/retention-sweeper" import { DeepAgentEventBus } from "@deepagent-code/core/deepagent/deepagent-event-bus" import { DeepAgentEvent } from "@deepagent-code/core/deepagent/deepagent-event" @@ -9,6 +10,12 @@ import { DeepAgentEventDeliveryTable } from "@deepagent-code/core/deepagent/deep import { ApprovalQueueTable } from "@deepagent-code/core/deepagent/approval-queue-sql" import { AgentPushLogTable } from "@deepagent-code/core/im/push-log-sql" import { EventSequenceTable, EventTable } from "@deepagent-code/core/event/sql" +import { EventV2 } from "@deepagent-code/core/event" +import { + FilePartArtifactChunkTable, + FilePartArtifactImportTable, + FilePartArtifactTable, +} from "@deepagent-code/core/file-part-artifact.sql" import { SessionTable } from "@deepagent-code/core/session/sql" import { testEffect } from "./lib/effect" @@ -335,15 +342,21 @@ describe("RetentionSweeper", () => { yield* db .run(`INSERT OR IGNORE INTO event_sequence (aggregate_id, seq) VALUES ('${opts.sessionID}', 1)`) .pipe(Effect.orDie) + yield* db + .run("UPDATE event_sync_sequence SET seq = seq + 1 WHERE id = 1") + .pipe(Effect.orDie) yield* db .run( - `INSERT OR IGNORE INTO event (id, aggregate_id, seq, type, data) - VALUES ('${opts.eventID}', '${opts.sessionID}', 1, 'message.part.updated.1', '{}')`, + `INSERT OR IGNORE INTO event (id, aggregate_id, seq, type, data, sync_seq) + VALUES ( + '${opts.eventID}', '${opts.sessionID}', 1, 'message.part.updated.1', '{}', + (SELECT seq FROM event_sync_sequence WHERE id = 1) + )`, ) .pipe(Effect.orDie) }) - it.effect("§EventV2-retention: prunes event_sequence + events for archived sessions past retentionDays", () => + it.effect("§EventV2-retention: preserves archived EventV2 roots until explicit removal", () => Effect.gen(function* () { const { db } = yield* Database.Service const b = yield* DeepAgentEventBus.Service @@ -359,13 +372,220 @@ describe("RetentionSweeper", () => { setNow(100 * DAY) const summary = yield* s.sweepOnce() - expect(summary.deletedEventV2Sequences).toBe(1) + expect(summary.deletedEventV2Sequences).toBe(0) - // Both the event_sequence header and the event row (cascaded) must be gone. + // Archived roots remain readable; Session/EventV2.remove owns the explicit cascade. const seqs = yield* db.select().from(EventSequenceTable).all().pipe(Effect.orDie) - expect(seqs.find((r) => r.aggregate_id === "ses_old")).toBeUndefined() + expect(seqs.find((r) => r.aggregate_id === "ses_old")).toBeDefined() const evts = yield* db.select().from(EventTable).all().pipe(Effect.orDie) - expect(evts.find((r) => r.id === "evt_old")).toBeUndefined() + expect(evts.find((r) => r.id === "evt_old")).toBeDefined() + }), + ) + + it.effect("§EventV2-retention: discovers archived Session workspaces without a DeepAgent event", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + const c = yield* WorkspaceConfig.Service + const s = yield* RetentionSweeper.Service + yield* c.set("wrk_session_only", { retentionDays: 30 }) + yield* insertArchivedSession({ sessionID: "ses_session_only", workspaceID: "wrk_session_only", timeArchived: 1_000 }) + yield* insertEventV2({ sessionID: "ses_session_only", eventID: "evt_session_only" }) + + setNow(100 * DAY) + const summary = yield* s.sweepOnce() + expect(summary.workspacesSwept).toBe(1) + expect(summary.deletedEventV2Sequences).toBe(0) + expect(yield* db.select().from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, "ses_session_only")).get()).toBeDefined() + }), + ) + + it.effect("§EventV2-retention: preserves sidecar authorities without blocking other archived sessions", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + const b = yield* DeepAgentEventBus.Service + const c = yield* WorkspaceConfig.Service + const s = yield* RetentionSweeper.Service + yield* c.set("wrk_1", { retentionDays: 30 }) + yield* publishAt(b, 1_000) + + yield* insertArchivedSession({ sessionID: "ses_artifact", workspaceID: "wrk_1", timeArchived: 1_000 }) + yield* insertEventV2({ sessionID: "ses_artifact", eventID: "evt_artifact" }) + yield* db.run(` + INSERT INTO message (id, session_id, time_created, time_updated, data) + VALUES ('msg_artifact', 'ses_artifact', 1, 1, '{"role":"user","time":{"created":1}}') + `).pipe(Effect.orDie) + yield* db.run(` + INSERT INTO event_artifact ( + artifact_id, event_id, aggregate_id, seq, kind, original_data_hash, + canonical_data_hash, canonical_data, body_hash, body_bytes, chunk_count, + codec_version, created_at + ) VALUES ( + 'evtart_retention', 'evt_artifact', 'ses_artifact', 1, 'legacy_message_diff', + '${"a".repeat(64)}', '${"b".repeat(64)}', '{}', '${"c".repeat(64)}', 0, 1, 2, 1 + ) + `).pipe(Effect.orDie) + yield* db.run(` + INSERT INTO session_diff_migration_receipt ( + message_id, session_id, artifact_id, source_event_id, + expected_message_data_hash, committed_message_data_hash, + expected_session_summary_hash, committed_session_summary_hash, + canonicalizer_version, canonicalization_version, epoch_hashes, state, + failure_reason, created_at, updated_at, committed_at + ) VALUES ( + 'msg_artifact', 'ses_artifact', 'evtart_retention', 'evt_artifact', + '${"d".repeat(64)}', '${"e".repeat(64)}', '${"f".repeat(64)}', '${"0".repeat(64)}', + 1, 1, '[]', 'committed', NULL, 1, 1, 1 + ) + `).pipe(Effect.orDie) + yield* insertArchivedSession({ sessionID: "ses_prunable", workspaceID: "wrk_1", timeArchived: 1_000 }) + yield* insertEventV2({ sessionID: "ses_prunable", eventID: "evt_prunable" }) + + setNow(100 * DAY) + const summary = yield* s.sweepOnce() + expect(summary.deletedEventV2Sequences).toBe(0) + expect( + yield* db.select().from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, "ses_artifact")).get().pipe(Effect.orDie), + ).toBeDefined() + expect( + yield* db.select().from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, "ses_prunable")).get().pipe(Effect.orDie), + ).toBeDefined() + expect(yield* db.all<{ state: string }>(sql` + SELECT state FROM session_diff_migration_receipt WHERE message_id = 'msg_artifact' + `).pipe(Effect.orDie)).toEqual([{ state: "committed" }]) + }), + ) + + it.effect("§EventV2-retention: invokes bounded compaction for sidecar sessions instead of leaking raw history", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + const b = yield* DeepAgentEventBus.Service + const c = yield* WorkspaceConfig.Service + yield* c.set("wrk_1", { retentionDays: 30 }) + yield* publishAt(b, 1_000) + yield* insertArchivedSession({ sessionID: "ses_compact", workspaceID: "wrk_1", timeArchived: 1_000 }) + yield* insertEventV2({ sessionID: "ses_compact", eventID: "evt_compact" }) + yield* db.run(` + INSERT INTO event_artifact ( + artifact_id, event_id, aggregate_id, seq, kind, original_data_hash, + canonical_data_hash, canonical_data, body_hash, body_bytes, chunk_count, + codec_version, created_at + ) VALUES ( + 'evtart_compact', 'evt_compact', 'ses_compact', 1, 'legacy_message_diff', + '${"a".repeat(64)}', '${"b".repeat(64)}', '{}', '${"c".repeat(64)}', 0, 1, 2, 1 + ) + `).pipe(Effect.orDie) + setNow(100 * DAY) + const invoked: string[] = [] + const compacting = RetentionSweeper.layerWith({ + now, + runLoop: false, + compactSession: (sessionID) => + Effect.sync(() => invoked.push(sessionID)).pipe(Effect.as(true)), + }).pipe( + Layer.provide(Layer.succeed(DeepAgentEventBus.Service)(b)), + Layer.provide(Layer.succeed(WorkspaceConfig.Service)(c)), + Layer.provide(Layer.succeed(Database.Service)({ db })), + ) + const summary = yield* Effect.gen(function* () { + const service = yield* RetentionSweeper.Service + return yield* service.sweepOnce() + }).pipe(Effect.provide(Layer.fresh(compacting))) + expect(summary.deletedEventV2Sequences).toBe(0) + expect(invoked).toEqual(["ses_compact"]) + expect(yield* db.select().from(EventSequenceTable).where(eq(EventSequenceTable.aggregate_id, "ses_compact")).get()).toBeDefined() + }), + ) + + it.effect("§EventV2-retention: sidecar compaction cannot starve an independent prunable batch", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + const b = yield* DeepAgentEventBus.Service + const c = yield* WorkspaceConfig.Service + yield* c.set("wrk_1", { retentionDays: 30 }) + yield* publishAt(b, 1_000) + yield* Effect.forEach( + Array.from({ length: 101 }, (_, index) => String(index).padStart(3, "0")), + (suffix) => Effect.gen(function* () { + const sessionID = `ses_sidecar_${suffix}` + const eventID = `evt_sidecar_${suffix}` + yield* insertArchivedSession({ sessionID, workspaceID: "wrk_1", timeArchived: 1_000 }) + yield* insertEventV2({ sessionID, eventID }) + yield* db.run(sql` + INSERT INTO event_artifact ( + artifact_id, event_id, aggregate_id, seq, kind, original_data_hash, + canonical_data_hash, canonical_data, body_hash, body_bytes, chunk_count, + codec_version, created_at + ) VALUES ( + ${`evtart_sidecar_${suffix}`}, ${eventID}, ${sessionID}, 1, 'legacy_message_diff', + ${"a".repeat(64)}, ${"b".repeat(64)}, '{}', ${"c".repeat(64)}, 0, 1, 2, 1 + ) + `).pipe(Effect.orDie) + }), + { discard: true }, + ) + yield* insertArchivedSession({ sessionID: "ses_zz_prunable", workspaceID: "wrk_1", timeArchived: 1_000 }) + yield* insertEventV2({ sessionID: "ses_zz_prunable", eventID: "evt_zz_prunable" }) + + setNow(100 * DAY) + const invoked: string[] = [] + const compacting = RetentionSweeper.layerWith({ + now, + runLoop: false, + compactSession: (sessionID) => Effect.sync(() => invoked.push(sessionID)).pipe(Effect.as(false)), + }).pipe( + Layer.provide(Layer.succeed(DeepAgentEventBus.Service)(b)), + Layer.provide(Layer.succeed(WorkspaceConfig.Service)(c)), + Layer.provide(Layer.succeed(Database.Service)({ db })), + ) + const summary = yield* Effect.gen(function* () { + const service = yield* RetentionSweeper.Service + return yield* service.sweepOnce() + }).pipe(Effect.provide(Layer.fresh(compacting))) + expect(summary.deletedEventV2Sequences).toBe(0) + expect(invoked).toHaveLength(100) + expect(yield* db.select().from(EventSequenceTable) + .where(eq(EventSequenceTable.aggregate_id, "ses_zz_prunable")).get()).toBeDefined() + }), + ) + + it.effect("§EventV2-retention: reclaims an expired abandoned FilePart import", () => + Effect.gen(function* () { + const { db } = yield* Database.Service + const s = yield* RetentionSweeper.Service + yield* db.insert(FilePartArtifactTable).values({ + artifact_id: "fpart_abandoned", + body_hash: "a".repeat(64), + body_bytes: 1, + chunk_bytes: 262_144, + chunk_count: 1, + codec_version: 1, + complete: true, + created_at: 1, + }).run().pipe(Effect.orDie) + yield* db.insert(FilePartArtifactChunkTable).values({ + artifact_id: "fpart_abandoned", + chunk_index: 0, + data: Buffer.from("x"), + chunk_hash: "b".repeat(64), + }).run().pipe(Effect.orDie) + yield* db.insert(FilePartArtifactImportTable).values({ + event_id: EventV2.ID.make("evt_abandoned"), + aggregate_id: "ses_abandoned", + seq: 1, + artifact_id: "fpart_abandoned", + original_data_hash: "c".repeat(64), + canonical_data_hash: "d".repeat(64), + canonical_data: {}, + created_at: 1, + }).run().pipe(Effect.orDie) + + yield* s.sweepOnce(2 * DAY) + expect(yield* db.select().from(FilePartArtifactImportTable).all()).toEqual([]) + expect(yield* db.select().from(FilePartArtifactTable).all()).toEqual([]) + expect(yield* db.select().from(FilePartArtifactChunkTable).all()).toEqual([]) }), ) diff --git a/packages/deepagent-code/script/bug-407-010-release-gate.ts b/packages/deepagent-code/script/bug-407-010-release-gate.ts index 377aae2bd..9dc2b8d54 100644 --- a/packages/deepagent-code/script/bug-407-010-release-gate.ts +++ b/packages/deepagent-code/script/bug-407-010-release-gate.ts @@ -16,15 +16,16 @@ import { import { FilePartArtifact } from "@deepagent-code/core/file-part-artifact" import { FilePartArtifactBindingTable } from "@deepagent-code/core/file-part-artifact.sql" import { SessionProjector } from "@deepagent-code/core/session/projector" -import { MessageTable, SessionTable } from "@deepagent-code/core/session/sql" +import { MessageTable, PartTable, SessionTable } from "@deepagent-code/core/session/sql" import { Hash } from "@deepagent-code/core/util/hash" import { Cause, Effect, Exit, Layer, Option, Schema } from "effect" -import { and, asc, eq, sql } from "drizzle-orm" +import { and, asc, eq, gt, sql } from "drizzle-orm" +import { isDeepStrictEqual } from "node:util" import path from "node:path" import { parseArgs } from "node:util" import { SessionDiffArtifact } from "@/session/diff-artifact" import { SessionDiffMigrationReceiptTable } from "@/session/diff-artifact.sql" -import { SessionID } from "@/session/schema" +import { PartID, SessionID } from "@/session/schema" const args = parseArgs({ args: process.argv.slice(2), @@ -228,6 +229,98 @@ const program = Effect.gen(function* () { }, ) + const validateSnapshotFileAuthority = Effect.fn("Bug407010ReleaseGate.validateSnapshotFileAuthority")( + function* (input: { sessionID: SessionID; snapshotID: string }) { + let after = "" + let validated = 0 + while (true) { + const bindings = yield* db + .select({ + eventID: FilePartArtifactBindingTable.event_id, + seq: FilePartArtifactBindingTable.seq, + partID: FilePartArtifactBindingTable.part_id, + artifactID: FilePartArtifactBindingTable.artifact_id, + }) + .from(FilePartArtifactBindingTable) + .where(and( + eq(FilePartArtifactBindingTable.aggregate_id, input.sessionID), + after ? gt(FilePartArtifactBindingTable.event_id, after as EventV2.ID) : undefined, + )) + .orderBy(asc(FilePartArtifactBindingTable.event_id)) + .limit(100) + .all() + .pipe(Effect.orDie) + if (bindings.length === 0) break + for (const binding of bindings) { + const part = yield* db.select().from(PartTable).where(eq(PartTable.id, PartID.make(binding.partID))).get().pipe(Effect.orDie) + if (!part || part.session_id !== input.sessionID) + return yield* Effect.die(new Error(`File artifact ${binding.eventID} has no matching projected Part`)) + const descriptor = FilePartArtifact.descriptor({ + sessionID: input.sessionID, + part: { id: part.id, messageID: part.message_id, sessionID: input.sessionID, ...part.data }, + }) + if (!descriptor || descriptor.id !== binding.artifactID) + return yield* Effect.die(new Error(`File artifact ${binding.eventID} has a divergent Part descriptor`)) + const expected = yield* FilePartArtifact.metadata(db, { + eventID: binding.eventID, + aggregateID: input.sessionID, + seq: binding.seq, + artifactID: descriptor.id, + }) + for (const [index, hash] of expected.chunkHashes.entries()) + yield* FilePartArtifact.chunk(db, { artifactID: descriptor.id, index, expectedHash: hash }) + const partRow = yield* decodeSnapshotRow(input.snapshotID, "part", binding.partID) + const artifactRow = yield* decodeSnapshotRow( + input.snapshotID, + "file_part_artifact_binding", + binding.partID, + ) + if (!record(partRow) || !isDeepStrictEqual(partRow.data, part.data)) + return yield* Effect.die(new Error(`Snapshot ${input.snapshotID} has a stale Part ${binding.partID}`)) + if ( + !record(artifactRow) || + artifactRow.partID !== binding.partID || + !Schema.is(FilePartArtifact.Metadata)(artifactRow.metadata) || + !isDeepStrictEqual(artifactRow.metadata, expected) + ) + return yield* Effect.die( + new Error(`Snapshot ${input.snapshotID} has stale FilePart authority for ${binding.partID}`), + ) + if (JSON.stringify(partRow).includes('"data:')) + return yield* Effect.die(new Error(`Snapshot ${input.snapshotID} retained inline FilePart data`)) + validated += 1 + } + after = bindings.at(-1)!.eventID + } + return { fileArtifacts: validated } + }, + ) + + const decodeSnapshotRow = Effect.fn("Bug407010ReleaseGate.decodeSnapshotRow")( + function* (snapshotID: string, tableName: string, rowKey: string) { + const row = yield* db.select().from(EventSnapshotRowTable).where(and( + eq(EventSnapshotRowTable.snapshot_id, snapshotID), + eq(EventSnapshotRowTable.table_name, tableName), + eq(EventSnapshotRowTable.row_key, rowKey), + )).get().pipe(Effect.orDie) + if (!row) return yield* Effect.die(new Error(`Snapshot ${snapshotID} is missing ${tableName}:${rowKey}`)) + const chunks = yield* db.select().from(EventSnapshotChunkTable) + .where(eq(EventSnapshotChunkTable.row_hash, row.row_hash)) + .orderBy(asc(EventSnapshotChunkTable.chunk_index)).all().pipe(Effect.orDie) + if ( + chunks.length !== row.chunk_count || + chunks.some((chunk, index) => chunk.chunk_index !== index || Hash.sha256(chunk.data) !== chunk.chunk_hash) + ) + return yield* Effect.die(new Error(`Snapshot ${snapshotID} has invalid chunks for ${tableName}:${rowKey}`)) + const body = Buffer.concat(chunks.map((chunk) => chunk.data)) + if (body.length !== row.row_bytes || Hash.sha256(body) !== row.row_hash) + return yield* Effect.die(new Error(`Snapshot ${snapshotID} has an invalid row hash for ${tableName}:${rowKey}`)) + const decoded = Option.getOrUndefined(Schema.decodeUnknownOption(Schema.UnknownFromJsonString)(body.toString())) + if (!record(decoded)) return yield* Effect.die(new Error(`Snapshot ${snapshotID} row ${tableName}:${rowKey} is invalid`)) + return decoded + }, + ) + yield* stage("checkpoint-and-compact", Effect.forEach(checkpointSessions, (sessionID) => Effect.gen(function* () { const session = yield* db .select({ id: SessionTable.id }) @@ -243,11 +336,8 @@ const program = Effect.gen(function* () { .get() .pipe(Effect.orDie) if (!sequence) return yield* Effect.die(new Error(`Event sequence not found: ${sessionID}`)) - const current = yield* events.snapshot(sessionID) - const snapshot = current?.throughSeq === sequence.seq - ? current - : yield* Effect.gen(function* () { - let attempt = yield* events.prepareCheckpoint!({ + const snapshot = yield* Effect.gen(function* () { + let attempt = yield* events.prepareCheckpoint!({ aggregateID: sessionID, throughSeq: EventV2.Cursor.make(sequence.seq), expectedLatest: EventV2.Cursor.make(sequence.seq), @@ -255,16 +345,19 @@ const program = Effect.gen(function* () { schemaVersion: 1, ...(sequence.ownerID ? { ownerID: sequence.ownerID } : {}), }) - while (attempt.state === "prepared") - attempt = yield* events.stageCheckpoint!({ snapshotID: attempt.snapshotID }) - if (attempt.state === "complete") { - const active = yield* events.snapshot(sessionID) - if (active?.snapshotID === attempt.snapshotID) return active - return yield* Effect.die(new Error(`Completed snapshot is not active: ${attempt.snapshotID}`)) - } - return yield* events.finalizeCheckpoint!({ snapshotID: attempt.snapshotID }) - }) - const authority = yield* validateSnapshotDiffAuthority({ sessionID, snapshotID: snapshot.snapshotID }) + while (attempt.state === "prepared") + attempt = yield* events.stageCheckpoint!({ snapshotID: attempt.snapshotID }) + if (attempt.state === "complete") { + const active = yield* events.snapshot(sessionID) + if (active?.snapshotID === attempt.snapshotID) return active + return yield* Effect.die(new Error(`Completed snapshot is not active: ${attempt.snapshotID}`)) + } + return yield* events.finalizeCheckpoint!({ snapshotID: attempt.snapshotID }) + }) + const authority = { + ...(yield* validateSnapshotDiffAuthority({ sessionID, snapshotID: snapshot.snapshotID })), + ...(yield* validateSnapshotFileAuthority({ sessionID, snapshotID: snapshot.snapshotID })), + } let compacted = { deleted: 0, complete: false } let deleted = 0 let batches = 0 diff --git a/packages/deepagent-code/src/control-plane/workspace.ts b/packages/deepagent-code/src/control-plane/workspace.ts index bef62279e..9decf297d 100644 --- a/packages/deepagent-code/src/control-plane/workspace.ts +++ b/packages/deepagent-code/src/control-plane/workspace.ts @@ -373,6 +373,7 @@ export const layer = Layer.effect( const importArtifactMetadata = Effect.fn("Workspace.syncFilePartArtifactMetadata")(function* ( metadata: FilePartArtifact.Metadata, verified = false, + stagedImports?: FilePartArtifact.Metadata[], ) { const descriptor = metadata.descriptor const remote = verified @@ -391,6 +392,7 @@ export const layer = Layer.effect( ) if (JSON.stringify(remote) !== JSON.stringify(metadata)) return yield* new SyncHttpError({ message: "Workspace artifact metadata diverged from history", status: 409 }) + if (!stagedImports?.some((candidate) => candidate.eventID === remote.eventID)) stagedImports?.push(remote) if (!(yield* FilePartArtifact.has(descriptor).pipe(Effect.provideService(Database.Service, { db })))) for (const [index, hash] of remote.chunkHashes.entries()) { const chunk = yield* requestJson( @@ -425,7 +427,10 @@ export const layer = Layer.effect( } yield* FilePartArtifact.stageImport(remote).pipe(Effect.provideService(Database.Service, { db })) }) - const importArtifact = Effect.fn("Workspace.syncFilePartArtifact")(function* (item: HistoryEvent) { + const importArtifact = Effect.fn("Workspace.syncFilePartArtifact")(function* ( + item: HistoryEvent, + stagedImports: FilePartArtifact.Metadata[], + ) { const descriptor = FilePartArtifact.descriptor(item.data) if (!descriptor) return const metadata = Schema.decodeUnknownSync(FilePartArtifact.Metadata)( @@ -444,13 +449,18 @@ export const layer = Layer.effect( JSON.stringify(metadata.descriptor) !== JSON.stringify(descriptor) ) return yield* new SyncHttpError({ message: "Workspace artifact metadata diverged from history", status: 409 }) - yield* importArtifactMetadata(metadata, true) + yield* importArtifactMetadata(metadata, true, stagedImports) }) - const importSnapshot = Effect.fn("Workspace.syncSnapshot")(function* (item: HistoryResync) { + const importSnapshot = Effect.fn("Workspace.syncSnapshot")(function* ( + item: HistoryResync, + stagedImports: FilePartArtifact.Metadata[], + stagedSnapshots: HistoryResync["snapshot"][], + ) { if (!events.snapshotRows || !events.snapshotChunks || !events.stageSnapshotRows || !events.stageSnapshotChunks) return yield* new SyncHttpError({ message: "Workspace snapshot transfer is unavailable", status: 503 }) const stageRows = events.stageSnapshotRows const stageChunks = events.stageSnapshotChunks + stagedSnapshots.push(item.snapshot) let after = -1 while (true) { const page = Schema.decodeUnknownSync(SnapshotRowsPage)( @@ -503,7 +513,11 @@ export const layer = Layer.effect( ) if (!value || typeof value !== "object" || Array.isArray(value) || !("metadata" in value)) return yield* new SyncHttpError({ message: "Workspace snapshot artifact row is invalid", status: 409 }) - yield* importArtifactMetadata(Schema.decodeUnknownSync(FilePartArtifact.Metadata)(value.metadata)) + yield* importArtifactMetadata( + Schema.decodeUnknownSync(FilePartArtifact.Metadata)(value.metadata), + false, + stagedImports, + ) }), { discard: true }, ) @@ -563,25 +577,55 @@ export const layer = Layer.effect( yield* Effect.forEach( history, (item) => { + const stagedImports: FilePartArtifact.Metadata[] = [] + const stagedSnapshots: HistoryResync["snapshot"][] = [] return Effect.gen(function* () { if (item.kind === "resync_required") { - yield* importSnapshot(item) + yield* importSnapshot(item, stagedImports, stagedSnapshots) return } - yield* importArtifact(item) - yield* events - .replay( - { - id: EventV2.ID.make(item.id), - aggregateID: item.aggregate_id, - seq: item.seq, - type: item.type, - data: item.data, - }, - { publish: true, ownerID: space.id }, - ) - .pipe(Effect.provideService(WorkspaceRef, space.id)) - }) + yield* importArtifact(item, stagedImports) + yield* events.replay({ + id: EventV2.ID.make(item.id), + aggregateID: item.aggregate_id, + seq: item.seq, + type: item.type, + data: item.data, + }, { publish: true, ownerID: space.id }).pipe(Effect.provideService(WorkspaceRef, space.id)) + }).pipe( + Effect.ensuring( + Effect.suspend(() => Effect.gen(function* () { + yield* Effect.forEach( + stagedImports, + (metadata) => FilePartArtifact.discardImport({ + eventID: EventV2.ID.make(metadata.eventID), + aggregateID: metadata.aggregateID, + artifactID: metadata.descriptor.id, + }).pipe(Effect.provideService(Database.Service, { db }), Effect.ignore), + { discard: true }, + ) + yield* Effect.forEach( + stagedSnapshots, + (snapshot) => { + const discardImportedSnapshot = events.discardImportedSnapshot + if (!discardImportedSnapshot) return Effect.succeed(undefined) + return Effect.gen(function* () { + let complete = false + while (!complete) { + const result = yield* discardImportedSnapshot({ + snapshotID: snapshot.snapshotID, + aggregateID: snapshot.aggregateID, + limit: EventV2.SNAPSHOT_TRANSFER_ROWS, + }) + complete = result.complete + } + }).pipe(Effect.ignore) + }, + { discard: true }, + ) + })), + ), + ) }, { discard: true }, ) diff --git a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts index 4f1e845a3..e1f753a7e 100644 --- a/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts +++ b/packages/deepagent-code/src/server/routes/instance/httpapi/handlers/sync.ts @@ -290,11 +290,17 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl service: "event_sync_backfill_required", message: "Event sync index backfill is incomplete; run bounded maintenance before reading history", }) - const envelope = "version" in ctx.payload ? ctx.payload : { version: 1 as const, known: ctx.payload } - const known = envelope.known ?? {} + const versioned = "version" in ctx.payload + const known = versioned ? ctx.payload.known ?? {} : ctx.payload if (Object.keys(known).length > SyncHistoryLimits.legacyKnownAggregates) return yield* new HttpApiError.BadRequest({}) - const after = historyCursor(workspaceID, authority.generation, authority.secret, authority.seq, typeof envelope.cursor === "string" ? envelope.cursor : undefined) + const after = historyCursor( + workspaceID, + authority.generation, + authority.secret, + authority.seq, + versioned && typeof ctx.payload.cursor === "string" ? ctx.payload.cursor : undefined, + ) if (after === undefined) return yield* new ConflictError({ message: "Sync history cursor is invalid, stale, or belongs to another workspace", @@ -321,11 +327,13 @@ export const syncHandlers = HttpApiBuilder.group(InstanceHttpApi, "sync", (handl and( eq(SessionTable.workspace_id, workspaceID), gt(EventSnapshotTable.sync_seq, after), - sql`COALESCE(( - SELECT CAST(value AS INTEGER) - FROM json_each(${knownJson}) - WHERE key = ${EventSnapshotTable.aggregate_id} - ), -1) < ${EventSnapshotTable.through_seq}`, + !versioned + ? sql`COALESCE(( + SELECT CAST(value AS INTEGER) + FROM json_each(${knownJson}) + WHERE key = ${EventSnapshotTable.aggregate_id} + ), -1) < ${EventSnapshotTable.through_seq}` + : undefined, ), ) .orderBy(asc(EventSnapshotTable.sync_seq)) diff --git a/packages/deepagent-code/src/session/v4-event-runtime.ts b/packages/deepagent-code/src/session/v4-event-runtime.ts index e610b39df..7359b7341 100644 --- a/packages/deepagent-code/src/session/v4-event-runtime.ts +++ b/packages/deepagent-code/src/session/v4-event-runtime.ts @@ -44,6 +44,7 @@ import { goalStoreRoot } from "./goal-manager" import { SessionRevert } from "./revert" import { SessionSteer } from "./steer" import { EventV2Bridge } from "@/event-v2-bridge" +import { EventV2 } from "@deepagent-code/core/event" // §C3 (P2.9) — file locks + code-graph symbols. import { FileLock } from "@deepagent-code/core/file-lock" import { LocationIndexRuntime } from "@/location-index/runtime" @@ -661,15 +662,42 @@ const executionRuntimeLayer = Layer.mergeAll(runtimeLayer, handoffConsumerLayer) Layer.provide(AgentExecution.layer), ) -// The retention sweeper daemon — started only when a V4 daemon is enabled. This coupling is -// self-consistent, not a surprise: the durable event/audit tables are written ONLY by V4 publishers -// (the flag-gated IM double-write, goal-manager, agent-push), so with all V4 flags off nothing is -// written and there is nothing to prune. Turning any V4 flag on both starts writing those rows AND -// starts the 30-day sweep that bounds them — they activate together by design. +// EventV2 is written by every Session, independently of the V4 feature flags. Keep retention running +// even when the optional V4 daemons are disabled so archived legacy Sessions cannot accumulate raw +// EventV2 history indefinitely. const retentionLayer = Layer.unwrap( Effect.gen(function* () { - const flags = yield* RuntimeFlags.Service - return RetentionSweeper.layerWith({ runLoop: anyV4DaemonEnabled(flags) }) + const events = yield* EventV2Bridge.Service + return RetentionSweeper.layerWith({ + runLoop: true, + compactSession: (sessionID) => + Effect.gen(function* () { + if (!events.prepareCheckpoint || !events.stageCheckpoint || !events.finalizeCheckpoint) return false + if (!events.aggregateState) return false + const state = yield* events.aggregateState(sessionID) + if (!state || state.seq < 0) return false + let attempt = yield* events.prepareCheckpoint({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(state.seq), + expectedLatest: EventV2.Cursor.make(state.seq), + codec: "session-projection", + schemaVersion: 1, + ...(state.ownerID ? { ownerID: state.ownerID } : {}), + }) + if (attempt.state === "prepared") attempt = yield* events.stageCheckpoint({ snapshotID: attempt.snapshotID }) + if (attempt.state === "prepared") return true + const snapshot = attempt.state === "complete" + ? yield* events.snapshot(sessionID) + : yield* events.finalizeCheckpoint({ snapshotID: attempt.snapshotID }) + if (!snapshot || snapshot.snapshotID !== attempt.snapshotID) return false + const result = yield* events.compact({ + aggregateID: sessionID, + throughSeq: EventV2.Cursor.make(state.seq), + limit: 100, + }) + return result.complete + }), + }) }), ) diff --git a/packages/deepagent-code/test/server/httpapi-sync.test.ts b/packages/deepagent-code/test/server/httpapi-sync.test.ts index 7670d458c..091119c13 100644 --- a/packages/deepagent-code/test/server/httpapi-sync.test.ts +++ b/packages/deepagent-code/test/server/httpapi-sync.test.ts @@ -130,10 +130,74 @@ describe("sync HttpApi", () => { body: JSON.stringify({ version: 1, cursor: firstPage.nextCursor }), }) expect(resync.status, yield* resync.text).toBe(200) - expect(((yield* resync.json) as { items: { kind: string; snapshot?: { snapshotID: string } }[] }).items).toEqual([ + const resyncPage = (yield* resync.json) as { + items: { kind: string; snapshot?: { snapshotID: string } }[] + nextCursor: string + } + expect(resyncPage.items).toEqual([ { kind: "resync_required", snapshot: expect.objectContaining({ snapshotID: snapshot.snapshotID }) }, ]) + const { db } = yield* Database.Service + yield* db.update(SessionTable).set({ mutation_epoch: sql`${SessionTable.mutation_epoch} + 1` }) + .where(eq(SessionTable.id, session.id)).run().pipe(Effect.orDie) + const replacementPrepared = yield* requestInDirectory(SyncPaths.checkpointPrepare, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ aggregateID: session.id }), + }) + expect(replacementPrepared.status, yield* replacementPrepared.text).toBe(200) + const replacementAttempt = Schema.decodeUnknownSync( + Schema.Struct({ snapshotID: Schema.String, state: Schema.String, hasMore: Schema.Boolean }), + )(yield* replacementPrepared.json) + expect(replacementAttempt.snapshotID).not.toBe(snapshot.snapshotID) + let replacementState = replacementAttempt.state + while (replacementState === "prepared") { + const staged = yield* requestInDirectory(SyncPaths.checkpointStage, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ snapshotID: replacementAttempt.snapshotID, limit: 1 }), + }) + expect(staged.status, yield* staged.text).toBe(200) + replacementState = String(((yield* staged.json) as Record).state) + } + const replacementFinalized = yield* requestInDirectory(SyncPaths.checkpointFinalize, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ snapshotID: replacementAttempt.snapshotID }), + }) + expect(replacementFinalized.status, yield* replacementFinalized.text).toBe(200) + const later = yield* Session.use.create({ title: "event after replacement snapshot", workspaceID: syncWorkspaceID }) + + const replacementHistory = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ + version: 1, + cursor: resyncPage.nextCursor, + known: { [session.id]: snapshot.throughSeq }, + }), + }) + expect(replacementHistory.status, yield* replacementHistory.text).toBe(200) + const replacementPage = (yield* replacementHistory.json) as { + items: { kind: string; snapshot?: { snapshotID: string } }[] + nextCursor: string + complete: boolean + } + expect(replacementPage.items).toEqual([ + { kind: "resync_required", snapshot: expect.objectContaining({ snapshotID: replacementAttempt.snapshotID }) }, + ]) + expect(replacementPage.complete).toBe(false) + const afterReplacement = yield* requestInDirectory(SyncPaths.history, tmp.directory, { + method: "POST", + headers, + body: JSON.stringify({ version: 1, cursor: replacementPage.nextCursor }), + }) + expect(afterReplacement.status, yield* afterReplacement.text).toBe(200) + expect(((yield* afterReplacement.json) as HistoryResponse).items).toEqual([ + expect.objectContaining({ kind: "event", aggregate_id: later.id }), + ]) + let compacted = false while (!compacted) { const response = yield* requestInDirectory(SyncPaths.checkpointCompact, tmp.directory, { @@ -144,7 +208,6 @@ describe("sync HttpApi", () => { expect(response.status, yield* response.text).toBe(200) compacted = Boolean(((yield* response.json) as { complete: boolean }).complete) } - const { db } = yield* Database.Service expect( yield* db .select({ count: sql`count(*)` }) @@ -210,7 +273,7 @@ describe("sync HttpApi", () => { .pipe(Effect.orDie) yield* db .insert(EventSnapshotRowTable) - .values({ snapshot_id: snapshot.snapshotID, row_index: 0, table_name: "session", row_key: session.id, row_hash: rowHash, row_bytes: 2, chunk_count: 1, chain_hash: "2".repeat(64) }) + .values({ snapshot_id: snapshot.snapshotID, aggregate_id: snapshot.aggregateID, row_index: 0, table_name: "session", row_key: session.id, row_hash: rowHash, row_bytes: 2, chunk_count: 1, chain_hash: "2".repeat(64) }) .run() .pipe(Effect.orDie) yield* db From ff6e68dced2e12bc5db3bc1961196f1a7e59cde8 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Thu, 13 Aug 2026 22:20:39 +0800 Subject: [PATCH 30/44] fix(deepagent-code): validate all sessions in maintenance gate --- .../deepagent-code/script/bug-407-010-release-gate.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/deepagent-code/script/bug-407-010-release-gate.ts b/packages/deepagent-code/script/bug-407-010-release-gate.ts index 9dc2b8d54..b4fe087bb 100644 --- a/packages/deepagent-code/script/bug-407-010-release-gate.ts +++ b/packages/deepagent-code/script/bug-407-010-release-gate.ts @@ -32,16 +32,18 @@ const args = parseArgs({ options: { database: { type: "string" }, session: { type: "string", multiple: true }, + "all-sessions": { type: "boolean" }, }, strict: true, }) const filename = args.values.database ? path.resolve(args.values.database) : undefined const sessions = (args.values.session ?? []).map((sessionID) => SessionID.make(sessionID)) +const allSessions = args.values["all-sessions"] === true if (!filename || !path.basename(filename).includes("release-gate")) throw new Error("--database must name an explicit release-gate copy") if (!(await Bun.file(filename).exists())) throw new Error(`Release-gate database does not exist: ${filename}`) -if (sessions.length === 0) throw new Error("At least one --session is required") +if (!allSessions && sessions.length === 0) throw new Error("At least one --session or --all-sessions is required") const databaseLayer = Database.layerFromPath(filename) const eventLayer = EventV2.layer.pipe(Layer.provide(databaseLayer)) @@ -156,7 +158,11 @@ const program = Effect.gen(function* () { if (diffMigration.failed > 0) return yield* Effect.die(new Error(`Legacy Session diff migration failed: ${JSON.stringify(diffMigration)}`)) + const inventorySessions = allSessions + ? (yield* db.select({ id: SessionTable.id }).from(SessionTable).orderBy(asc(SessionTable.id)).all().pipe(Effect.orDie)) + : [] const checkpointSessions = [...new Set([ + ...inventorySessions.map((row) => SessionID.make(row.id)), ...sessions, ...artifactSessions.map((row) => SessionID.make(row.id)), ...fileArtifactSessions.map((row) => SessionID.make(row.id)), From 3c5d11a7cb6fbc83d56bc0d2384a5039b02ada8c Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Fri, 14 Aug 2026 00:25:26 +0800 Subject: [PATCH 31/44] feat(core): add durable authority schemas --- .../src/context-federation/session-sql.ts | 31 + packages/core/src/database/migration.gen.ts | 40 + ...7_released_knowledge_snapshot_authority.ts | 316 ++ ...260811190000_context_activation_receipt.ts | 76 + ...1193000_released_knowledge_turn_binding.ts | 375 ++ ...activity_objective_permission_authority.ts | 621 ++++ .../20260811203000_learning_job_authority.ts | 193 + .../20260811221351_w2_authority_schema.ts | 11 + ...11223000_prepared_provider_turn_receipt.ts | 227 ++ ...4500_activity_permission_terminal_fence.ts | 109 + ...1225000_activity_permission_owner_lease.ts | 33 + ...0260812005647_learning_admission_outbox.ts | 87 + ...4934_activity_permission_route_feedback.ts | 71 + ...812034116_learning_governance_authority.ts | 83 + ...812035000_learning_governance_lifecycle.ts | 323 ++ ..._provider_attempt_pre_dispatch_terminal.ts | 39 + ...20260812050000_provider_owner_authority.ts | 130 + ...053000_provider_turn_identity_authority.ts | 373 ++ ...activity_permission_authority_hardening.ts | 82 + ...812061000_provider_cross_state_recovery.ts | 193 + ...2_task_structured_output_receipt_schema.ts | 11 + ...ask_structured_output_receipt_authority.ts | 195 ++ .../20260812151010_confused_polaris.ts | 41 + ...2154657_task_structured_output_evidence.ts | 36 + ...8_task_structured_output_evidence_parts.ts | 25 + ...2_task_structured_output_evidence_fence.ts | 13 + .../20260812162033_new_titanium_man.ts | 14 + .../20260812184453_charming_anthem.ts | 29 + ...812210000_task_execution_spec_authority.ts | 212 ++ ...00_permission_effect_dispatch_authority.ts | 205 ++ ...sk_structured_output_evidence_authority.ts | 337 ++ ...3000_task_structured_finalizer_response.ts | 118 + ...2224000_learning_artifact_phase_receipt.ts | 68 + ...20260813040301_bug407_final_authorities.ts | 260 ++ ...000_learning_reviewer_attempt_authority.ts | 78 + ...00_learning_lifecycle_trigger_authority.ts | 40 + ...0813041200_v2_provider_shadow_authority.ts | 194 + ...rning_governance_compensation_authority.ts | 96 + ...0_context_activation_semantic_authority.ts | 92 + ...60813120346_v2_provider_parity_campaign.ts | 29 + ...v2_provider_parity_response_fingerprint.ts | 11 + ...0_v2_provider_parity_campaign_authority.ts | 251 ++ .../src/deepagent/activity-authority.sql.ts | 250 ++ .../learning-admission-outbox.sql.ts | 47 + .../src/deepagent/learning-governance.sql.ts | 172 + .../core/src/deepagent/learning-job.sql.ts | 105 + .../learning-lifecycle-trigger.sql.ts | 61 + .../learning-reviewer-attempt.sql.ts | 61 + .../src/deepagent/released-snapshot.sql.ts | 188 + packages/core/src/permission/sql.ts | 11 +- .../session/runner/v2-provider-turn.sql.ts | 105 + packages/core/src/session/sql.ts | 84 + packages/core/test/database-migration.test.ts | 3119 +++++++++++++++-- ...d-knowledge-turn-binding-migration.test.ts | 873 +++++ ...tured-finalizer-response-migration.test.ts | 160 + .../src/session/tool-request-receipt.sql.ts | 26 + 56 files changed, 10809 insertions(+), 221 deletions(-) create mode 100644 packages/core/src/database/migration/20260811185417_released_knowledge_snapshot_authority.ts create mode 100644 packages/core/src/database/migration/20260811190000_context_activation_receipt.ts create mode 100644 packages/core/src/database/migration/20260811193000_released_knowledge_turn_binding.ts create mode 100644 packages/core/src/database/migration/20260811200000_activity_objective_permission_authority.ts create mode 100644 packages/core/src/database/migration/20260811203000_learning_job_authority.ts create mode 100644 packages/core/src/database/migration/20260811221351_w2_authority_schema.ts create mode 100644 packages/core/src/database/migration/20260811223000_prepared_provider_turn_receipt.ts create mode 100644 packages/core/src/database/migration/20260811224500_activity_permission_terminal_fence.ts create mode 100644 packages/core/src/database/migration/20260811225000_activity_permission_owner_lease.ts create mode 100644 packages/core/src/database/migration/20260812005647_learning_admission_outbox.ts create mode 100644 packages/core/src/database/migration/20260812014934_activity_permission_route_feedback.ts create mode 100644 packages/core/src/database/migration/20260812034116_learning_governance_authority.ts create mode 100644 packages/core/src/database/migration/20260812035000_learning_governance_lifecycle.ts create mode 100644 packages/core/src/database/migration/20260812043000_provider_attempt_pre_dispatch_terminal.ts create mode 100644 packages/core/src/database/migration/20260812050000_provider_owner_authority.ts create mode 100644 packages/core/src/database/migration/20260812053000_provider_turn_identity_authority.ts create mode 100644 packages/core/src/database/migration/20260812060000_activity_permission_authority_hardening.ts create mode 100644 packages/core/src/database/migration/20260812061000_provider_cross_state_recovery.ts create mode 100644 packages/core/src/database/migration/20260812114412_task_structured_output_receipt_schema.ts create mode 100644 packages/core/src/database/migration/20260812114500_task_structured_output_receipt_authority.ts create mode 100644 packages/core/src/database/migration/20260812151010_confused_polaris.ts create mode 100644 packages/core/src/database/migration/20260812154657_task_structured_output_evidence.ts create mode 100644 packages/core/src/database/migration/20260812155138_task_structured_output_evidence_parts.ts create mode 100644 packages/core/src/database/migration/20260812155242_task_structured_output_evidence_fence.ts create mode 100644 packages/core/src/database/migration/20260812162033_new_titanium_man.ts create mode 100644 packages/core/src/database/migration/20260812184453_charming_anthem.ts create mode 100644 packages/core/src/database/migration/20260812210000_task_execution_spec_authority.ts create mode 100644 packages/core/src/database/migration/20260812213000_permission_effect_dispatch_authority.ts create mode 100644 packages/core/src/database/migration/20260812220000_task_structured_output_evidence_authority.ts create mode 100644 packages/core/src/database/migration/20260812223000_task_structured_finalizer_response.ts create mode 100644 packages/core/src/database/migration/20260812224000_learning_artifact_phase_receipt.ts create mode 100644 packages/core/src/database/migration/20260813040301_bug407_final_authorities.ts create mode 100644 packages/core/src/database/migration/20260813041000_learning_reviewer_attempt_authority.ts create mode 100644 packages/core/src/database/migration/20260813041100_learning_lifecycle_trigger_authority.ts create mode 100644 packages/core/src/database/migration/20260813041200_v2_provider_shadow_authority.ts create mode 100644 packages/core/src/database/migration/20260813041300_learning_governance_compensation_authority.ts create mode 100644 packages/core/src/database/migration/20260813041400_context_activation_semantic_authority.ts create mode 100644 packages/core/src/database/migration/20260813120346_v2_provider_parity_campaign.ts create mode 100644 packages/core/src/database/migration/20260813121129_v2_provider_parity_response_fingerprint.ts create mode 100644 packages/core/src/database/migration/20260813121200_v2_provider_parity_campaign_authority.ts create mode 100644 packages/core/src/deepagent/activity-authority.sql.ts create mode 100644 packages/core/src/deepagent/learning-admission-outbox.sql.ts create mode 100644 packages/core/src/deepagent/learning-governance.sql.ts create mode 100644 packages/core/src/deepagent/learning-job.sql.ts create mode 100644 packages/core/src/deepagent/learning-lifecycle-trigger.sql.ts create mode 100644 packages/core/src/deepagent/learning-reviewer-attempt.sql.ts create mode 100644 packages/core/src/deepagent/released-snapshot.sql.ts create mode 100644 packages/core/src/session/runner/v2-provider-turn.sql.ts create mode 100644 packages/core/test/released-knowledge-turn-binding-migration.test.ts create mode 100644 packages/core/test/task-structured-finalizer-response-migration.test.ts diff --git a/packages/core/src/context-federation/session-sql.ts b/packages/core/src/context-federation/session-sql.ts index c7afe9f48..e121658f6 100644 --- a/packages/core/src/context-federation/session-sql.ts +++ b/packages/core/src/context-federation/session-sql.ts @@ -1,6 +1,9 @@ import { blob, index, integer, primaryKey, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" import { sql } from "drizzle-orm" import { SessionInputTable, SessionTable } from "../session/sql" +import { ReleasedKnowledgeSnapshotTable } from "../deepagent/released-snapshot.sql" +import type { DocumentRef, StoredBindingState } from "../deepagent/released-snapshot" +import type { ProjectScopeKey, SecurityNamespaceID } from "./reference" export const SessionActivityTable = sqliteTable( "session_activity", @@ -60,6 +63,8 @@ export const SessionContextSelectionTable = sqliteTable( revision: integer().notNull(), trigger_input_id: text().notNull(), location_key: text().notNull(), + security_namespace_id: text().$type(), + project_scope_key: text().$type(), query_fingerprint: text().notNull(), authorization_fingerprint: text().notNull(), authorization_epoch: integer().notNull(), @@ -67,6 +72,13 @@ export const SessionContextSelectionTable = sqliteTable( selected_source_fingerprint: text().notNull(), observed_location_mutation_epoch: integer().notNull(), next_revalidation_at: integer().notNull(), + released_knowledge_binding_state: text().$type(), + released_knowledge_snapshot_id: text().references(() => ReleasedKnowledgeSnapshotTable.snapshot_id), + released_knowledge_generation: integer(), + released_knowledge_membership_hash: text(), + released_knowledge_manifest_hash: text(), + released_knowledge_exact_refs: text({ mode: "json" }).$type(), + released_knowledge_exact_refs_fingerprint: text(), graph_revisions: text().notNull(), graph_statuses: text().notNull(), selected_refs: text().notNull(), @@ -81,6 +93,9 @@ export const SessionContextSelectionTable = sqliteTable( (table) => [ uniqueIndex("session_context_selection_revision_idx").on(table.session_id, table.activity_id, table.revision), index("session_context_selection_activity_idx").on(table.session_id, table.activity_id, table.created_at), + index("session_context_selection_released_snapshot_idx") + .on(table.released_knowledge_snapshot_id, table.released_knowledge_generation) + .where(sql`${table.released_knowledge_snapshot_id} IS NOT NULL`), ], ) @@ -129,6 +144,18 @@ export const SessionContextValidationTable = sqliteTable( ], ) +export const SessionProviderOwnerLeaseTable = sqliteTable( + "session_provider_owner_lease", + { + owner_token: text().primaryKey(), + registered_at: integer().notNull(), + heartbeat_at: integer().notNull(), + lease_expires_at: integer().notNull(), + released_at: integer(), + }, + (table) => [index("session_provider_owner_lease_expiry_idx").on(table.lease_expires_at, table.owner_token)], +) + export const SessionProviderAttemptTable = sqliteTable( "session_provider_attempt", { @@ -145,7 +172,10 @@ export const SessionProviderAttemptTable = sqliteTable( .references(() => SessionContextSelectionTable.selection_id), projection_hash: text().notNull(), request_hash: text().notNull(), + prepared_turn_hash: text(), + wire_request_hash: text(), provider_id: text().notNull(), + owner_token: text().references(() => SessionProviderOwnerLeaseTable.owner_token), parent_attempt_id: text(), idempotency_key: text(), state: text() @@ -169,6 +199,7 @@ export const SessionProviderAttemptTable = sqliteTable( (table) => [ uniqueIndex("session_provider_attempt_turn_idx").on(table.session_id, table.provider_turn_seq), index("session_provider_attempt_activity_idx").on(table.session_id, table.activity_id, table.state), + index("session_provider_attempt_owner_state_idx").on(table.state, table.owner_token, table.created_at), ], ) diff --git a/packages/core/src/database/migration.gen.ts b/packages/core/src/database/migration.gen.ts index 7d9533195..3826f4e72 100644 --- a/packages/core/src/database/migration.gen.ts +++ b/packages/core/src/database/migration.gen.ts @@ -80,13 +80,53 @@ export const migrations = ( import("./migration/20260810170000_part_integrity_backfill"), import("./migration/20260811090000_legacy_activity_progress"), import("./migration/20260811100000_legacy_activity_owner"), + import("./migration/20260811185417_released_knowledge_snapshot_authority"), + import("./migration/20260811190000_context_activation_receipt"), + import("./migration/20260811193000_released_knowledge_turn_binding"), + import("./migration/20260811200000_activity_objective_permission_authority"), + import("./migration/20260811203000_learning_job_authority"), + import("./migration/20260811221351_w2_authority_schema"), + import("./migration/20260811223000_prepared_provider_turn_receipt"), + import("./migration/20260811224500_activity_permission_terminal_fence"), + import("./migration/20260811225000_activity_permission_owner_lease"), + import("./migration/20260812005647_learning_admission_outbox"), + import("./migration/20260812014934_activity_permission_route_feedback"), + import("./migration/20260812034116_learning_governance_authority"), + import("./migration/20260812035000_learning_governance_lifecycle"), + import("./migration/20260812043000_provider_attempt_pre_dispatch_terminal"), + import("./migration/20260812050000_provider_owner_authority"), + import("./migration/20260812053000_provider_turn_identity_authority"), + import("./migration/20260812060000_activity_permission_authority_hardening"), + import("./migration/20260812061000_provider_cross_state_recovery"), + import("./migration/20260812114412_task_structured_output_receipt_schema"), + import("./migration/20260812114500_task_structured_output_receipt_authority"), import("./migration/20260812120000_legacy_provider_recovery"), import("./migration/20260812130000_legacy_activity_lifecycle_expand"), import("./migration/20260812140000_session_diff_manifest"), + import("./migration/20260812151010_confused_polaris"), + import("./migration/20260812154657_task_structured_output_evidence"), + import("./migration/20260812155138_task_structured_output_evidence_parts"), + import("./migration/20260812155242_task_structured_output_evidence_fence"), + import("./migration/20260812162033_new_titanium_man"), + import("./migration/20260812184453_charming_anthem"), + import("./migration/20260812210000_task_execution_spec_authority"), + import("./migration/20260812213000_permission_effect_dispatch_authority"), + import("./migration/20260812220000_task_structured_output_evidence_authority"), + import("./migration/20260812223000_task_structured_finalizer_response"), + import("./migration/20260812224000_learning_artifact_phase_receipt"), + import("./migration/20260813040301_bug407_final_authorities"), + import("./migration/20260813041000_learning_reviewer_attempt_authority"), + import("./migration/20260813041100_learning_lifecycle_trigger_authority"), + import("./migration/20260813041200_v2_provider_shadow_authority"), + import("./migration/20260813041300_learning_governance_compensation_authority"), + import("./migration/20260813041400_context_activation_semantic_authority"), import("./migration/20260813074240_bug_407_010_maintenance"), import("./migration/20260813100000_event_snapshot_authority"), import("./migration/20260813110000_provider_recovery_authority_bridge"), import("./migration/20260813120000_legacy_provider_receipt_supersession"), + import("./migration/20260813120346_v2_provider_parity_campaign"), + import("./migration/20260813121129_v2_provider_parity_response_fingerprint"), + import("./migration/20260813121200_v2_provider_parity_campaign_authority"), import("./migration/20260813125000_event_sync_backfill_authority"), import("./migration/20260813130000_file_part_artifact"), import("./migration/20260813131000_event_snapshot_chunks"), diff --git a/packages/core/src/database/migration/20260811185417_released_knowledge_snapshot_authority.ts b/packages/core/src/database/migration/20260811185417_released_knowledge_snapshot_authority.ts new file mode 100644 index 000000000..ad1f9c4b6 --- /dev/null +++ b/packages/core/src/database/migration/20260811185417_released_knowledge_snapshot_authority.ts @@ -0,0 +1,316 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260811185417_released_knowledge_snapshot_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`released_knowledge_evaluation\` ( + \`evaluation_id\` text PRIMARY KEY, + \`security_namespace_id\` text NOT NULL, + \`project_scope_key\` text NOT NULL, + \`matrix_hash\` text NOT NULL, + \`matrix_json\` text NOT NULL, + \`document_manifest_json\` text NOT NULL, + \`baseline_ref\` text NOT NULL, + \`repetitions\` integer NOT NULL, + \`evaluator_type\` text NOT NULL, + \`evaluator_id\` text NOT NULL, + \`created_at\` integer NOT NULL, + CONSTRAINT \`fk_released_knowledge_evaluation_security_namespace_id_context_security_namespace_id_fk\` FOREIGN KEY (\`security_namespace_id\`) REFERENCES \`context_security_namespace\`(\`id\`), + CONSTRAINT \`fk_released_knowledge_evaluation_security_namespace_id_project_scope_key_context_project_scope_identity_security_namespace_id_project_scope_key_fk\` FOREIGN KEY (\`security_namespace_id\`,\`project_scope_key\`) REFERENCES \`context_project_scope_identity\`(\`security_namespace_id\`,\`project_scope_key\`), + CONSTRAINT "released_knowledge_evaluation_matrix_hash_check" CHECK(length("matrix_hash") = 64 AND "matrix_hash" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "released_knowledge_evaluation_matrix_json_check" CHECK(json_valid("matrix_json")), + CONSTRAINT "released_knowledge_evaluation_document_manifest_json_check" CHECK(json_valid("document_manifest_json") AND json_type("document_manifest_json") = 'array'), + CONSTRAINT "released_knowledge_evaluation_repetitions_check" CHECK("repetitions" > 0), + CONSTRAINT "released_knowledge_evaluation_actor_type_check" CHECK("evaluator_type" IN ('human', 'agent', 'system')) + ); + `) + yield* tx.run(` + CREATE TABLE \`released_knowledge_snapshot_document\` ( + \`snapshot_id\` text NOT NULL, + \`ordinal\` integer NOT NULL, + \`source_store\` text NOT NULL, + \`doc_id\` text NOT NULL, + \`doc_version\` integer NOT NULL, + \`doc_hash\` text NOT NULL, + \`doc_type\` text NOT NULL, + \`doc_scope\` text NOT NULL, + CONSTRAINT \`released_knowledge_snapshot_document_pk\` PRIMARY KEY(\`snapshot_id\`, \`source_store\`, \`doc_id\`), + CONSTRAINT \`fk_released_knowledge_snapshot_document_snapshot_id_released_knowledge_snapshot_snapshot_id_fk\` FOREIGN KEY (\`snapshot_id\`) REFERENCES \`released_knowledge_snapshot\`(\`snapshot_id\`) ON DELETE CASCADE, + CONSTRAINT "released_knowledge_snapshot_document_ordinal_check" CHECK("ordinal" >= 0), + CONSTRAINT "released_knowledge_snapshot_document_source_store_check" CHECK("source_store" IN ('user_global', 'project')), + CONSTRAINT "released_knowledge_snapshot_document_version_check" CHECK("doc_version" > 0), + CONSTRAINT "released_knowledge_snapshot_document_hash_check" CHECK(length("doc_hash") = 71 AND substr("doc_hash", 1, 7) = 'sha256:' AND substr("doc_hash", 8) NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "released_knowledge_snapshot_document_type_check" CHECK("doc_type" IN ('knowledge', 'strategy', 'methodology', 'memory', 'skill')) + ); + `) + yield* tx.run(` + CREATE TABLE \`released_knowledge_snapshot_head\` ( + \`security_namespace_id\` text NOT NULL, + \`project_scope_key\` text NOT NULL, + \`snapshot_id\` text, + \`generation\` integer NOT NULL, + \`updated_at\` integer NOT NULL, + CONSTRAINT \`released_knowledge_snapshot_head_pk\` PRIMARY KEY(\`security_namespace_id\`, \`project_scope_key\`), + CONSTRAINT \`fk_released_knowledge_snapshot_head_security_namespace_id_context_security_namespace_id_fk\` FOREIGN KEY (\`security_namespace_id\`) REFERENCES \`context_security_namespace\`(\`id\`), + CONSTRAINT \`fk_released_knowledge_snapshot_head_security_namespace_id_project_scope_key_context_project_scope_identity_security_namespace_id_project_scope_key_fk\` FOREIGN KEY (\`security_namespace_id\`,\`project_scope_key\`) REFERENCES \`context_project_scope_identity\`(\`security_namespace_id\`,\`project_scope_key\`), + CONSTRAINT \`fk_released_knowledge_snapshot_head_security_namespace_id_project_scope_key_snapshot_id_released_knowledge_snapshot_security_namespace_id_project_scope_key_snapshot_id_fk\` FOREIGN KEY (\`security_namespace_id\`,\`project_scope_key\`,\`snapshot_id\`) REFERENCES \`released_knowledge_snapshot\`(\`security_namespace_id\`,\`project_scope_key\`,\`snapshot_id\`), + CONSTRAINT "released_knowledge_snapshot_head_generation_check" CHECK(("snapshot_id" IS NULL AND "generation" = 0) OR ("snapshot_id" IS NOT NULL AND "generation" > 0)) + ); + `) + yield* tx.run(` + CREATE TABLE \`released_knowledge_snapshot\` ( + \`snapshot_id\` text PRIMARY KEY, + \`security_namespace_id\` text NOT NULL, + \`project_scope_key\` text NOT NULL, + \`legacy_project_id\` text NOT NULL, + \`parent_snapshot_id\` text, + \`evaluation_id\` text NOT NULL, + \`release_kind\` text NOT NULL, + \`document_count\` integer NOT NULL, + \`published_generation\` integer NOT NULL, + \`verdict\` text NOT NULL, + \`failure_reason\` text, + \`actor_type\` text NOT NULL, + \`actor_id\` text NOT NULL, + \`created_at\` integer NOT NULL, + \`finalized_at\` integer, + CONSTRAINT \`fk_released_knowledge_snapshot_security_namespace_id_context_security_namespace_id_fk\` FOREIGN KEY (\`security_namespace_id\`) REFERENCES \`context_security_namespace\`(\`id\`), + CONSTRAINT \`fk_released_knowledge_snapshot_security_namespace_id_project_scope_key_context_project_scope_identity_security_namespace_id_project_scope_key_fk\` FOREIGN KEY (\`security_namespace_id\`,\`project_scope_key\`) REFERENCES \`context_project_scope_identity\`(\`security_namespace_id\`,\`project_scope_key\`), + CONSTRAINT \`fk_released_knowledge_snapshot_security_namespace_id_project_scope_key_parent_snapshot_id_released_knowledge_snapshot_security_namespace_id_project_scope_key_snapshot_id_fk\` FOREIGN KEY (\`security_namespace_id\`,\`project_scope_key\`,\`parent_snapshot_id\`) REFERENCES \`released_knowledge_snapshot\`(\`security_namespace_id\`,\`project_scope_key\`,\`snapshot_id\`), + CONSTRAINT \`fk_released_knowledge_snapshot_security_namespace_id_project_scope_key_evaluation_id_released_knowledge_evaluation_security_namespace_id_project_scope_key_evaluation_id_fk\` FOREIGN KEY (\`security_namespace_id\`,\`project_scope_key\`,\`evaluation_id\`) REFERENCES \`released_knowledge_evaluation\`(\`security_namespace_id\`,\`project_scope_key\`,\`evaluation_id\`), + CONSTRAINT "released_knowledge_snapshot_release_kind_check" CHECK("release_kind" IN ('legacy_baseline', 'evaluated', 'rollback')), + CONSTRAINT "released_knowledge_snapshot_document_count_check" CHECK("document_count" >= 0), + CONSTRAINT "released_knowledge_snapshot_published_generation_check" CHECK(("verdict" = 'passed' AND "published_generation" > 0) OR ("verdict" = 'failed' AND "published_generation" >= 0)), + CONSTRAINT "released_knowledge_snapshot_verdict_check" CHECK("verdict" IN ('passed', 'failed')), + CONSTRAINT "released_knowledge_snapshot_failure_reason_check" CHECK(("verdict" = 'passed' AND "failure_reason" IS NULL) OR ("verdict" = 'failed' AND length(trim("failure_reason")) > 0)), + CONSTRAINT "released_knowledge_snapshot_release_chain_check" CHECK(("release_kind" = 'legacy_baseline' AND "parent_snapshot_id" IS NULL AND "verdict" = 'passed') OR ("release_kind" <> 'legacy_baseline' AND "parent_snapshot_id" IS NOT NULL)), + CONSTRAINT "released_knowledge_snapshot_evaluated_membership_check" CHECK("release_kind" <> 'evaluated' OR "verdict" = 'failed' OR "document_count" > 0), + CONSTRAINT "released_knowledge_snapshot_actor_type_check" CHECK("actor_type" IN ('human', 'agent', 'system')) + ); + `) + yield* tx.run( + `CREATE UNIQUE INDEX \`released_knowledge_evaluation_scope_identity_idx\` ON \`released_knowledge_evaluation\` (\`security_namespace_id\`,\`project_scope_key\`,\`evaluation_id\`);`, + ) + yield* tx.run( + `CREATE INDEX \`released_knowledge_evaluation_matrix_idx\` ON \`released_knowledge_evaluation\` (\`security_namespace_id\`,\`project_scope_key\`,\`matrix_hash\`);`, + ) + yield* tx.run( + `CREATE UNIQUE INDEX \`released_knowledge_snapshot_document_ordinal_idx\` ON \`released_knowledge_snapshot_document\` (\`snapshot_id\`,\`ordinal\`);`, + ) + yield* tx.run( + `CREATE UNIQUE INDEX \`released_knowledge_snapshot_scope_identity_idx\` ON \`released_knowledge_snapshot\` (\`security_namespace_id\`,\`project_scope_key\`,\`snapshot_id\`);`, + ) + yield* tx.run( + `CREATE INDEX \`released_knowledge_snapshot_parent_idx\` ON \`released_knowledge_snapshot\` (\`parent_snapshot_id\`);`, + ) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_evaluation_update_forbidden\` + BEFORE UPDATE ON \`released_knowledge_evaluation\` + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_evaluation_immutable'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_evaluation_delete_forbidden\` + BEFORE DELETE ON \`released_knowledge_evaluation\` + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_evaluation_immutable'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_scope_guard\` + BEFORE INSERT ON \`released_knowledge_snapshot\` + WHEN NOT EXISTS ( + SELECT 1 + FROM \`context_project_scope_identity\` AS \`project\` + WHERE \`project\`.\`security_namespace_id\` = NEW.\`security_namespace_id\` + AND \`project\`.\`project_scope_key\` = NEW.\`project_scope_key\` + AND \`project\`.\`observed_project_id\` = NEW.\`legacy_project_id\` + AND \`project\`.\`retired_at\` IS NULL + ) + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_legacy_scope_mismatch'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_publish_guard\` + BEFORE INSERT ON \`released_knowledge_snapshot\` + WHEN NEW.\`finalized_at\` IS NOT NULL OR NOT EXISTS ( + SELECT 1 + FROM \`released_knowledge_snapshot_head\` AS \`head\` + WHERE \`head\`.\`security_namespace_id\` = NEW.\`security_namespace_id\` + AND \`head\`.\`project_scope_key\` = NEW.\`project_scope_key\` + AND \`head\`.\`snapshot_id\` IS NEW.\`parent_snapshot_id\` + AND ( + (NEW.\`verdict\` = 'passed' AND NEW.\`published_generation\` = \`head\`.\`generation\` + 1) + OR (NEW.\`verdict\` = 'failed' AND NEW.\`published_generation\` = \`head\`.\`generation\`) + ) + ) + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_stale_parent'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_document_insert_guard\` + BEFORE INSERT ON \`released_knowledge_snapshot_document\` + WHEN NOT EXISTS ( + SELECT 1 + FROM \`released_knowledge_snapshot\` AS \`snapshot\` + WHERE \`snapshot\`.\`snapshot_id\` = NEW.\`snapshot_id\` + AND \`snapshot\`.\`finalized_at\` IS NULL + AND ( + (NEW.\`source_store\` = 'user_global' AND NEW.\`doc_scope\` = 'durable') + OR ( + NEW.\`source_store\` = 'project' + AND NEW.\`doc_scope\` = 'durable:project:' || \`snapshot\`.\`legacy_project_id\` + ) + ) + ) + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_document_scope_or_state_invalid'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_document_update_forbidden\` + BEFORE UPDATE ON \`released_knowledge_snapshot_document\` + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_document_immutable'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_document_delete_forbidden\` + BEFORE DELETE ON \`released_knowledge_snapshot_document\` + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_document_immutable'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_finalize_guard\` + BEFORE UPDATE ON \`released_knowledge_snapshot\` + WHEN NOT ( + OLD.\`finalized_at\` IS NULL + AND NEW.\`finalized_at\` IS NOT NULL + AND NEW.\`finalized_at\` >= OLD.\`created_at\` + AND NEW.\`snapshot_id\` IS OLD.\`snapshot_id\` + AND NEW.\`security_namespace_id\` IS OLD.\`security_namespace_id\` + AND NEW.\`project_scope_key\` IS OLD.\`project_scope_key\` + AND NEW.\`legacy_project_id\` IS OLD.\`legacy_project_id\` + AND NEW.\`parent_snapshot_id\` IS OLD.\`parent_snapshot_id\` + AND NEW.\`evaluation_id\` IS OLD.\`evaluation_id\` + AND NEW.\`release_kind\` IS OLD.\`release_kind\` + AND NEW.\`document_count\` IS OLD.\`document_count\` + AND NEW.\`published_generation\` IS OLD.\`published_generation\` + AND NEW.\`verdict\` IS OLD.\`verdict\` + AND NEW.\`failure_reason\` IS OLD.\`failure_reason\` + AND NEW.\`actor_type\` IS OLD.\`actor_type\` + AND NEW.\`actor_id\` IS OLD.\`actor_id\` + AND NEW.\`created_at\` IS OLD.\`created_at\` + AND ( + SELECT count(*) + FROM \`released_knowledge_snapshot_document\` + WHERE \`snapshot_id\` = OLD.\`snapshot_id\` + ) = OLD.\`document_count\` + AND ( + OLD.\`document_count\` = 0 + OR ( + SELECT min(\`ordinal\`) = 0 AND max(\`ordinal\`) = OLD.\`document_count\` - 1 + FROM \`released_knowledge_snapshot_document\` + WHERE \`snapshot_id\` = OLD.\`snapshot_id\` + ) + ) + AND NOT EXISTS ( + SELECT 1 + FROM \`released_knowledge_snapshot_document\` AS \`current_document\` + JOIN \`released_knowledge_snapshot_document\` AS \`previous_document\` + ON \`previous_document\`.\`snapshot_id\` = \`current_document\`.\`snapshot_id\` + AND \`previous_document\`.\`ordinal\` = \`current_document\`.\`ordinal\` - 1 + WHERE \`current_document\`.\`snapshot_id\` = OLD.\`snapshot_id\` + AND ( + \`previous_document\`.\`source_store\` > \`current_document\`.\`source_store\` + OR ( + \`previous_document\`.\`source_store\` = \`current_document\`.\`source_store\` + AND \`previous_document\`.\`doc_id\` >= \`current_document\`.\`doc_id\` + ) + ) + ) + AND ( + SELECT evaluation.\`document_manifest_json\` = ( + SELECT COALESCE(json_group_array(json(document_ref)), '[]') + FROM ( + SELECT json_object( + 'hash', document.\`doc_hash\`, + 'id', document.\`doc_id\`, + 'scope', document.\`doc_scope\`, + 'sourceStore', document.\`source_store\`, + 'type', document.\`doc_type\`, + 'version', document.\`doc_version\` + ) AS document_ref + FROM \`released_knowledge_snapshot_document\` document + WHERE document.\`snapshot_id\` = OLD.\`snapshot_id\` + ORDER BY document.\`ordinal\` + ) + ) + FROM \`released_knowledge_evaluation\` evaluation + WHERE evaluation.\`evaluation_id\` = OLD.\`evaluation_id\` + AND evaluation.\`security_namespace_id\` = OLD.\`security_namespace_id\` + AND evaluation.\`project_scope_key\` = OLD.\`project_scope_key\` + ) + ) + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_illegal_finalize'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_delete_forbidden\` + BEFORE DELETE ON \`released_knowledge_snapshot\` + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_immutable'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_head_insert_guard\` + BEFORE INSERT ON \`released_knowledge_snapshot_head\` + WHEN NEW.\`snapshot_id\` IS NOT NULL OR NEW.\`generation\` <> 0 + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_head_invalid_initial_state'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_head_update_guard\` + BEFORE UPDATE ON \`released_knowledge_snapshot_head\` + WHEN NOT ( + NEW.\`security_namespace_id\` IS OLD.\`security_namespace_id\` + AND NEW.\`project_scope_key\` IS OLD.\`project_scope_key\` + AND NEW.\`snapshot_id\` IS NOT NULL + AND NEW.\`snapshot_id\` IS NOT OLD.\`snapshot_id\` + AND NEW.\`generation\` = OLD.\`generation\` + 1 + AND NEW.\`updated_at\` >= OLD.\`updated_at\` + AND EXISTS ( + SELECT 1 + FROM \`released_knowledge_snapshot\` AS \`snapshot\` + WHERE \`snapshot\`.\`security_namespace_id\` = NEW.\`security_namespace_id\` + AND \`snapshot\`.\`project_scope_key\` = NEW.\`project_scope_key\` + AND \`snapshot\`.\`snapshot_id\` = NEW.\`snapshot_id\` + AND \`snapshot\`.\`parent_snapshot_id\` IS OLD.\`snapshot_id\` + AND \`snapshot\`.\`published_generation\` = NEW.\`generation\` + AND \`snapshot\`.\`verdict\` = 'passed' + AND \`snapshot\`.\`finalized_at\` IS NOT NULL + ) + ) + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_head_illegal_transition'); + END; + `) + yield* tx.run(` + CREATE TRIGGER \`released_knowledge_snapshot_head_delete_forbidden\` + BEFORE DELETE ON \`released_knowledge_snapshot_head\` + BEGIN + SELECT RAISE(ABORT, 'released_knowledge_snapshot_head_immutable'); + END; + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260811190000_context_activation_receipt.ts b/packages/core/src/database/migration/20260811190000_context_activation_receipt.ts new file mode 100644 index 000000000..e701643f0 --- /dev/null +++ b/packages/core/src/database/migration/20260811190000_context_activation_receipt.ts @@ -0,0 +1,76 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260811190000_context_activation_receipt", + up(tx) { + return Effect.gen(function* () { + yield* tx.run("ALTER TABLE session_tool_request_receipt ADD COLUMN context_selection_id TEXT") + yield* tx.run("ALTER TABLE session_tool_request_receipt ADD COLUMN context_eligibility TEXT") + yield* tx.run("ALTER TABLE session_tool_request_receipt ADD COLUMN context_readiness TEXT") + yield* tx.run("ALTER TABLE session_tool_request_receipt ADD COLUMN context_activation TEXT") + yield* tx.run("ALTER TABLE session_tool_request_receipt ADD COLUMN context_activation_fingerprint TEXT") + yield* tx.run(` + CREATE INDEX session_tool_request_receipt_context_selection_idx + ON session_tool_request_receipt (context_selection_id) + WHERE context_selection_id IS NOT NULL + `) + yield* tx.run("DROP TRIGGER IF EXISTS session_tool_request_receipt_dispatch_guard") + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_dispatch_guard + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_state = 'dispatching' AND ( + NEW.final_request_hash IS NULL OR + NEW.adapter_prepared_at IS NULL OR + NEW.prompt_epoch IS NULL OR + NEW.prompt_window_id IS NULL OR + NEW.effective_history_hash IS NULL OR + NEW.context_eligibility IS NULL OR + json_valid(NEW.context_eligibility) != 1 OR + NEW.context_readiness IS NULL OR + json_valid(NEW.context_readiness) != 1 OR + NEW.context_activation IS NULL OR + json_valid(NEW.context_activation) != 1 OR + NEW.context_activation_fingerprint IS NULL OR + length(NEW.context_activation_fingerprint) != 64 OR + NEW.context_activation_fingerprint GLOB '*[^0-9a-f]*' + ) + BEGIN + SELECT RAISE(ABORT, 'provider dispatch requires final request and context activation authority'); + END + `) + yield* tx.run("DROP TRIGGER IF EXISTS session_tool_request_receipt_binding_immutable") + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_binding_immutable + BEFORE UPDATE ON session_tool_request_receipt + WHEN NEW.receipt_id != OLD.receipt_id + OR NEW.request_ordinal != OLD.request_ordinal + OR NEW.session_id != OLD.session_id + OR NEW.user_message_id != OLD.user_message_id + OR NEW.assistant_message_id IS NOT OLD.assistant_message_id + OR NEW.provider_attempt_id IS NOT OLD.provider_attempt_id + OR NEW.provider_id != OLD.provider_id + OR NEW.model_id != OLD.model_id + OR NEW.protocol IS NOT OLD.protocol + OR NEW.context_selection_id IS NOT OLD.context_selection_id + OR NEW.context_eligibility IS NOT OLD.context_eligibility + OR NEW.context_readiness IS NOT OLD.context_readiness + OR NEW.context_activation IS NOT OLD.context_activation + OR NEW.context_activation_fingerprint IS NOT OLD.context_activation_fingerprint + OR NEW.prompt_epoch IS NOT OLD.prompt_epoch + OR NEW.prompt_window_id IS NOT OLD.prompt_window_id + OR NEW.effective_history_hash IS NOT OLD.effective_history_hash + OR NEW.world_state_baseline_hash IS NOT OLD.world_state_baseline_hash + OR NEW.request_input_hash IS NOT OLD.request_input_hash + OR NEW.owner_token IS NOT OLD.owner_token + OR NEW.created_at != OLD.created_at + OR (OLD.final_request_hash IS NOT NULL AND NEW.final_request_hash IS NOT OLD.final_request_hash) + OR (OLD.prompt_cache_key IS NOT NULL AND NEW.prompt_cache_key IS NOT OLD.prompt_cache_key) + OR (OLD.response_fingerprint IS NOT NULL AND NEW.response_fingerprint IS NOT OLD.response_fingerprint) + BEGIN + SELECT RAISE(ABORT, 'provider receipt binding is immutable'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260811193000_released_knowledge_turn_binding.ts b/packages/core/src/database/migration/20260811193000_released_knowledge_turn_binding.ts new file mode 100644 index 000000000..e3bedaf84 --- /dev/null +++ b/packages/core/src/database/migration/20260811193000_released_knowledge_turn_binding.ts @@ -0,0 +1,375 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260811193000_released_knowledge_turn_binding", + up(tx) { + return Effect.gen(function* () { + yield* addBindingColumns(tx, "session_tool_request_receipt") + yield* addBindingColumns(tx, "session_context_selection") + yield* tx.run("ALTER TABLE session_tool_request_receipt ADD COLUMN released_knowledge_security_namespace_id TEXT") + yield* tx.run("ALTER TABLE session_tool_request_receipt ADD COLUMN released_knowledge_project_scope_key TEXT") + yield* tx.run(` + ALTER TABLE session_tool_request_receipt ADD COLUMN released_knowledge_selected_refs TEXT + CHECK (released_knowledge_selected_refs IS NULL OR ( + json_valid(released_knowledge_selected_refs) = 1 AND + json_type(released_knowledge_selected_refs) = 'array' + )) + `) + yield* tx.run(` + ALTER TABLE session_tool_request_receipt ADD COLUMN released_knowledge_selected_refs_fingerprint TEXT + CHECK (released_knowledge_selected_refs_fingerprint IS NULL OR ( + length(released_knowledge_selected_refs_fingerprint) = 64 AND + released_knowledge_selected_refs_fingerprint NOT GLOB '*[^0-9a-f]*' + )) + `) + yield* tx.run("ALTER TABLE session_context_selection ADD COLUMN security_namespace_id TEXT") + yield* tx.run("ALTER TABLE session_context_selection ADD COLUMN project_scope_key TEXT") + yield* tx.run("DROP TRIGGER session_context_selection_immutable") + yield* tx.run(` + UPDATE session_tool_request_receipt + SET released_knowledge_binding_state = 'legacy_unbound' + WHERE released_knowledge_binding_state IS NULL + `) + yield* tx.run(` + UPDATE session_context_selection + SET released_knowledge_binding_state = 'legacy_unbound' + WHERE released_knowledge_binding_state IS NULL + `) + yield* tx.run(` + UPDATE session_activity + SET state = 'interrupted', settled_at = created_at + WHERE state = 'active' + AND EXISTS ( + SELECT 1 + FROM session_context_selection selection + WHERE selection.activity_id = session_activity.activity_id + AND selection.released_knowledge_binding_state = 'legacy_unbound' + ) + `) + yield* tx.run(` + CREATE TRIGGER session_context_selection_immutable + BEFORE UPDATE ON session_context_selection + BEGIN + SELECT RAISE(ABORT, 'session_context_selection is immutable'); + END + `) + yield* tx.run(` + CREATE INDEX session_tool_request_receipt_released_snapshot_idx + ON session_tool_request_receipt (released_knowledge_snapshot_id, released_knowledge_generation) + WHERE released_knowledge_snapshot_id IS NOT NULL + `) + yield* tx.run(` + CREATE INDEX session_context_selection_released_snapshot_idx + ON session_context_selection (released_knowledge_snapshot_id, released_knowledge_generation) + WHERE released_knowledge_snapshot_id IS NOT NULL + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_released_knowledge_immutable + BEFORE UPDATE ON session_tool_request_receipt + WHEN NEW.released_knowledge_security_namespace_id IS NOT OLD.released_knowledge_security_namespace_id + OR NEW.released_knowledge_project_scope_key IS NOT OLD.released_knowledge_project_scope_key + OR NEW.released_knowledge_binding_state IS NOT OLD.released_knowledge_binding_state + OR NEW.released_knowledge_snapshot_id IS NOT OLD.released_knowledge_snapshot_id + OR NEW.released_knowledge_generation IS NOT OLD.released_knowledge_generation + OR NEW.released_knowledge_membership_hash IS NOT OLD.released_knowledge_membership_hash + OR NEW.released_knowledge_manifest_hash IS NOT OLD.released_knowledge_manifest_hash + OR NEW.released_knowledge_exact_refs IS NOT OLD.released_knowledge_exact_refs + OR NEW.released_knowledge_exact_refs_fingerprint IS NOT OLD.released_knowledge_exact_refs_fingerprint + BEGIN + SELECT RAISE(ABORT, 'released knowledge receipt binding is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_released_knowledge_selected_refs_seal + BEFORE UPDATE OF released_knowledge_selected_refs, released_knowledge_selected_refs_fingerprint + ON session_tool_request_receipt + WHEN ( + NEW.released_knowledge_selected_refs IS NOT OLD.released_knowledge_selected_refs OR + NEW.released_knowledge_selected_refs_fingerprint IS NOT OLD.released_knowledge_selected_refs_fingerprint + ) AND NOT ( + OLD.provider_state = 'preparing' AND + NEW.provider_state = 'preparing' AND + OLD.released_knowledge_selected_refs IS NULL AND + OLD.released_knowledge_selected_refs_fingerprint IS NULL AND + NOT ${invalidSelectedRefs("NEW")} + ) + BEGIN + SELECT RAISE(ABORT, 'released knowledge selected refs may only be sealed once while preparing'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_authority_insert_guard + BEFORE INSERT ON session_tool_request_receipt + WHEN ${invalidReceiptAuthority("NEW", true)} + BEGIN + SELECT RAISE(ABORT, 'provider receipt requires durable context and released knowledge authority'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_released_knowledge_dispatch_guard + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_state IN ('prepared', 'dispatching') AND ${invalidReceiptAuthority("NEW", false)} + BEGIN + SELECT RAISE(ABORT, 'provider preparation and dispatch require released knowledge authority'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_context_selection_released_knowledge_insert_guard + BEFORE INSERT ON session_context_selection + WHEN ${invalidSelectionAuthority("NEW")} + BEGIN + SELECT RAISE(ABORT, 'context selection requires released knowledge authority'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration + +function addBindingColumns(tx: Parameters[0], table: string) { + return Effect.gen(function* () { + yield* tx.run(` + ALTER TABLE ${table} ADD COLUMN released_knowledge_binding_state TEXT + CHECK (released_knowledge_binding_state IS NULL OR released_knowledge_binding_state IN ('legacy_unbound', 'bound', 'unavailable')) + `) + yield* tx.run(` + ALTER TABLE ${table} ADD COLUMN released_knowledge_snapshot_id TEXT + REFERENCES released_knowledge_snapshot(snapshot_id) + `) + yield* tx.run(` + ALTER TABLE ${table} ADD COLUMN released_knowledge_generation INTEGER + CHECK (released_knowledge_generation IS NULL OR released_knowledge_generation > 0) + `) + yield* tx.run(` + ALTER TABLE ${table} ADD COLUMN released_knowledge_membership_hash TEXT + CHECK (released_knowledge_membership_hash IS NULL OR ( + length(released_knowledge_membership_hash) = 64 AND + released_knowledge_membership_hash NOT GLOB '*[^0-9a-f]*' + )) + `) + yield* tx.run(` + ALTER TABLE ${table} ADD COLUMN released_knowledge_manifest_hash TEXT + CHECK (released_knowledge_manifest_hash IS NULL OR ( + length(released_knowledge_manifest_hash) = 64 AND + released_knowledge_manifest_hash NOT GLOB '*[^0-9a-f]*' + )) + `) + yield* tx.run(` + ALTER TABLE ${table} ADD COLUMN released_knowledge_exact_refs TEXT + CHECK (released_knowledge_exact_refs IS NULL OR ( + json_valid(released_knowledge_exact_refs) = 1 AND + json_type(released_knowledge_exact_refs) = 'array' + )) + `) + yield* tx.run(` + ALTER TABLE ${table} ADD COLUMN released_knowledge_exact_refs_fingerprint TEXT + CHECK (released_knowledge_exact_refs_fingerprint IS NULL OR ( + length(released_knowledge_exact_refs_fingerprint) = 64 AND + released_knowledge_exact_refs_fingerprint NOT GLOB '*[^0-9a-f]*' + )) + `) + }) +} + +function invalidReceiptAuthority(row: string, insert: boolean) { + return `( + ${invalidReceiptScope(row)} OR + ${invalidBinding( + row, + `${row}.released_knowledge_security_namespace_id`, + `${row}.released_knowledge_project_scope_key`, + )} OR + ${invalidReceiptLinks(row)} OR + ${ + insert + ? `( + ${row}.provider_state <> 'preparing' OR + ${row}.released_knowledge_selected_refs IS NOT NULL OR + ${row}.released_knowledge_selected_refs_fingerprint IS NOT NULL + )` + : `(${invalidContextActivation(row)} OR ${invalidSelectedRefs(row)})` + } + )` +} + +function invalidReceiptScope(row: string) { + return `( + ${row}.released_knowledge_security_namespace_id IS NULL OR + ${row}.released_knowledge_project_scope_key IS NULL OR + NOT EXISTS ( + SELECT 1 + FROM context_project_scope_identity project + WHERE project.security_namespace_id = ${row}.released_knowledge_security_namespace_id + AND project.project_scope_key = ${row}.released_knowledge_project_scope_key + AND project.retired_at IS NULL + ) + )` +} + +function invalidSelectionAuthority(row: string) { + return `( + ${row}.security_namespace_id IS NULL OR + ${row}.project_scope_key IS NULL OR + NOT EXISTS ( + SELECT 1 + FROM context_location_identity location + WHERE location.security_namespace_id = ${row}.security_namespace_id + AND location.project_scope_key = ${row}.project_scope_key + AND location.location_key = ${row}.location_key + AND location.retired_at IS NULL + ) OR + ${invalidBinding(row, `${row}.security_namespace_id`, `${row}.project_scope_key`)} + )` +} + +function invalidReceiptLinks(row: string) { + return `( + (${row}.context_selection_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 + FROM session_context_selection selection + WHERE selection.selection_id = ${row}.context_selection_id + AND selection.session_id = ${row}.session_id + AND selection.security_namespace_id IS ${row}.released_knowledge_security_namespace_id + AND selection.project_scope_key IS ${row}.released_knowledge_project_scope_key + AND selection.released_knowledge_binding_state IS ${row}.released_knowledge_binding_state + AND selection.released_knowledge_snapshot_id IS ${row}.released_knowledge_snapshot_id + AND selection.released_knowledge_generation IS ${row}.released_knowledge_generation + AND selection.released_knowledge_membership_hash IS ${row}.released_knowledge_membership_hash + AND selection.released_knowledge_manifest_hash IS ${row}.released_knowledge_manifest_hash + AND selection.released_knowledge_exact_refs IS ${row}.released_knowledge_exact_refs + AND selection.released_knowledge_exact_refs_fingerprint IS ${row}.released_knowledge_exact_refs_fingerprint + )) OR + (${row}.provider_attempt_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 + FROM session_provider_attempt attempt + WHERE attempt.attempt_id = ${row}.provider_attempt_id + AND attempt.session_id = ${row}.session_id + AND attempt.selection_id IS ${row}.context_selection_id + )) + )` +} + +function invalidContextActivation(row: string) { + return `( + ${row}.final_request_hash IS NULL OR + ${row}.adapter_prepared_at IS NULL OR + ${row}.prompt_epoch IS NULL OR + ${row}.prompt_window_id IS NULL OR + ${row}.effective_history_hash IS NULL OR + ${row}.context_eligibility IS NULL OR + json_valid(${row}.context_eligibility) != 1 OR + ${row}.context_readiness IS NULL OR + json_valid(${row}.context_readiness) != 1 OR + ${row}.context_activation IS NULL OR + json_valid(${row}.context_activation) != 1 OR + ${row}.context_activation_fingerprint IS NULL OR + length(${row}.context_activation_fingerprint) != 64 OR + ${row}.context_activation_fingerprint GLOB '*[^0-9a-f]*' + )` +} + +function invalidBinding(row: string, securityNamespaceId: string, projectScopeKey: string) { + return `( + ${row}.released_knowledge_binding_state IS NULL OR + ${row}.released_knowledge_binding_state = 'legacy_unbound' OR + ${row}.released_knowledge_exact_refs IS NULL OR + ${row}.released_knowledge_exact_refs_fingerprint IS NULL OR + (${row}.released_knowledge_binding_state = 'unavailable' AND ( + ${row}.released_knowledge_snapshot_id IS NOT NULL OR + ${row}.released_knowledge_generation IS NOT NULL OR + ${row}.released_knowledge_membership_hash IS NOT NULL OR + ${row}.released_knowledge_manifest_hash IS NOT NULL OR + ${row}.released_knowledge_exact_refs <> '[]' OR + ${row}.released_knowledge_exact_refs_fingerprint <> '4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945' + )) OR + (${row}.released_knowledge_binding_state = 'bound' AND ( + ${row}.released_knowledge_snapshot_id IS NULL OR + ${row}.released_knowledge_generation IS NULL OR + ${row}.released_knowledge_membership_hash IS NULL OR + ${row}.released_knowledge_manifest_hash IS NULL OR + ${row}.released_knowledge_exact_refs_fingerprint <> ${row}.released_knowledge_membership_hash OR + NOT EXISTS ( + SELECT 1 + FROM released_knowledge_snapshot snapshot + WHERE snapshot.snapshot_id = ${row}.released_knowledge_snapshot_id + AND snapshot.security_namespace_id = ${securityNamespaceId} + AND snapshot.project_scope_key = ${projectScopeKey} + AND snapshot.published_generation = ${row}.released_knowledge_generation + AND snapshot.verdict = 'passed' + AND snapshot.finalized_at IS NOT NULL + AND snapshot.document_count = json_array_length(${row}.released_knowledge_exact_refs) + ) OR + EXISTS ( + SELECT 1 + FROM json_each(${row}.released_knowledge_exact_refs) exact_ref + WHERE json_type(exact_ref.value) <> 'object' + OR (SELECT count(*) FROM json_each(exact_ref.value)) <> 6 + OR EXISTS ( + SELECT 1 + FROM json_each(exact_ref.value) field + WHERE field.key NOT IN ('sourceStore', 'id', 'version', 'hash', 'type', 'scope') + ) + OR NOT EXISTS ( + SELECT 1 + FROM released_knowledge_snapshot_document document + WHERE document.snapshot_id = ${row}.released_knowledge_snapshot_id + AND document.ordinal = CAST(exact_ref.key AS INTEGER) + AND document.source_store = json_extract(exact_ref.value, '$.sourceStore') + AND document.doc_id = json_extract(exact_ref.value, '$.id') + AND document.doc_version = json_extract(exact_ref.value, '$.version') + AND document.doc_hash = json_extract(exact_ref.value, '$.hash') + AND document.doc_type = json_extract(exact_ref.value, '$.type') + AND document.doc_scope = json_extract(exact_ref.value, '$.scope') + ) + ) + )) + )` +} + +function invalidSelectedRefs(row: string) { + return `( + ${row}.released_knowledge_selected_refs IS NULL OR + ${row}.released_knowledge_selected_refs_fingerprint IS NULL OR + length(${row}.released_knowledge_selected_refs_fingerprint) <> 64 OR + ${row}.released_knowledge_selected_refs_fingerprint GLOB '*[^0-9a-f]*' OR + (${row}.released_knowledge_binding_state = 'unavailable' AND ( + ${row}.released_knowledge_selected_refs <> '[]' OR + ${row}.released_knowledge_selected_refs_fingerprint <> '4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945' + )) OR + (${row}.released_knowledge_binding_state = 'bound' AND ( + EXISTS ( + SELECT 1 + FROM json_each(${row}.released_knowledge_selected_refs) selected_ref + WHERE json_type(selected_ref.value) <> 'object' + OR (SELECT count(*) FROM json_each(selected_ref.value)) <> 6 + OR EXISTS ( + SELECT 1 + FROM json_each(selected_ref.value) field + WHERE field.key NOT IN ('sourceStore', 'id', 'version', 'hash', 'type', 'scope') + ) + OR NOT EXISTS ( + SELECT 1 + FROM released_knowledge_snapshot_document document + WHERE document.snapshot_id = ${row}.released_knowledge_snapshot_id + AND document.source_store = json_extract(selected_ref.value, '$.sourceStore') + AND document.doc_id = json_extract(selected_ref.value, '$.id') + AND document.doc_version = json_extract(selected_ref.value, '$.version') + AND document.doc_hash = json_extract(selected_ref.value, '$.hash') + AND document.doc_type = json_extract(selected_ref.value, '$.type') + AND document.doc_scope = json_extract(selected_ref.value, '$.scope') + ) + ) OR + EXISTS ( + SELECT 1 + FROM json_each(${row}.released_knowledge_selected_refs) current_ref + JOIN json_each(${row}.released_knowledge_selected_refs) previous_ref + ON CAST(previous_ref.key AS INTEGER) = CAST(current_ref.key AS INTEGER) - 1 + WHERE json_extract(previous_ref.value, '$.sourceStore') > json_extract(current_ref.value, '$.sourceStore') + OR ( + json_extract(previous_ref.value, '$.sourceStore') = json_extract(current_ref.value, '$.sourceStore') AND + json_extract(previous_ref.value, '$.id') >= json_extract(current_ref.value, '$.id') + ) + ) + )) + )` +} diff --git a/packages/core/src/database/migration/20260811200000_activity_objective_permission_authority.ts b/packages/core/src/database/migration/20260811200000_activity_objective_permission_authority.ts new file mode 100644 index 000000000..fd5bd6917 --- /dev/null +++ b/packages/core/src/database/migration/20260811200000_activity_objective_permission_authority.ts @@ -0,0 +1,621 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260811200000_activity_objective_permission_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE session_activity_objective ( + activity_kind TEXT NOT NULL CHECK (activity_kind IN ('legacy', 'v2')), + activity_id TEXT NOT NULL, + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + version INTEGER NOT NULL CHECK (version >= 1), + admission_fingerprint TEXT NOT NULL, + objective_fingerprint TEXT, + objective_text TEXT, + completion_criteria TEXT NOT NULL, + enforcement_state TEXT NOT NULL CHECK (enforcement_state IN ('disabled', 'monitoring')), + stall_threshold INTEGER CHECK (stall_threshold IS NULL OR stall_threshold > 0), + state TEXT NOT NULL CHECK (state IN ('active', 'completed', 'needs_human', 'interrupted', 'recovery_required')), + no_progress_count INTEGER NOT NULL CHECK (no_progress_count >= 0), + latest_observation_revision INTEGER NOT NULL CHECK (latest_observation_revision >= -1), + latest_vector_hash TEXT, + next_action TEXT, + terminal_reason TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + settled_at INTEGER, + PRIMARY KEY (activity_kind, activity_id), + CHECK ( + (enforcement_state = 'disabled' AND stall_threshold IS NULL) OR + (enforcement_state = 'monitoring' AND stall_threshold IS NOT NULL) + ), + CHECK ( + (state = 'active' AND settled_at IS NULL AND terminal_reason IS NULL) OR + (state != 'active' AND settled_at IS NOT NULL AND terminal_reason IS NOT NULL) + ), + CHECK ( + (latest_observation_revision = -1 AND latest_vector_hash IS NULL) OR + (latest_observation_revision >= 0 AND latest_vector_hash IS NOT NULL) + ) + ) + `) + yield* tx.run(` + CREATE INDEX session_activity_objective_session_idx + ON session_activity_objective(session_id, state, updated_at) + `) + yield* tx.run(` + CREATE TRIGGER session_activity_objective_validate_insert + BEFORE INSERT ON session_activity_objective + BEGIN + SELECT CASE WHEN NEW.activity_kind = 'legacy' AND NOT EXISTS ( + SELECT 1 FROM session_legacy_activity activity + WHERE activity.activity_id = NEW.activity_id + AND activity.session_id = NEW.session_id + ) THEN RAISE(ABORT, 'legacy activity objective owner mismatch') END; + SELECT CASE WHEN NEW.activity_kind = 'v2' AND NOT EXISTS ( + SELECT 1 FROM session_activity activity + WHERE activity.activity_id = NEW.activity_id + AND activity.session_id = NEW.session_id + ) THEN RAISE(ABORT, 'v2 activity objective owner mismatch') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_objective_legal_update + BEFORE UPDATE ON session_activity_objective + WHEN NEW.activity_kind != OLD.activity_kind + OR NEW.activity_id != OLD.activity_id + OR NEW.session_id != OLD.session_id + OR NEW.admission_fingerprint != OLD.admission_fingerprint + OR NEW.created_at != OLD.created_at + OR NEW.version != OLD.version + 1 + OR NEW.updated_at < OLD.updated_at + OR (OLD.objective_fingerprint IS NOT NULL AND ( + NEW.objective_fingerprint IS NULL + OR NEW.objective_fingerprint != OLD.objective_fingerprint + OR COALESCE(NEW.objective_text, '') != COALESCE(OLD.objective_text, '') + OR NEW.completion_criteria != OLD.completion_criteria + OR NEW.enforcement_state != OLD.enforcement_state + OR COALESCE(NEW.stall_threshold, -1) != COALESCE(OLD.stall_threshold, -1) + )) + OR OLD.state IN ('completed', 'interrupted') + OR (OLD.state = 'recovery_required' AND NEW.state != 'recovery_required') + OR (OLD.state = 'needs_human' AND NEW.state NOT IN ('needs_human', 'active', 'interrupted', 'recovery_required')) + OR (OLD.state = 'active' AND NEW.state NOT IN ('active', 'completed', 'needs_human', 'interrupted', 'recovery_required')) + BEGIN + SELECT RAISE(ABORT, 'illegal session_activity_objective transition'); + END + `) + yield* tx.run(` + INSERT INTO session_activity_objective ( + activity_kind, activity_id, session_id, version, admission_fingerprint, objective_fingerprint, objective_text, + completion_criteria, enforcement_state, stall_threshold, state, no_progress_count, + latest_observation_revision, latest_vector_hash, next_action, terminal_reason, + created_at, updated_at, settled_at + ) + SELECT + 'legacy', + activity.activity_id, + activity.session_id, + 1, + admission.payload_fingerprint, + NULL, + NULL, + '[]', + 'disabled', + NULL, + CASE + WHEN activity.state = 'settled' THEN 'completed' + WHEN activity.state = 'failed' THEN 'recovery_required' + ELSE activity.state + END, + 0, + -1, + NULL, + NULL, + activity.terminal_reason, + activity.created_at, + COALESCE(activity.settled_at, activity.created_at), + activity.settled_at + FROM session_legacy_activity activity + JOIN session_activity_admission admission + ON admission.admission_id = activity.trigger_admission_id + `) + yield* tx.run(` + INSERT INTO session_activity_objective ( + activity_kind, activity_id, session_id, version, admission_fingerprint, objective_fingerprint, objective_text, + completion_criteria, enforcement_state, stall_threshold, state, no_progress_count, + latest_observation_revision, latest_vector_hash, next_action, terminal_reason, + created_at, updated_at, settled_at + ) + SELECT + 'v2', + activity.activity_id, + activity.session_id, + 1, + 'session-input:' || activity.trigger_input_id, + NULL, + NULL, + '[]', + 'disabled', + NULL, + CASE + WHEN activity.state = 'settled' THEN 'completed' + WHEN activity.state = 'failed' THEN 'recovery_required' + ELSE activity.state + END, + 0, + -1, + NULL, + NULL, + CASE WHEN activity.state = 'active' THEN NULL ELSE activity.state END, + activity.created_at, + COALESCE(activity.settled_at, activity.created_at), + activity.settled_at + FROM session_activity activity + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_objective_insert + AFTER INSERT ON session_legacy_activity + BEGIN + INSERT INTO session_activity_objective ( + activity_kind, activity_id, session_id, version, admission_fingerprint, objective_fingerprint, objective_text, + completion_criteria, enforcement_state, stall_threshold, state, no_progress_count, + latest_observation_revision, latest_vector_hash, next_action, terminal_reason, + created_at, updated_at, settled_at + ) + SELECT + 'legacy', NEW.activity_id, NEW.session_id, 1, admission.payload_fingerprint, NULL, NULL, + '[]', 'disabled', NULL, + CASE + WHEN NEW.state = 'settled' THEN 'completed' + WHEN NEW.state = 'failed' THEN 'recovery_required' + ELSE NEW.state + END, + 0, -1, NULL, NULL, NEW.terminal_reason, NEW.created_at, + COALESCE(NEW.settled_at, NEW.created_at), NEW.settled_at + FROM session_activity_admission admission + WHERE admission.admission_id = NEW.trigger_admission_id; + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_activity_objective_insert + AFTER INSERT ON session_activity + BEGIN + INSERT INTO session_activity_objective ( + activity_kind, activity_id, session_id, version, admission_fingerprint, objective_fingerprint, objective_text, + completion_criteria, enforcement_state, stall_threshold, state, no_progress_count, + latest_observation_revision, latest_vector_hash, next_action, terminal_reason, + created_at, updated_at, settled_at + ) VALUES ( + 'v2', NEW.activity_id, NEW.session_id, 1, 'session-input:' || NEW.trigger_input_id, NULL, NULL, + '[]', 'disabled', NULL, + CASE + WHEN NEW.state = 'settled' THEN 'completed' + WHEN NEW.state = 'failed' THEN 'recovery_required' + ELSE NEW.state + END, + 0, -1, NULL, NULL, + CASE WHEN NEW.state = 'active' THEN NULL ELSE NEW.state END, + NEW.created_at, COALESCE(NEW.settled_at, NEW.created_at), NEW.settled_at + ); + END + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_objective_terminal_projection + AFTER UPDATE OF state ON session_legacy_activity + WHEN OLD.state = 'active' AND NEW.state != 'active' + BEGIN + UPDATE session_activity_objective + SET version = version + 1, + state = CASE + WHEN NEW.state = 'settled' THEN 'completed' + WHEN NEW.state = 'failed' THEN 'recovery_required' + ELSE NEW.state + END, + terminal_reason = NEW.terminal_reason, + updated_at = NEW.settled_at, + settled_at = NEW.settled_at + WHERE activity_kind = 'legacy' + AND activity_id = NEW.activity_id + AND enforcement_state = 'disabled' + AND state = 'active'; + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_activity_objective_terminal_projection + AFTER UPDATE OF state ON session_activity + WHEN OLD.state = 'active' AND NEW.state != 'active' + BEGIN + UPDATE session_activity_objective + SET version = version + 1, + state = CASE + WHEN NEW.state = 'settled' THEN 'completed' + WHEN NEW.state = 'failed' THEN 'recovery_required' + ELSE NEW.state + END, + terminal_reason = NEW.state, + updated_at = NEW.settled_at, + settled_at = NEW.settled_at + WHERE activity_kind = 'v2' + AND activity_id = NEW.activity_id + AND enforcement_state = 'disabled' + AND state = 'active'; + END + `) + + yield* tx.run(` + CREATE TABLE session_activity_evidence ( + activity_kind TEXT NOT NULL CHECK (activity_kind IN ('legacy', 'v2')), + activity_id TEXT NOT NULL, + evidence_fingerprint TEXT NOT NULL, + evidence_kind TEXT NOT NULL, + source_receipt_id TEXT, + first_observation_revision INTEGER NOT NULL CHECK (first_observation_revision >= 0), + created_at INTEGER NOT NULL, + PRIMARY KEY (activity_kind, activity_id, evidence_fingerprint), + FOREIGN KEY (activity_kind, activity_id) + REFERENCES session_activity_objective(activity_kind, activity_id) ON DELETE CASCADE + ) + `) + yield* tx.run(` + CREATE INDEX session_activity_evidence_activity_idx + ON session_activity_evidence(activity_kind, activity_id, first_observation_revision) + `) + yield* tx.run(` + CREATE TRIGGER session_activity_evidence_validate_insert + BEFORE INSERT ON session_activity_evidence + WHEN NOT EXISTS ( + SELECT 1 FROM session_activity_objective objective + WHERE objective.activity_kind = NEW.activity_kind + AND objective.activity_id = NEW.activity_id + AND objective.state = 'active' + AND NEW.first_observation_revision = objective.latest_observation_revision + 1 + ) + BEGIN + SELECT RAISE(ABORT, 'activity evidence does not match next active observation'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_evidence_immutable + BEFORE UPDATE ON session_activity_evidence + BEGIN + SELECT RAISE(ABORT, 'session_activity_evidence is immutable'); + END + `) + + yield* tx.run(` + CREATE TABLE session_activity_effect_receipt ( + activity_kind TEXT NOT NULL CHECK (activity_kind IN ('legacy', 'v2')), + activity_id TEXT NOT NULL, + receipt_id TEXT NOT NULL, + effect_fingerprint TEXT NOT NULL, + first_observation_revision INTEGER NOT NULL CHECK (first_observation_revision >= 0), + created_at INTEGER NOT NULL, + PRIMARY KEY (activity_kind, activity_id, receipt_id), + FOREIGN KEY (activity_kind, activity_id) + REFERENCES session_activity_objective(activity_kind, activity_id) ON DELETE CASCADE + ) + `) + yield* tx.run(` + CREATE INDEX session_activity_effect_receipt_activity_idx + ON session_activity_effect_receipt(activity_kind, activity_id, first_observation_revision) + `) + yield* tx.run(` + CREATE TRIGGER session_activity_effect_receipt_validate_insert + BEFORE INSERT ON session_activity_effect_receipt + WHEN NOT EXISTS ( + SELECT 1 FROM session_activity_objective objective + WHERE objective.activity_kind = NEW.activity_kind + AND objective.activity_id = NEW.activity_id + AND objective.state = 'active' + AND NEW.first_observation_revision = objective.latest_observation_revision + 1 + ) + BEGIN + SELECT RAISE(ABORT, 'activity effect receipt does not match next active observation'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_effect_receipt_immutable + BEFORE UPDATE ON session_activity_effect_receipt + BEGIN + SELECT RAISE(ABORT, 'session_activity_effect_receipt is immutable'); + END + `) + + yield* tx.run(` + CREATE TABLE session_activity_progress_observation ( + activity_kind TEXT NOT NULL CHECK (activity_kind IN ('legacy', 'v2')), + activity_id TEXT NOT NULL, + revision INTEGER NOT NULL CHECK (revision >= 0), + idempotency_key TEXT NOT NULL UNIQUE, + observation_fingerprint TEXT NOT NULL, + expected_objective_version INTEGER NOT NULL CHECK (expected_objective_version >= 1), + workspace_revision TEXT, + plan_version INTEGER, + validation_fingerprint TEXT, + evidence_set_hash TEXT NOT NULL, + effect_receipt_set_hash TEXT NOT NULL, + vector_hash TEXT NOT NULL, + next_action TEXT, + changed INTEGER NOT NULL CHECK (changed IN (0, 1)), + no_progress_count INTEGER NOT NULL CHECK (no_progress_count >= 0), + observed_at INTEGER NOT NULL, + PRIMARY KEY (activity_kind, activity_id, revision), + FOREIGN KEY (activity_kind, activity_id) + REFERENCES session_activity_objective(activity_kind, activity_id) ON DELETE CASCADE + ) + `) + yield* tx.run(` + CREATE INDEX session_activity_progress_observation_latest_idx + ON session_activity_progress_observation(activity_kind, activity_id, observed_at) + `) + yield* tx.run(` + CREATE TRIGGER session_activity_progress_observation_validate_insert + BEFORE INSERT ON session_activity_progress_observation + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_activity_objective objective + WHERE objective.activity_kind = NEW.activity_kind + AND objective.activity_id = NEW.activity_id + AND objective.state = 'active' + AND objective.version = NEW.expected_objective_version + AND NEW.revision = objective.latest_observation_revision + 1 + ) THEN RAISE(ABORT, 'activity observation objective CAS mismatch') END; + SELECT CASE WHEN NEW.revision = 0 AND (NEW.changed != 1 OR NEW.no_progress_count != 0) + THEN RAISE(ABORT, 'initial activity observation must establish progress') END; + SELECT CASE WHEN NEW.revision > 0 AND NOT EXISTS ( + SELECT 1 FROM session_activity_progress_observation previous + WHERE previous.activity_kind = NEW.activity_kind + AND previous.activity_id = NEW.activity_id + AND previous.revision = NEW.revision - 1 + ) THEN RAISE(ABORT, 'activity observation revision is not continuous') END; + SELECT CASE WHEN NEW.revision > 0 AND NOT EXISTS ( + SELECT 1 FROM session_activity_objective objective + WHERE objective.activity_kind = NEW.activity_kind + AND objective.activity_id = NEW.activity_id + AND ( + (NEW.vector_hash != objective.latest_vector_hash AND NEW.changed = 1 AND NEW.no_progress_count = 0) OR + (NEW.vector_hash = objective.latest_vector_hash AND NEW.changed = 0 + AND NEW.no_progress_count = objective.no_progress_count + 1) + ) + ) THEN RAISE(ABORT, 'activity observation progress classification mismatch') END; + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_progress_observation_immutable + BEFORE UPDATE ON session_activity_progress_observation + BEGIN + SELECT RAISE(ABORT, 'session_activity_progress_observation is immutable'); + END + `) + + yield* tx.run(` + CREATE TABLE permission_saved_epoch ( + project_id TEXT PRIMARY KEY REFERENCES project(id) ON DELETE CASCADE, + epoch INTEGER NOT NULL CHECK (epoch >= 0), + updated_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + INSERT INTO permission_saved_epoch (project_id, epoch, updated_at) + SELECT id, 0, CAST(strftime('%s', 'now') AS INTEGER) * 1000 FROM project + `) + yield* tx.run(` + CREATE TRIGGER permission_saved_epoch_project_insert + AFTER INSERT ON project + BEGIN + INSERT INTO permission_saved_epoch (project_id, epoch, updated_at) + VALUES (NEW.id, 0, CAST(strftime('%s', 'now') AS INTEGER) * 1000); + END + `) + yield* tx.run(` + CREATE TRIGGER permission_saved_epoch_legal_update + BEFORE UPDATE ON permission_saved_epoch + WHEN NEW.project_id != OLD.project_id OR NEW.epoch != OLD.epoch + 1 OR NEW.updated_at < OLD.updated_at + BEGIN + SELECT RAISE(ABORT, 'illegal permission_saved_epoch transition'); + END + `) + + yield* tx.run(` + CREATE TABLE session_activity_permission_request ( + request_id TEXT PRIMARY KEY, + activity_kind TEXT NOT NULL CHECK (activity_kind IN ('legacy', 'v2')), + activity_id TEXT NOT NULL, + session_id TEXT NOT NULL REFERENCES session(id) ON DELETE CASCADE, + project_id TEXT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + request_kind TEXT NOT NULL CHECK (request_kind IN ('tool', 'no_progress')), + idempotency_key TEXT NOT NULL UNIQUE, + permission TEXT NOT NULL, + patterns TEXT NOT NULL, + always_patterns TEXT NOT NULL, + metadata_hash TEXT NOT NULL, + tool_message_id TEXT, + tool_call_id TEXT, + state TEXT NOT NULL CHECK (state IN ('pending', 'approved_once', 'approved_always', 'denied', 'expired', 'interrupted')), + authority_epoch INTEGER NOT NULL CHECK (authority_epoch >= 0), + requested_scope TEXT NOT NULL CHECK (requested_scope IN ('once', 'project')), + owner_type TEXT NOT NULL CHECK (owner_type IN ('runtime', 'system')), + owner_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER, + decided_at INTEGER, + FOREIGN KEY (activity_kind, activity_id) + REFERENCES session_activity_objective(activity_kind, activity_id) ON DELETE CASCADE, + CHECK ( + (state = 'pending' AND decided_at IS NULL) OR + (state != 'pending' AND decided_at IS NOT NULL) + ), + CHECK (expires_at IS NULL OR expires_at > created_at) + ) + `) + yield* tx.run(` + CREATE INDEX session_activity_permission_request_pending_idx + ON session_activity_permission_request(session_id, state, created_at) + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_activity_permission_request_pending_no_progress_idx + ON session_activity_permission_request(activity_kind, activity_id) + WHERE state = 'pending' AND request_kind = 'no_progress' + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_request_validate_insert + BEFORE INSERT ON session_activity_permission_request + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_activity_objective objective + JOIN session session ON session.id = objective.session_id + WHERE objective.activity_kind = NEW.activity_kind + AND objective.activity_id = NEW.activity_id + AND objective.session_id = NEW.session_id + AND session.project_id = NEW.project_id + AND ( + (NEW.request_kind = 'tool' AND objective.state = 'active') OR + (NEW.request_kind = 'no_progress' AND objective.state = 'needs_human') + ) + ) THEN RAISE(ABORT, 'permission request activity owner mismatch') END; + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM permission_saved_epoch epoch + WHERE epoch.project_id = NEW.project_id AND epoch.epoch = NEW.authority_epoch + ) THEN RAISE(ABORT, 'permission request authority epoch mismatch') END; + SELECT CASE WHEN + (NEW.request_kind = 'tool' AND (NEW.tool_message_id IS NULL OR NEW.tool_call_id IS NULL)) OR + (NEW.request_kind = 'no_progress' AND (NEW.tool_message_id IS NOT NULL OR NEW.tool_call_id IS NOT NULL)) + THEN RAISE(ABORT, 'permission request tool identity mismatch') END; + END + `) + + yield* tx.run(` + CREATE TABLE session_activity_permission_decision ( + decision_id TEXT PRIMARY KEY, + request_id TEXT NOT NULL UNIQUE REFERENCES session_activity_permission_request(request_id) ON DELETE CASCADE, + idempotency_key TEXT NOT NULL UNIQUE, + decision TEXT NOT NULL CHECK (decision IN ('approved_once', 'approved_always', 'denied', 'expired', 'interrupted')), + actor_type TEXT NOT NULL CHECK (actor_type IN ('user', 'administrator', 'system')), + actor_id TEXT NOT NULL, + scope TEXT NOT NULL CHECK (scope IN ('once', 'project')), + authority_epoch INTEGER NOT NULL CHECK (authority_epoch >= 0), + decided_at INTEGER NOT NULL, + expires_at INTEGER, + CHECK (expires_at IS NULL OR expires_at > decided_at), + CHECK (decision != 'approved_always' OR expires_at IS NULL) + ) + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_decision_validate_insert + BEFORE INSERT ON session_activity_permission_decision + BEGIN + SELECT CASE WHEN NOT EXISTS ( + SELECT 1 FROM session_activity_permission_request request + WHERE request.request_id = NEW.request_id + AND request.state = 'pending' + AND ( + (NEW.decision = 'expired' AND request.expires_at IS NOT NULL AND request.expires_at <= NEW.decided_at) OR + (NEW.decision != 'expired' AND (request.expires_at IS NULL OR request.expires_at > NEW.decided_at)) + ) + AND ( + (NEW.decision = 'approved_always' AND NEW.scope = 'project' + AND NEW.authority_epoch >= request.authority_epoch + AND EXISTS ( + SELECT 1 FROM permission_saved_epoch epoch + WHERE epoch.project_id = request.project_id AND epoch.epoch = NEW.authority_epoch + ) + AND NOT EXISTS ( + SELECT 1 FROM json_each(request.always_patterns) pattern + WHERE NOT EXISTS ( + SELECT 1 FROM permission saved + WHERE saved.project_id = request.project_id + AND saved.action = request.permission + AND saved.resource = pattern.value + ) + )) OR + (NEW.decision != 'approved_always' AND NEW.authority_epoch = request.authority_epoch) + ) + ) THEN RAISE(ABORT, 'permission decision request CAS mismatch') END; + SELECT CASE WHEN + (NEW.decision = 'approved_once' AND NEW.scope != 'once') OR + (NEW.decision = 'approved_always' AND NEW.scope != 'project') OR + (NEW.decision IN ('denied', 'expired', 'interrupted') AND NEW.scope != request.requested_scope) + THEN RAISE(ABORT, 'permission decision scope mismatch') END + FROM session_activity_permission_request request + WHERE request.request_id = NEW.request_id; + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_decision_immutable + BEFORE UPDATE ON session_activity_permission_decision + BEGIN + SELECT RAISE(ABORT, 'session_activity_permission_decision is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_request_legal_update + BEFORE UPDATE ON session_activity_permission_request + WHEN NEW.request_id != OLD.request_id + OR NEW.activity_kind != OLD.activity_kind + OR NEW.activity_id != OLD.activity_id + OR NEW.session_id != OLD.session_id + OR NEW.project_id != OLD.project_id + OR NEW.request_kind != OLD.request_kind + OR NEW.idempotency_key != OLD.idempotency_key + OR NEW.permission != OLD.permission + OR NEW.patterns != OLD.patterns + OR NEW.always_patterns != OLD.always_patterns + OR NEW.metadata_hash != OLD.metadata_hash + OR COALESCE(NEW.tool_message_id, '') != COALESCE(OLD.tool_message_id, '') + OR COALESCE(NEW.tool_call_id, '') != COALESCE(OLD.tool_call_id, '') + OR NEW.authority_epoch != OLD.authority_epoch + OR NEW.requested_scope != OLD.requested_scope + OR NEW.owner_type != OLD.owner_type + OR NEW.owner_id != OLD.owner_id + OR NEW.created_at != OLD.created_at + OR COALESCE(NEW.expires_at, -1) != COALESCE(OLD.expires_at, -1) + OR OLD.state != 'pending' + OR NEW.state = 'pending' + OR NEW.decided_at IS NULL + OR NOT EXISTS ( + SELECT 1 FROM session_activity_permission_decision decision + WHERE decision.request_id = OLD.request_id AND decision.decision = NEW.state + ) + BEGIN + SELECT RAISE(ABORT, 'illegal session_activity_permission_request transition'); + END + `) + yield* tx.run(` + CREATE TABLE session_activity_permission_once_consumption ( + request_id TEXT PRIMARY KEY REFERENCES session_activity_permission_request(request_id) ON DELETE CASCADE, + consumer_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL UNIQUE, + consumed_at INTEGER NOT NULL + ) + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_once_consumption_validate_insert + BEFORE INSERT ON session_activity_permission_once_consumption + WHEN NOT EXISTS ( + SELECT 1 + FROM session_activity_permission_request request + JOIN session_activity_permission_decision decision ON decision.request_id = request.request_id + WHERE request.request_id = NEW.request_id + AND request.state = 'approved_once' + AND decision.decision = 'approved_once' + AND decision.scope = 'once' + AND (request.expires_at IS NULL OR request.expires_at > CAST(strftime('%s', 'now') AS INTEGER) * 1000) + AND (decision.expires_at IS NULL OR decision.expires_at > CAST(strftime('%s', 'now') AS INTEGER) * 1000) + ) + BEGIN + SELECT RAISE(ABORT, 'permission once receipt is not consumable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_once_consumption_immutable + BEFORE UPDATE ON session_activity_permission_once_consumption + BEGIN + SELECT RAISE(ABORT, 'session_activity_permission_once_consumption is immutable'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260811203000_learning_job_authority.ts b/packages/core/src/database/migration/20260811203000_learning_job_authority.ts new file mode 100644 index 000000000..81ec89a7d --- /dev/null +++ b/packages/core/src/database/migration/20260811203000_learning_job_authority.ts @@ -0,0 +1,193 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260811203000_learning_job_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE learning_job ( + job_id TEXT PRIMARY KEY NOT NULL, + project_id TEXT NOT NULL REFERENCES project(id), + session_id TEXT REFERENCES session(id), + run_id TEXT, + trigger TEXT NOT NULL CHECK (trigger IN ('idle', 'pause', 'project_switch', 'session_finalization')), + dedupe_key TEXT NOT NULL, + candidate_input_ref TEXT NOT NULL, + policy TEXT NOT NULL CHECK (policy IN ('auto_merge_safe_project', 'manual_review')), + max_attempts INTEGER NOT NULL CHECK (max_attempts > 0), + admission_fingerprint TEXT NOT NULL + CHECK (length(admission_fingerprint) = 64 AND admission_fingerprint NOT GLOB '*[^0-9a-f]*'), + state TEXT NOT NULL + CHECK (state IN ('queued', 'running', 'reviewing', 'governance', 'completed', 'failed', 'cancelled', 'recovery_required')), + attempts INTEGER NOT NULL CHECK (attempts >= 0), + owner TEXT, + lease_expires_at INTEGER, + version INTEGER NOT NULL CHECK (version >= 0), + side_effect_state TEXT NOT NULL + CHECK (side_effect_state IN ('not_started', 'started', 'settled', 'unknown')), + side_effect_kind TEXT + CHECK (side_effect_kind IS NULL OR side_effect_kind IN ('extraction', 'reviewer', 'governance')), + review_job_id TEXT, + result_ref TEXT, + error_code TEXT, + error_detail TEXT, + settlement_fingerprint TEXT + CHECK ( + settlement_fingerprint IS NULL OR + (length(settlement_fingerprint) = 64 AND settlement_fingerprint NOT GLOB '*[^0-9a-f]*') + ), + next_attempt_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + started_at INTEGER, + settled_at INTEGER, + updated_at INTEGER NOT NULL, + CHECK ( + (side_effect_state = 'not_started' AND side_effect_kind IS NULL) OR + (side_effect_state IN ('started', 'settled', 'unknown') AND side_effect_kind IS NOT NULL) + ), + CHECK ( + side_effect_state <> 'settled' OR length(trim(result_ref)) > 0 + ), + CHECK ( + state NOT IN ('running', 'reviewing', 'governance') OR + side_effect_state = 'not_started' OR + (state = 'running' AND side_effect_kind = 'extraction') OR + (state = 'reviewing' AND side_effect_kind = 'reviewer') OR + (state = 'governance' AND side_effect_kind = 'governance') + ), + CHECK ( + ( + state = 'queued' AND owner IS NULL AND lease_expires_at IS NULL AND + side_effect_state = 'not_started' + ) OR ( + state IN ('running', 'reviewing', 'governance') AND + started_at IS NOT NULL AND settled_at IS NULL AND ( + (length(trim(owner)) > 0 AND lease_expires_at IS NOT NULL) OR + (state IN ('reviewing', 'governance') AND owner IS NULL AND + lease_expires_at IS NULL AND side_effect_state = 'not_started') + ) + ) OR ( + state IN ('completed', 'failed', 'cancelled', 'recovery_required') AND + owner IS NULL AND lease_expires_at IS NULL AND settled_at IS NOT NULL + ) + ), + CHECK ( + state <> 'recovery_required' OR + (side_effect_state <> 'not_started' AND error_code IS NOT NULL) + ), + CHECK ( + state <> 'completed' OR + (side_effect_state = 'settled' AND length(trim(result_ref)) > 0) + ), + CHECK ( + state <> 'failed' OR length(trim(error_code)) > 0 + ), + CHECK ( + state NOT IN ('completed', 'failed', 'cancelled') OR + side_effect_state IN ('not_started', 'settled') + ) + ) + `) + yield* tx.run("CREATE UNIQUE INDEX learning_job_dedupe_idx ON learning_job (dedupe_key)") + yield* tx.run("CREATE INDEX learning_job_due_idx ON learning_job (state, next_attempt_at, created_at)") + yield* tx.run("CREATE INDEX learning_job_project_created_idx ON learning_job (project_id, created_at)") + yield* tx.run( + "CREATE INDEX learning_job_owner_lease_idx ON learning_job (owner, lease_expires_at) WHERE owner IS NOT NULL", + ) + yield* tx.run(` + CREATE TRIGGER learning_job_scope_validate_insert + BEFORE INSERT ON learning_job + WHEN NEW.session_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM session + WHERE session.id = NEW.session_id AND session.project_id = NEW.project_id + ) + BEGIN + SELECT RAISE(ABORT, 'learning_job_session_project_mismatch'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_identity_immutable + BEFORE UPDATE ON learning_job + WHEN NEW.job_id != OLD.job_id + OR NEW.project_id != OLD.project_id + OR NEW.session_id IS NOT OLD.session_id + OR NEW.run_id IS NOT OLD.run_id + OR NEW.trigger != OLD.trigger + OR NEW.dedupe_key != OLD.dedupe_key + OR NEW.candidate_input_ref != OLD.candidate_input_ref + OR NEW.policy != OLD.policy + OR NEW.max_attempts != OLD.max_attempts + OR NEW.admission_fingerprint != OLD.admission_fingerprint + OR NEW.created_at != OLD.created_at + BEGIN + SELECT RAISE(ABORT, 'learning_job_identity_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_version_fence + BEFORE UPDATE ON learning_job + WHEN NEW.version != OLD.version + 1 + BEGIN + SELECT RAISE(ABORT, 'learning_job_version_must_advance_once'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_terminal_immutable + BEFORE UPDATE ON learning_job + WHEN OLD.state IN ('completed', 'failed', 'cancelled', 'recovery_required') + BEGIN + SELECT RAISE(ABORT, 'learning_job_terminal_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_transition_guard + BEFORE UPDATE OF state ON learning_job + WHEN NOT ( + (OLD.state = 'queued' AND NEW.state IN ('running', 'cancelled')) OR + (OLD.state = 'running' AND + NEW.state IN ('queued', 'running', 'reviewing', 'failed', 'cancelled', 'recovery_required')) OR + (OLD.state = 'reviewing' AND + NEW.state IN ('reviewing', 'governance', 'failed', 'cancelled', 'recovery_required')) OR + (OLD.state = 'governance' AND + NEW.state IN ('governance', 'completed', 'failed', 'cancelled', 'recovery_required')) + ) + BEGIN + SELECT RAISE(ABORT, 'learning_job_transition_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_phase_advance_guard + BEFORE UPDATE OF state ON learning_job + WHEN ( + OLD.state = 'running' AND NEW.state = 'reviewing' AND NOT ( + OLD.side_effect_state = 'settled' AND OLD.side_effect_kind = 'extraction' AND + NEW.side_effect_state = 'not_started' AND NEW.side_effect_kind IS NULL + ) + ) OR ( + OLD.state = 'reviewing' AND NEW.state = 'governance' AND NOT ( + OLD.side_effect_state = 'settled' AND OLD.side_effect_kind = 'reviewer' AND + NEW.side_effect_state = 'not_started' AND NEW.side_effect_kind IS NULL + ) + ) OR ( + OLD.state = 'governance' AND NEW.state = 'completed' AND NOT ( + OLD.side_effect_state = 'settled' AND OLD.side_effect_kind = 'governance' AND + NEW.result_ref = OLD.result_ref + ) + ) + BEGIN + SELECT RAISE(ABORT, 'learning_job_phase_advance_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_side_effect_settlement_guard + BEFORE UPDATE ON learning_job + WHEN OLD.side_effect_state = 'started' AND NEW.side_effect_state = 'settled' + AND NEW.result_ref IS OLD.result_ref + BEGIN + SELECT RAISE(ABORT, 'learning_job_side_effect_result_required'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260811221351_w2_authority_schema.ts b/packages/core/src/database/migration/20260811221351_w2_authority_schema.ts new file mode 100644 index 000000000..f46826041 --- /dev/null +++ b/packages/core/src/database/migration/20260811221351_w2_authority_schema.ts @@ -0,0 +1,11 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +// Drizzle snapshot marker only. Runtime DDL is owned by the preceding hand-written +// released snapshot, activity authority, and learning job migrations. +export default { + id: "20260811221351_w2_authority_schema", + up() { + return Effect.void + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260811223000_prepared_provider_turn_receipt.ts b/packages/core/src/database/migration/20260811223000_prepared_provider_turn_receipt.ts new file mode 100644 index 000000000..641416e74 --- /dev/null +++ b/packages/core/src/database/migration/20260811223000_prepared_provider_turn_receipt.ts @@ -0,0 +1,227 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260811223000_prepared_provider_turn_receipt", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + ALTER TABLE session_tool_request_receipt ADD COLUMN prepared_turn_hash TEXT + CHECK (prepared_turn_hash IS NULL OR ( + length(prepared_turn_hash) = 64 AND + prepared_turn_hash NOT GLOB '*[^0-9a-f]*' + )) + `) + yield* tx.run(` + ALTER TABLE session_tool_request_receipt ADD COLUMN system_stable_hash TEXT + CHECK (system_stable_hash IS NULL OR ( + length(system_stable_hash) = 64 AND + system_stable_hash NOT GLOB '*[^0-9a-f]*' + )) + `) + yield* tx.run(` + ALTER TABLE session_tool_request_receipt ADD COLUMN system_volatile_hash TEXT + CHECK (system_volatile_hash IS NULL OR ( + length(system_volatile_hash) = 64 AND + system_volatile_hash NOT GLOB '*[^0-9a-f]*' + )) + `) + yield* tx.run(` + ALTER TABLE session_tool_request_receipt ADD COLUMN wire_request_hash TEXT + CHECK (wire_request_hash IS NULL OR ( + length(wire_request_hash) = 64 AND + wire_request_hash NOT GLOB '*[^0-9a-f]*' + )) + `) + yield* tx.run(` + ALTER TABLE session_tool_request_receipt ADD COLUMN tool_result_reference_ids TEXT NOT NULL DEFAULT '[]' + CHECK ( + json_valid(tool_result_reference_ids) = 1 AND + json_type(tool_result_reference_ids) = 'array' + ) + `) + yield* tx.run(` + ALTER TABLE session_tool_request_receipt ADD COLUMN tool_result_reference_count INTEGER NOT NULL DEFAULT 0 + CHECK (tool_result_reference_count >= 0) + `) + + yield* tx.run("DROP TRIGGER IF EXISTS session_tool_request_receipt_dispatch_guard") + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_dispatch_guard + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_state IN ('prepared', 'dispatching') AND ( + NEW.final_request_hash IS NULL OR + NEW.adapter_prepared_at IS NULL OR + NEW.prompt_epoch IS NULL OR + NEW.prompt_window_id IS NULL OR + NEW.effective_history_hash IS NULL OR + NEW.context_eligibility IS NULL OR + json_valid(NEW.context_eligibility) != 1 OR + NEW.context_readiness IS NULL OR + json_valid(NEW.context_readiness) != 1 OR + NEW.context_activation IS NULL OR + json_valid(NEW.context_activation) != 1 OR + NEW.context_activation_fingerprint IS NULL OR + length(NEW.context_activation_fingerprint) != 64 OR + NEW.context_activation_fingerprint GLOB '*[^0-9a-f]*' OR + ${invalidPreparedTurn("NEW")} + ) + BEGIN + SELECT RAISE(ABORT, 'provider preparation and dispatch require a durable prepared turn'); + END + `) + + yield* tx.run("DROP TRIGGER IF EXISTS session_tool_request_receipt_binding_immutable") + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_binding_immutable + BEFORE UPDATE ON session_tool_request_receipt + WHEN NEW.receipt_id != OLD.receipt_id + OR NEW.request_ordinal != OLD.request_ordinal + OR NEW.session_id != OLD.session_id + OR NEW.user_message_id != OLD.user_message_id + OR NEW.assistant_message_id IS NOT OLD.assistant_message_id + OR NEW.provider_attempt_id IS NOT OLD.provider_attempt_id + OR NEW.provider_id != OLD.provider_id + OR NEW.model_id != OLD.model_id + OR NEW.protocol IS NOT OLD.protocol + OR NEW.context_selection_id IS NOT OLD.context_selection_id + OR NEW.context_eligibility IS NOT OLD.context_eligibility + OR NEW.context_readiness IS NOT OLD.context_readiness + OR NEW.context_activation IS NOT OLD.context_activation + OR NEW.context_activation_fingerprint IS NOT OLD.context_activation_fingerprint + OR NEW.prompt_epoch IS NOT OLD.prompt_epoch + OR NEW.prompt_window_id IS NOT OLD.prompt_window_id + OR NEW.effective_history_hash IS NOT OLD.effective_history_hash + OR NEW.world_state_baseline_hash IS NOT OLD.world_state_baseline_hash + OR NEW.request_input_hash IS NOT OLD.request_input_hash + OR NEW.owner_token IS NOT OLD.owner_token + OR NEW.created_at != OLD.created_at + OR (OLD.final_request_hash IS NOT NULL AND NEW.final_request_hash IS NOT OLD.final_request_hash) + OR (OLD.provider_request_hash IS NOT NULL AND NEW.provider_request_hash IS NOT OLD.provider_request_hash) + OR (OLD.adapter_prepared_at IS NOT NULL AND NEW.adapter_prepared_at IS NOT OLD.adapter_prepared_at) + OR (OLD.prompt_cache_key IS NOT NULL AND NEW.prompt_cache_key IS NOT OLD.prompt_cache_key) + OR (OLD.tool_definition_hash IS NOT NULL AND NEW.tool_definition_hash IS NOT OLD.tool_definition_hash) + OR (OLD.response_fingerprint IS NOT NULL AND NEW.response_fingerprint IS NOT OLD.response_fingerprint) + OR (OLD.prepared_turn_hash IS NOT NULL AND NEW.prepared_turn_hash IS NOT OLD.prepared_turn_hash) + OR (OLD.prepared_turn_hash IS NOT NULL AND NEW.system_stable_hash IS NOT OLD.system_stable_hash) + OR (OLD.prepared_turn_hash IS NOT NULL AND NEW.system_volatile_hash IS NOT OLD.system_volatile_hash) + OR (OLD.prepared_turn_hash IS NOT NULL AND NEW.wire_request_hash IS NOT OLD.wire_request_hash) + OR (OLD.prepared_turn_hash IS NOT NULL AND NEW.tool_result_reference_ids IS NOT OLD.tool_result_reference_ids) + OR (OLD.prepared_turn_hash IS NOT NULL AND NEW.tool_result_reference_count IS NOT OLD.tool_result_reference_count) + BEGIN + SELECT RAISE(ABORT, 'provider receipt binding is immutable'); + END + `) + + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_prepared_turn_insert_guard + BEFORE INSERT ON session_tool_request_receipt + WHEN NEW.final_request_hash IS NOT NULL + OR NEW.provider_request_hash IS NOT NULL + OR NEW.adapter_prepared_at IS NOT NULL + OR NEW.prompt_cache_key IS NOT NULL + OR NEW.tool_definition_hash IS NOT NULL + OR NEW.final_offered_tool_ids != '[]' + OR NEW.prepared_turn_hash IS NOT NULL + OR NEW.system_stable_hash IS NOT NULL + OR NEW.system_volatile_hash IS NOT NULL + OR NEW.wire_request_hash IS NOT NULL + OR NEW.tool_result_reference_ids != '[]' + OR NEW.tool_result_reference_count != 0 + BEGIN + SELECT RAISE(ABORT, 'prepared provider turn must be sealed after receipt admission'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_prepared_turn_seal + BEFORE UPDATE OF + final_request_hash, + provider_request_hash, + adapter_prepared_at, + prompt_cache_key, + final_offered_tool_ids, + tool_definition_hash, + prepared_turn_hash, + system_stable_hash, + system_volatile_hash, + wire_request_hash, + tool_result_reference_ids, + tool_result_reference_count + ON session_tool_request_receipt + WHEN ( + NEW.final_request_hash IS NOT OLD.final_request_hash OR + NEW.provider_request_hash IS NOT OLD.provider_request_hash OR + NEW.adapter_prepared_at IS NOT OLD.adapter_prepared_at OR + NEW.prompt_cache_key IS NOT OLD.prompt_cache_key OR + NEW.final_offered_tool_ids IS NOT OLD.final_offered_tool_ids OR + NEW.tool_definition_hash IS NOT OLD.tool_definition_hash OR + NEW.prepared_turn_hash IS NOT OLD.prepared_turn_hash OR + NEW.system_stable_hash IS NOT OLD.system_stable_hash OR + NEW.system_volatile_hash IS NOT OLD.system_volatile_hash OR + NEW.wire_request_hash IS NOT OLD.wire_request_hash OR + NEW.tool_result_reference_ids IS NOT OLD.tool_result_reference_ids OR + NEW.tool_result_reference_count IS NOT OLD.tool_result_reference_count + ) AND NOT ( + OLD.provider_state = 'preparing' AND + NEW.provider_state = 'prepared' AND + OLD.final_request_hash IS NULL AND + OLD.provider_request_hash IS NULL AND + OLD.adapter_prepared_at IS NULL AND + OLD.prompt_cache_key IS NULL AND + OLD.final_offered_tool_ids = '[]' AND + OLD.tool_definition_hash IS NULL AND + OLD.prepared_turn_hash IS NULL AND + OLD.system_stable_hash IS NULL AND + OLD.system_volatile_hash IS NULL AND + OLD.wire_request_hash IS NULL AND + OLD.tool_result_reference_ids = '[]' AND + OLD.tool_result_reference_count = 0 AND + NOT ${invalidPreparedTurn("NEW")} + ) + BEGIN + SELECT RAISE(ABORT, 'prepared provider turn may only be sealed once during adapter preparation'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration + +function invalidPreparedTurn(row: string) { + return `( + ${invalidHash(`${row}.prepared_turn_hash`)} OR + ${invalidHash(`${row}.system_stable_hash`)} OR + ${invalidHash(`${row}.system_volatile_hash`)} OR + ${invalidHash(`${row}.wire_request_hash`)} OR + ${row}.wire_request_hash IS NOT ${row}.final_request_hash OR + ${row}.provider_request_hash IS NOT ${row}.final_request_hash OR + ${row}.adapter_prepared_at IS NULL OR + ${invalidHash(`${row}.tool_definition_hash`)} OR + json_valid(${row}.final_offered_tool_ids) != 1 OR + json_type(${row}.final_offered_tool_ids) != 'array' OR + EXISTS ( + SELECT 1 + FROM json_each(${row}.final_offered_tool_ids) offered_tool + WHERE offered_tool.type != 'text' OR offered_tool.value = '' + ) OR + (SELECT count(*) FROM json_each(${row}.final_offered_tool_ids)) != + (SELECT count(DISTINCT offered_tool.value) FROM json_each(${row}.final_offered_tool_ids) offered_tool) OR + json_valid(${row}.tool_result_reference_ids) != 1 OR + json_type(${row}.tool_result_reference_ids) != 'array' OR + ${row}.tool_result_reference_count != json_array_length(${row}.tool_result_reference_ids) OR + EXISTS ( + SELECT 1 + FROM json_each(${row}.tool_result_reference_ids) reference + WHERE reference.type != 'text' OR reference.value = '' + ) OR + (SELECT count(*) FROM json_each(${row}.tool_result_reference_ids)) != + (SELECT count(DISTINCT reference.value) FROM json_each(${row}.tool_result_reference_ids) reference) + )` +} + +function invalidHash(value: string) { + return `( + ${value} IS NULL OR + length(${value}) != 64 OR + ${value} GLOB '*[^0-9a-f]*' + )` +} diff --git a/packages/core/src/database/migration/20260811224500_activity_permission_terminal_fence.ts b/packages/core/src/database/migration/20260811224500_activity_permission_terminal_fence.ts new file mode 100644 index 000000000..e1638bc66 --- /dev/null +++ b/packages/core/src/database/migration/20260811224500_activity_permission_terminal_fence.ts @@ -0,0 +1,109 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260811224500_activity_permission_terminal_fence", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + INSERT INTO session_activity_permission_decision ( + decision_id, request_id, idempotency_key, decision, + actor_type, actor_id, scope, authority_epoch, decided_at, expires_at + ) + SELECT + 'permission-terminal:' || request.request_id, + request.request_id, + 'permission-terminal:' || request.request_id, + CASE + WHEN request.expires_at IS NOT NULL + AND request.expires_at <= CAST(strftime('%s', 'now') AS INTEGER) * 1000 + THEN 'expired' + ELSE 'interrupted' + END, + 'system', + 'activity-authority', + request.requested_scope, + request.authority_epoch, + CAST(strftime('%s', 'now') AS INTEGER) * 1000, + NULL + FROM session_activity_permission_request request + JOIN session_activity_objective objective + ON objective.activity_kind = request.activity_kind + AND objective.activity_id = request.activity_id + WHERE request.state = 'pending' + AND objective.state IN ('completed', 'interrupted', 'recovery_required') + ON CONFLICT(request_id) DO NOTHING + `) + yield* tx.run(` + UPDATE session_activity_permission_request + SET state = ( + SELECT decision.decision + FROM session_activity_permission_decision decision + WHERE decision.request_id = session_activity_permission_request.request_id + ), + decided_at = ( + SELECT decision.decided_at + FROM session_activity_permission_decision decision + WHERE decision.request_id = session_activity_permission_request.request_id + ) + WHERE state = 'pending' + AND EXISTS ( + SELECT 1 + FROM session_activity_permission_decision decision + WHERE decision.request_id = session_activity_permission_request.request_id + AND decision.decision IN ('expired', 'interrupted') + ) + AND EXISTS ( + SELECT 1 + FROM session_activity_objective objective + WHERE objective.activity_kind = session_activity_permission_request.activity_kind + AND objective.activity_id = session_activity_permission_request.activity_id + AND objective.state IN ('completed', 'interrupted', 'recovery_required') + ) + `) + yield* tx.run(` + CREATE TRIGGER session_activity_objective_permission_terminal + AFTER UPDATE OF state ON session_activity_objective + WHEN OLD.state IN ('active', 'needs_human') + AND NEW.state IN ('completed', 'interrupted', 'recovery_required') + BEGIN + INSERT INTO session_activity_permission_decision ( + decision_id, request_id, idempotency_key, decision, + actor_type, actor_id, scope, authority_epoch, decided_at, expires_at + ) + SELECT + 'permission-terminal:' || request.request_id, + request.request_id, + 'permission-terminal:' || request.request_id, + CASE + WHEN request.expires_at IS NOT NULL AND request.expires_at <= NEW.updated_at + THEN 'expired' + ELSE 'interrupted' + END, + 'system', + 'activity-authority', + request.requested_scope, + request.authority_epoch, + NEW.updated_at, + NULL + FROM session_activity_permission_request request + WHERE request.activity_kind = NEW.activity_kind + AND request.activity_id = NEW.activity_id + AND request.state = 'pending' + ON CONFLICT(request_id) DO NOTHING; + + UPDATE session_activity_permission_request + SET state = ( + SELECT decision.decision + FROM session_activity_permission_decision decision + WHERE decision.request_id = session_activity_permission_request.request_id + ), + decided_at = NEW.updated_at + WHERE activity_kind = NEW.activity_kind + AND activity_id = NEW.activity_id + AND state = 'pending'; + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260811225000_activity_permission_owner_lease.ts b/packages/core/src/database/migration/20260811225000_activity_permission_owner_lease.ts new file mode 100644 index 000000000..2258dbdbd --- /dev/null +++ b/packages/core/src/database/migration/20260811225000_activity_permission_owner_lease.ts @@ -0,0 +1,33 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260811225000_activity_permission_owner_lease", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE session_activity_permission_owner_lease ( + owner_id TEXT PRIMARY KEY, + lease_expires_at INTEGER NOT NULL, + heartbeat_at INTEGER NOT NULL, + CHECK (length(trim(owner_id)) > 0), + CHECK (lease_expires_at > heartbeat_at) + ) + `) + yield* tx.run(` + CREATE INDEX session_activity_permission_owner_lease_expiry_idx + ON session_activity_permission_owner_lease(lease_expires_at, owner_id) + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_owner_lease_legal_update + BEFORE UPDATE ON session_activity_permission_owner_lease + WHEN NEW.owner_id != OLD.owner_id + OR NEW.heartbeat_at < OLD.heartbeat_at + OR NEW.lease_expires_at <= NEW.heartbeat_at + BEGIN + SELECT RAISE(ABORT, 'illegal session_activity_permission_owner_lease update'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812005647_learning_admission_outbox.ts b/packages/core/src/database/migration/20260812005647_learning_admission_outbox.ts new file mode 100644 index 000000000..601bb1da6 --- /dev/null +++ b/packages/core/src/database/migration/20260812005647_learning_admission_outbox.ts @@ -0,0 +1,87 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812005647_learning_admission_outbox", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`learning_admission_outbox\` ( + \`intent_id\` text PRIMARY KEY, + \`session_id\` text NOT NULL, + \`run_id\` text NOT NULL, + \`trigger\` text NOT NULL, + \`dedupe_key\` text NOT NULL, + \`payload_json\` text NOT NULL, + \`payload_fingerprint\` text NOT NULL, + \`state\` text NOT NULL, + \`job_id\` text, + \`candidate_input_ref\` text, + \`rejection_code\` text, + \`rejection_detail\` text, + \`created_at\` integer NOT NULL, + \`settled_at\` integer, + \`updated_at\` integer NOT NULL, + CONSTRAINT \`fk_learning_admission_outbox_job_id_learning_job_job_id_fk\` FOREIGN KEY (\`job_id\`) REFERENCES \`learning_job\`(\`job_id\`), + CONSTRAINT "learning_admission_outbox_trigger_check" CHECK("trigger" IN ('idle', 'pause', 'project_switch', 'session_finalization')), + CONSTRAINT "learning_admission_outbox_payload_json_check" CHECK(json_valid("payload_json") AND json_type("payload_json") = 'object'), + CONSTRAINT "learning_admission_outbox_payload_fingerprint_check" CHECK(length("payload_fingerprint") = 64 AND "payload_fingerprint" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_admission_outbox_state_check" CHECK("state" IN ('pending', 'admitted', 'rejected')), + CONSTRAINT "learning_admission_outbox_settlement_check" CHECK(("state" = 'pending' AND "job_id" IS NULL AND "candidate_input_ref" IS NULL AND "rejection_code" IS NULL AND "rejection_detail" IS NULL AND "settled_at" IS NULL) OR ("state" = 'admitted' AND "job_id" IS NOT NULL AND length(trim("candidate_input_ref")) > 0 AND "rejection_code" IS NULL AND "rejection_detail" IS NULL AND "settled_at" IS NOT NULL) OR ("state" = 'rejected' AND "job_id" IS NULL AND length(trim("rejection_code")) > 0 AND length(trim("rejection_detail")) > 0 AND "settled_at" IS NOT NULL)) + ); + `) + yield* tx.run( + `CREATE UNIQUE INDEX \`learning_admission_outbox_dedupe_idx\` ON \`learning_admission_outbox\` (\`dedupe_key\`);`, + ) + yield* tx.run( + `CREATE INDEX \`learning_admission_outbox_pending_idx\` ON \`learning_admission_outbox\` (\`created_at\`) WHERE "learning_admission_outbox"."state" = 'pending';`, + ) + yield* tx.run(` + CREATE TRIGGER learning_admission_outbox_identity_immutable + BEFORE UPDATE ON learning_admission_outbox + WHEN NEW.intent_id != OLD.intent_id + OR NEW.session_id != OLD.session_id + OR NEW.run_id != OLD.run_id + OR NEW.trigger != OLD.trigger + OR NEW.dedupe_key != OLD.dedupe_key + OR NEW.payload_json != OLD.payload_json + OR NEW.payload_fingerprint != OLD.payload_fingerprint + OR NEW.created_at != OLD.created_at + BEGIN + SELECT RAISE(ABORT, 'learning_admission_outbox_identity_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_admission_outbox_transition_guard + BEFORE UPDATE OF state ON learning_admission_outbox + WHEN NOT (OLD.state = 'pending' AND NEW.state IN ('admitted', 'rejected')) + BEGIN + SELECT RAISE(ABORT, 'learning_admission_outbox_transition_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_admission_outbox_terminal_immutable + BEFORE UPDATE ON learning_admission_outbox + WHEN OLD.state IN ('admitted', 'rejected') + BEGIN + SELECT RAISE(ABORT, 'learning_admission_outbox_terminal_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_admission_outbox_job_binding + BEFORE UPDATE OF state ON learning_admission_outbox + WHEN NEW.state = 'admitted' AND NOT EXISTS ( + SELECT 1 FROM learning_job + WHERE learning_job.job_id = NEW.job_id + AND learning_job.session_id = NEW.session_id + AND learning_job.run_id = NEW.run_id + AND learning_job.trigger = NEW.trigger + AND learning_job.candidate_input_ref = NEW.candidate_input_ref + ) + BEGIN + SELECT RAISE(ABORT, 'learning_admission_outbox_job_mismatch'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812014934_activity_permission_route_feedback.ts b/packages/core/src/database/migration/20260812014934_activity_permission_route_feedback.ts new file mode 100644 index 000000000..b737dcfa8 --- /dev/null +++ b/packages/core/src/database/migration/20260812014934_activity_permission_route_feedback.ts @@ -0,0 +1,71 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812014934_activity_permission_route_feedback", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(`ALTER TABLE \`session_activity_permission_decision\` ADD \`feedback\` text;`) + yield* tx.run(`ALTER TABLE \`session_activity_permission_request\` ADD \`workspace_id\` text;`) + yield* tx.run(`DROP TRIGGER session_activity_permission_request_legal_update`) + yield* tx.run(` + UPDATE session_activity_permission_request + SET workspace_id = ( + SELECT session.workspace_id + FROM session + WHERE session.id = session_activity_permission_request.session_id + ) + WHERE workspace_id IS NULL + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_request_legal_update + BEFORE UPDATE ON session_activity_permission_request + WHEN NEW.request_id != OLD.request_id + OR NEW.activity_kind != OLD.activity_kind + OR NEW.activity_id != OLD.activity_id + OR NEW.session_id != OLD.session_id + OR NEW.project_id != OLD.project_id + OR NEW.request_kind != OLD.request_kind + OR NEW.idempotency_key != OLD.idempotency_key + OR NEW.permission != OLD.permission + OR NEW.patterns != OLD.patterns + OR NEW.always_patterns != OLD.always_patterns + OR NEW.metadata_hash != OLD.metadata_hash + OR COALESCE(NEW.tool_message_id, '') != COALESCE(OLD.tool_message_id, '') + OR COALESCE(NEW.tool_call_id, '') != COALESCE(OLD.tool_call_id, '') + OR NEW.authority_epoch != OLD.authority_epoch + OR NEW.requested_scope != OLD.requested_scope + OR NEW.owner_type != OLD.owner_type + OR NEW.owner_id != OLD.owner_id + OR NEW.created_at != OLD.created_at + OR COALESCE(NEW.expires_at, -1) != COALESCE(OLD.expires_at, -1) + OR OLD.state != 'pending' + OR NEW.state = 'pending' + OR NEW.decided_at IS NULL + OR NOT EXISTS ( + SELECT 1 FROM session_activity_permission_decision decision + WHERE decision.request_id = OLD.request_id AND decision.decision = NEW.state + ) + BEGIN + SELECT RAISE(ABORT, 'illegal session_activity_permission_request transition'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_route_immutable + BEFORE UPDATE ON session_activity_permission_request + WHEN NEW.workspace_id IS NOT OLD.workspace_id + BEGIN + SELECT RAISE(ABORT, 'permission request workspace identity is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_feedback_immutable + BEFORE UPDATE ON session_activity_permission_decision + WHEN NEW.feedback IS NOT OLD.feedback + BEGIN + SELECT RAISE(ABORT, 'permission decision feedback is immutable'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812034116_learning_governance_authority.ts b/packages/core/src/database/migration/20260812034116_learning_governance_authority.ts new file mode 100644 index 000000000..cc6e763f7 --- /dev/null +++ b/packages/core/src/database/migration/20260812034116_learning_governance_authority.ts @@ -0,0 +1,83 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812034116_learning_governance_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`learning_governance_action\` ( + \`action_id\` text PRIMARY KEY, + \`plan_id\` text NOT NULL, + \`candidate_id\` text NOT NULL, + \`sequence\` integer NOT NULL, + \`kind\` text NOT NULL, + \`predecessor_action_id\` text, + \`payload_json\` text NOT NULL, + \`payload_fingerprint\` text NOT NULL, + \`state\` text NOT NULL, + \`owner\` text, + \`lease_expires_at\` integer, + \`version\` integer NOT NULL, + \`result_ref\` text, + \`result_hash\` text, + \`result_fingerprint\` text, + \`error_code\` text, + \`error_detail\` text, + \`created_at\` integer NOT NULL, + \`settled_at\` integer, + \`updated_at\` integer NOT NULL, + CONSTRAINT \`fk_learning_governance_action_plan_id_learning_governance_plan_plan_id_fk\` FOREIGN KEY (\`plan_id\`) REFERENCES \`learning_governance_plan\`(\`plan_id\`), + CONSTRAINT "learning_governance_action_sequence_check" CHECK("sequence" >= 0), + CONSTRAINT "learning_governance_action_version_check" CHECK("version" >= 0), + CONSTRAINT "learning_governance_action_candidate_check" CHECK(length(trim("candidate_id")) > 0), + CONSTRAINT "learning_governance_action_payload_json_check" CHECK(json_valid("payload_json")), + CONSTRAINT "learning_governance_action_kind_check" CHECK("kind" IN ('document_stage', 'memory_inbox')), + CONSTRAINT "learning_governance_action_payload_fingerprint_check" CHECK(length("payload_fingerprint") = 64 AND "payload_fingerprint" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_governance_action_result_hash_check" CHECK("result_hash" IS NULL OR (length("result_hash") = 64 AND "result_hash" NOT GLOB '*[^0-9a-f]*')), + CONSTRAINT "learning_governance_action_result_fingerprint_check" CHECK("result_fingerprint" IS NULL OR (length("result_fingerprint") = 64 AND "result_fingerprint" NOT GLOB '*[^0-9a-f]*')), + CONSTRAINT "learning_governance_action_state_check" CHECK("state" IN ('prepared', 'running', 'settled', 'recovery_required')), + CONSTRAINT "learning_governance_action_lifecycle_check" CHECK(("state" = 'prepared' AND "owner" IS NULL AND "lease_expires_at" IS NULL AND "result_ref" IS NULL AND "result_hash" IS NULL AND "result_fingerprint" IS NULL AND "error_code" IS NULL AND "settled_at" IS NULL) OR ("state" = 'running' AND length(trim("owner")) > 0 AND "lease_expires_at" IS NOT NULL AND "result_ref" IS NULL AND "result_hash" IS NULL AND "result_fingerprint" IS NULL AND "error_code" IS NULL AND "settled_at" IS NULL) OR ("state" = 'settled' AND "owner" IS NULL AND "lease_expires_at" IS NULL AND length(trim("result_ref")) > 0 AND "result_hash" IS NOT NULL AND "result_fingerprint" IS NOT NULL AND "error_code" IS NULL AND "settled_at" IS NOT NULL) OR ("state" = 'recovery_required' AND "owner" IS NULL AND "lease_expires_at" IS NULL AND length(trim("error_code")) > 0 AND "settled_at" IS NOT NULL)) + ); + `) + yield* tx.run(` + CREATE TABLE \`learning_governance_plan\` ( + \`plan_id\` text PRIMARY KEY, + \`job_id\` text NOT NULL, + \`policy\` text NOT NULL, + \`payload_json\` text NOT NULL, + \`payload_fingerprint\` text NOT NULL, + \`action_count\` integer NOT NULL, + \`job_owner\` text NOT NULL, + \`source_job_version\` integer NOT NULL, + \`job_started_version\` integer NOT NULL, + \`state\` text NOT NULL, + \`version\` integer NOT NULL, + \`result_ref\` text, + \`result_hash\` text, + \`result_fingerprint\` text, + \`error_code\` text, + \`error_detail\` text, + \`created_at\` integer NOT NULL, + \`settled_at\` integer, + \`updated_at\` integer NOT NULL, + CONSTRAINT \`fk_learning_governance_plan_job_id_learning_job_job_id_fk\` FOREIGN KEY (\`job_id\`) REFERENCES \`learning_job\`(\`job_id\`), + CONSTRAINT "learning_governance_plan_policy_check" CHECK("policy" = 'manual_review'), + CONSTRAINT "learning_governance_plan_action_count_check" CHECK("action_count" >= 0), + CONSTRAINT "learning_governance_plan_version_check" CHECK("version" >= 0), + CONSTRAINT "learning_governance_plan_payload_json_check" CHECK(json_valid("payload_json")), + CONSTRAINT "learning_governance_plan_payload_fingerprint_check" CHECK(length("payload_fingerprint") = 64 AND "payload_fingerprint" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_governance_plan_result_hash_check" CHECK("result_hash" IS NULL OR (length("result_hash") = 64 AND "result_hash" NOT GLOB '*[^0-9a-f]*')), + CONSTRAINT "learning_governance_plan_result_fingerprint_check" CHECK("result_fingerprint" IS NULL OR (length("result_fingerprint") = 64 AND "result_fingerprint" NOT GLOB '*[^0-9a-f]*')), + CONSTRAINT "learning_governance_plan_state_check" CHECK("state" IN ('prepared', 'settled', 'recovery_required')), + CONSTRAINT "learning_governance_plan_settlement_check" CHECK(("state" = 'prepared' AND "result_ref" IS NULL AND "result_hash" IS NULL AND "result_fingerprint" IS NULL AND "error_code" IS NULL AND "settled_at" IS NULL) OR ("state" = 'settled' AND length(trim("result_ref")) > 0 AND "result_hash" IS NOT NULL AND "result_fingerprint" IS NOT NULL AND "error_code" IS NULL AND "settled_at" IS NOT NULL) OR ("state" = 'recovery_required' AND length(trim("error_code")) > 0 AND "settled_at" IS NOT NULL)) + ); + `) + yield* tx.run(`CREATE UNIQUE INDEX \`learning_governance_action_plan_sequence_idx\` ON \`learning_governance_action\` (\`plan_id\`,\`sequence\`);`) + yield* tx.run(`CREATE UNIQUE INDEX \`learning_governance_action_plan_candidate_kind_idx\` ON \`learning_governance_action\` (\`plan_id\`,\`candidate_id\`,\`kind\`);`) + yield* tx.run(`CREATE INDEX \`learning_governance_action_claim_idx\` ON \`learning_governance_action\` (\`plan_id\`,\`state\`,\`sequence\`);`) + yield* tx.run(`CREATE UNIQUE INDEX \`learning_governance_plan_job_idx\` ON \`learning_governance_plan\` (\`job_id\`);`) + yield* tx.run(`CREATE INDEX \`learning_governance_plan_state_idx\` ON \`learning_governance_plan\` (\`state\`,\`created_at\`);`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812035000_learning_governance_lifecycle.ts b/packages/core/src/database/migration/20260812035000_learning_governance_lifecycle.ts new file mode 100644 index 000000000..28eba7fb7 --- /dev/null +++ b/packages/core/src/database/migration/20260812035000_learning_governance_lifecycle.ts @@ -0,0 +1,323 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +// Runtime DDL is hand-written because governance correctness also depends on +// cross-table lifecycle triggers that Drizzle snapshots cannot express. +export default { + id: "20260812035000_learning_governance_lifecycle", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TRIGGER learning_governance_plan_identity_immutable + BEFORE UPDATE ON learning_governance_plan + WHEN NEW.plan_id != OLD.plan_id + OR NEW.job_id != OLD.job_id + OR NEW.policy != OLD.policy + OR NEW.payload_json != OLD.payload_json + OR NEW.payload_fingerprint != OLD.payload_fingerprint + OR NEW.action_count != OLD.action_count + OR NEW.source_job_version != OLD.source_job_version + OR NEW.created_at != OLD.created_at + BEGIN + SELECT RAISE(ABORT, 'learning_governance_plan_identity_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_plan_takeover_guard + BEFORE UPDATE OF job_owner, job_started_version ON learning_governance_plan + WHEN OLD.state != 'prepared' OR NOT EXISTS ( + SELECT 1 FROM learning_job job + WHERE job.job_id = OLD.job_id + AND job.state = 'governance' + AND job.side_effect_state = 'started' + AND job.side_effect_kind = 'governance' + AND job.owner = NEW.job_owner + AND job.version = NEW.job_started_version + AND job.lease_expires_at > NEW.updated_at + AND NEW.job_started_version = OLD.job_started_version + 1 + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_plan_takeover_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_plan_job_fence + BEFORE INSERT ON learning_governance_plan + WHEN NOT EXISTS ( + SELECT 1 FROM learning_job job + WHERE job.job_id = NEW.job_id + AND job.state = 'governance' + AND job.side_effect_state = 'not_started' + AND job.side_effect_kind IS NULL + AND job.owner = NEW.job_owner + AND job.version = NEW.source_job_version + AND job.lease_expires_at > NEW.created_at + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_plan_job_fence_lost'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_plan_version_fence + BEFORE UPDATE ON learning_governance_plan + WHEN NEW.version != OLD.version + 1 + BEGIN + SELECT RAISE(ABORT, 'learning_governance_plan_version_must_advance_once'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_plan_transition_guard + BEFORE UPDATE OF state ON learning_governance_plan + WHEN NOT ( + (OLD.state = 'prepared' AND NEW.state IN ('settled', 'recovery_required')) OR + (OLD.state = 'settled' AND NEW.state = 'recovery_required' AND EXISTS ( + SELECT 1 FROM learning_job job + WHERE job.job_id = OLD.job_id AND job.state != 'completed' + )) + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_plan_transition_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_plan_settle_guard + BEFORE UPDATE OF state ON learning_governance_plan + WHEN NEW.state = 'settled' AND ( + (SELECT count(*) FROM learning_governance_action action WHERE action.plan_id = OLD.plan_id) != OLD.action_count OR + EXISTS ( + SELECT 1 FROM learning_governance_action action + WHERE action.plan_id = OLD.plan_id AND action.state != 'settled' + ) + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_plan_actions_not_settled'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_plan_delete_guard + BEFORE DELETE ON learning_governance_plan + BEGIN + SELECT RAISE(ABORT, 'learning_governance_plan_delete_forbidden'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_action_identity_immutable + BEFORE UPDATE ON learning_governance_action + WHEN NEW.action_id != OLD.action_id + OR NEW.plan_id != OLD.plan_id + OR NEW.candidate_id != OLD.candidate_id + OR NEW.sequence != OLD.sequence + OR NEW.kind != OLD.kind + OR NEW.predecessor_action_id IS NOT OLD.predecessor_action_id + OR NEW.payload_json != OLD.payload_json + OR NEW.payload_fingerprint != OLD.payload_fingerprint + OR NEW.created_at != OLD.created_at + BEGIN + SELECT RAISE(ABORT, 'learning_governance_action_identity_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_action_insert_guard + BEFORE INSERT ON learning_governance_action + WHEN NOT EXISTS ( + SELECT 1 FROM learning_governance_plan plan + WHERE plan.plan_id = NEW.plan_id + AND plan.state = 'prepared' + AND NEW.sequence < plan.action_count + AND ( + NEW.predecessor_action_id IS NULL OR EXISTS ( + SELECT 1 FROM learning_governance_action predecessor + WHERE predecessor.action_id = NEW.predecessor_action_id + AND predecessor.plan_id = NEW.plan_id + AND predecessor.sequence < NEW.sequence + ) + ) + AND ( + SELECT count(*) FROM learning_governance_action action + WHERE action.plan_id = NEW.plan_id + ) < plan.action_count + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_action_insert_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_action_version_fence + BEFORE UPDATE ON learning_governance_action + WHEN NEW.version != OLD.version + 1 + BEGIN + SELECT RAISE(ABORT, 'learning_governance_action_version_must_advance_once'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_action_transition_guard + BEFORE UPDATE OF state ON learning_governance_action + WHEN NOT ( + (OLD.state = 'prepared' AND NEW.state = 'running') OR + (OLD.state = 'running' AND ( + NEW.state IN ('running', 'settled') OR + (NEW.state = 'recovery_required' AND EXISTS ( + SELECT 1 + FROM learning_governance_plan plan + JOIN learning_job job ON job.job_id = plan.job_id + WHERE plan.plan_id = OLD.plan_id + AND plan.state = 'prepared' + AND job.state = 'governance' + AND job.side_effect_state = 'started' + AND job.side_effect_kind = 'governance' + AND job.owner = OLD.owner + AND job.version = plan.job_started_version + AND job.lease_expires_at > NEW.updated_at + )) + )) OR + (OLD.state = 'settled' AND NEW.state = 'recovery_required' AND EXISTS ( + SELECT 1 + FROM learning_governance_plan plan + JOIN learning_job job ON job.job_id = plan.job_id + WHERE plan.plan_id = OLD.plan_id AND job.state != 'completed' + )) + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_action_transition_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_action_claim_guard + BEFORE UPDATE OF state ON learning_governance_action + WHEN NEW.state = 'running' AND ( + NOT EXISTS ( + SELECT 1 FROM learning_governance_plan plan + JOIN learning_job job ON job.job_id = plan.job_id + WHERE plan.plan_id = OLD.plan_id AND plan.state = 'prepared' + AND job.state = 'governance' + AND job.side_effect_state = 'started' + AND job.side_effect_kind = 'governance' + AND job.owner = NEW.owner + AND job.version = plan.job_started_version + AND job.lease_expires_at > NEW.updated_at + AND NEW.lease_expires_at <= job.lease_expires_at + ) OR + (OLD.state = 'prepared' AND NOT (NEW.lease_expires_at > NEW.updated_at)) OR + (OLD.state = 'running' AND NOT ( + OLD.lease_expires_at <= NEW.updated_at AND NEW.lease_expires_at > NEW.updated_at + )) OR + (OLD.predecessor_action_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM learning_governance_action predecessor + WHERE predecessor.action_id = OLD.predecessor_action_id + AND predecessor.plan_id = OLD.plan_id + AND predecessor.state = 'settled' + )) + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_action_claim_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_governance_takeover_guard + BEFORE UPDATE OF owner, lease_expires_at ON learning_job + WHEN OLD.state = 'governance' AND OLD.side_effect_state = 'started' + AND OLD.side_effect_kind = 'governance' AND NEW.state = 'governance' + AND NEW.side_effect_state = 'started' AND NEW.side_effect_kind = 'governance' + AND NEW.owner IS NOT OLD.owner AND NOT ( + OLD.lease_expires_at <= NEW.updated_at AND length(trim(NEW.owner)) > 0 + AND NEW.lease_expires_at > NEW.updated_at + AND EXISTS ( + SELECT 1 FROM learning_governance_plan plan + WHERE plan.job_id = OLD.job_id + AND plan.state = 'prepared' + AND plan.job_started_version = OLD.version + ) + ) + BEGIN + SELECT RAISE(ABORT, 'learning_job_governance_takeover_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_action_settle_guard + BEFORE UPDATE OF state ON learning_governance_action + WHEN NEW.state = 'settled' AND NOT EXISTS ( + SELECT 1 FROM learning_governance_plan plan + JOIN learning_job job ON job.job_id = plan.job_id + WHERE plan.plan_id = OLD.plan_id AND plan.state = 'prepared' + AND job.state = 'governance' + AND job.side_effect_state = 'started' + AND job.side_effect_kind = 'governance' + AND job.owner = OLD.owner + AND job.version = plan.job_started_version + AND job.lease_expires_at > NEW.updated_at + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_action_plan_not_prepared'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_action_result_conflict_guard + BEFORE UPDATE OF state ON learning_governance_action + WHEN OLD.state = 'settled' AND NEW.state = 'recovery_required' AND NOT ( + NEW.result_ref = OLD.result_ref AND NEW.result_hash = OLD.result_hash AND + NEW.result_fingerprint = OLD.result_fingerprint AND + NEW.error_code = 'governance_action_output_conflict' + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_action_conflict_evidence_required'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_action_delete_guard + BEFORE DELETE ON learning_governance_action + BEGIN + SELECT RAISE(ABORT, 'learning_governance_action_delete_forbidden'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_governance_plan_begin_guard + BEFORE UPDATE ON learning_job + WHEN OLD.state = 'governance' AND OLD.side_effect_state = 'not_started' + AND NEW.state = 'governance' AND NEW.side_effect_state = 'started' + AND NEW.side_effect_kind = 'governance' + AND NOT EXISTS ( + SELECT 1 FROM learning_governance_plan plan + WHERE plan.job_id = OLD.job_id + AND plan.state = 'prepared' + AND plan.job_owner = NEW.owner + AND plan.source_job_version = OLD.version + AND plan.job_started_version = NEW.version + AND plan.action_count = ( + SELECT count(*) FROM learning_governance_action action WHERE action.plan_id = plan.plan_id + ) + ) + BEGIN + SELECT RAISE(ABORT, 'learning_job_governance_plan_required'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_governance_plan_settle_guard + BEFORE UPDATE ON learning_job + WHEN OLD.state = 'governance' AND OLD.side_effect_state = 'started' + AND OLD.side_effect_kind = 'governance' AND NEW.side_effect_state = 'settled' + AND NOT EXISTS ( + SELECT 1 FROM learning_governance_plan plan + WHERE plan.job_id = OLD.job_id + AND plan.state = 'settled' + AND plan.result_ref = NEW.result_ref + ) + BEGIN + SELECT RAISE(ABORT, 'learning_job_governance_plan_not_settled'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_governance_completion_guard + BEFORE UPDATE OF state ON learning_job + WHEN OLD.state = 'governance' AND NEW.state = 'completed' AND NOT EXISTS ( + SELECT 1 FROM learning_governance_plan plan + WHERE plan.job_id = OLD.job_id + AND plan.state = 'settled' + AND plan.result_ref = NEW.result_ref + ) + BEGIN + SELECT RAISE(ABORT, 'learning_job_governance_plan_completion_required'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812043000_provider_attempt_pre_dispatch_terminal.ts b/packages/core/src/database/migration/20260812043000_provider_attempt_pre_dispatch_terminal.ts new file mode 100644 index 000000000..b4c39cf78 --- /dev/null +++ b/packages/core/src/database/migration/20260812043000_provider_attempt_pre_dispatch_terminal.ts @@ -0,0 +1,39 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812043000_provider_attempt_pre_dispatch_terminal", + up(tx) { + return Effect.gen(function* () { + yield* tx.run("DROP TRIGGER IF EXISTS session_provider_attempt_legal_update") + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_legal_update + BEFORE UPDATE ON session_provider_attempt + WHEN NEW.attempt_id != OLD.attempt_id + OR NEW.session_id != OLD.session_id + OR NEW.activity_id != OLD.activity_id + OR NEW.provider_turn_seq != OLD.provider_turn_seq + OR NEW.selection_id != OLD.selection_id + OR NEW.projection_hash != OLD.projection_hash + OR NEW.request_hash != OLD.request_hash + OR NEW.provider_id != OLD.provider_id + OR NEW.parent_attempt_id IS NOT OLD.parent_attempt_id + OR NEW.idempotency_key IS NOT OLD.idempotency_key + OR NEW.created_at != OLD.created_at + OR (NEW.first_event_at IS NOT OLD.first_event_at AND NOT (OLD.state = 'dispatching' AND NEW.state = 'streaming' AND OLD.first_event_at IS NULL AND NEW.first_event_at IS NOT NULL)) + OR (NEW.error_code IS NOT OLD.error_code AND NEW.state NOT IN ('failed', 'indeterminate_after_crash')) + OR NOT ( + (OLD.state = 'prepared' AND NEW.state = 'dispatching' AND NEW.first_event_at IS NULL AND NEW.settled_at IS NULL AND NEW.error_code IS NULL) OR + (OLD.state = 'prepared' AND NEW.state = 'failed' AND NEW.first_event_at IS NULL AND NEW.settled_at IS NOT NULL AND length(trim(COALESCE(NEW.error_code, ''))) > 0) OR + (OLD.state = 'dispatching' AND NEW.state = 'streaming' AND NEW.first_event_at IS NOT NULL AND NEW.settled_at IS NULL) OR + (OLD.state IN ('dispatching', 'streaming') AND NEW.state IN ('settled', 'failed') AND NEW.settled_at IS NOT NULL) OR + (OLD.state IN ('dispatching', 'streaming') AND NEW.state = 'indeterminate_after_crash' AND NEW.settled_at IS NULL) OR + (OLD.state = 'indeterminate_after_crash' AND NEW.state IN ('resolved_abandoned', 'resolved_settled', 'resolved_replayed') AND NEW.settled_at IS NOT NULL) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal session_provider_attempt transition'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812050000_provider_owner_authority.ts b/packages/core/src/database/migration/20260812050000_provider_owner_authority.ts new file mode 100644 index 000000000..dc434c2cd --- /dev/null +++ b/packages/core/src/database/migration/20260812050000_provider_owner_authority.ts @@ -0,0 +1,130 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812050000_provider_owner_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE session_provider_owner_lease ( + owner_token TEXT PRIMARY KEY, + registered_at INTEGER NOT NULL, + heartbeat_at INTEGER NOT NULL, + lease_expires_at INTEGER NOT NULL, + released_at INTEGER, + CHECK (length(trim(owner_token)) > 0), + CHECK (registered_at >= 0), + CHECK (heartbeat_at >= registered_at), + CHECK (lease_expires_at > heartbeat_at), + CHECK (released_at IS NULL OR released_at >= registered_at) + ) + `) + yield* tx.run(` + CREATE INDEX session_provider_owner_lease_expiry_idx + ON session_provider_owner_lease(lease_expires_at, owner_token) + `) + yield* tx.run(` + ALTER TABLE session_provider_attempt + ADD COLUMN owner_token TEXT REFERENCES session_provider_owner_lease(owner_token) + `) + yield* tx.run(` + CREATE INDEX session_provider_attempt_owner_state_idx + ON session_provider_attempt(state, owner_token, created_at) + `) + yield* tx.run(` + CREATE TRIGGER session_provider_owner_lease_legal_update + BEFORE UPDATE ON session_provider_owner_lease + WHEN NEW.owner_token != OLD.owner_token + OR NEW.registered_at != OLD.registered_at + OR NOT ( + ( + OLD.released_at IS NULL AND NEW.released_at IS NULL + AND NEW.heartbeat_at >= OLD.heartbeat_at + AND NEW.heartbeat_at < OLD.lease_expires_at + AND NEW.lease_expires_at >= OLD.lease_expires_at + AND NEW.lease_expires_at > NEW.heartbeat_at + ) OR ( + OLD.released_at IS NULL AND NEW.released_at IS NOT NULL + AND NEW.heartbeat_at = OLD.heartbeat_at + AND NEW.lease_expires_at = OLD.lease_expires_at + ) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal session_provider_owner_lease update'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_owner_lease_delete_guard + BEFORE DELETE ON session_provider_owner_lease + BEGIN + SELECT RAISE(ABORT, 'session_provider_owner_lease is append only'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_owner_insert_guard + BEFORE INSERT ON session_provider_attempt + WHEN NEW.owner_token IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM session_provider_owner_lease owner + WHERE owner.owner_token = NEW.owner_token + AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.created_at + ) + BEGIN + SELECT RAISE(ABORT, 'new provider attempt requires a live exact owner lease'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_owner_immutable + BEFORE UPDATE ON session_provider_attempt + WHEN NEW.owner_token IS NOT OLD.owner_token + BEGIN + SELECT RAISE(ABORT, 'provider attempt owner is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_attempt_owner_insert_guard + BEFORE INSERT ON session_tool_request_receipt + WHEN NEW.owner_token IS NULL OR + NOT EXISTS ( + SELECT 1 + FROM session_provider_owner_lease owner + WHERE owner.owner_token = NEW.owner_token + AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.created_at + ) OR + (NEW.provider_attempt_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 + FROM session_provider_attempt attempt + WHERE attempt.attempt_id = NEW.provider_attempt_id + AND attempt.session_id = NEW.session_id + AND attempt.owner_token = NEW.owner_token + )) + BEGIN + SELECT RAISE(ABORT, 'provider receipt requires the attempt live exact owner'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_attempt_owner_update_guard + BEFORE UPDATE ON session_tool_request_receipt + WHEN NEW.provider_attempt_id IS NOT NULL + AND EXISTS ( + SELECT 1 FROM session_provider_attempt attempt + WHERE attempt.attempt_id = NEW.provider_attempt_id + AND attempt.owner_token IS NOT NULL + ) + AND NOT EXISTS ( + SELECT 1 + FROM session_provider_attempt attempt + WHERE attempt.attempt_id = NEW.provider_attempt_id + AND attempt.session_id = NEW.session_id + AND attempt.owner_token = NEW.owner_token + ) + BEGIN + SELECT RAISE(ABORT, 'provider receipt attempt owner binding mismatch'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812053000_provider_turn_identity_authority.ts b/packages/core/src/database/migration/20260812053000_provider_turn_identity_authority.ts new file mode 100644 index 000000000..353956646 --- /dev/null +++ b/packages/core/src/database/migration/20260812053000_provider_turn_identity_authority.ts @@ -0,0 +1,373 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812053000_provider_turn_identity_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + ALTER TABLE session_provider_attempt ADD COLUMN prepared_turn_hash TEXT + CHECK (prepared_turn_hash IS NULL OR ( + length(prepared_turn_hash) = 64 AND + prepared_turn_hash NOT GLOB '*[^0-9a-f]*' + )) + `) + yield* tx.run(` + ALTER TABLE session_provider_attempt ADD COLUMN wire_request_hash TEXT + CHECK ( + (prepared_turn_hash IS NULL AND wire_request_hash IS NULL) OR + ( + prepared_turn_hash IS NOT NULL AND + wire_request_hash IS NOT NULL AND + length(wire_request_hash) = 64 AND + wire_request_hash NOT GLOB '*[^0-9a-f]*' + ) + ) + `) + yield* tx.run(` + CREATE TEMP TABLE provider_turn_identity_preflight ( + valid INTEGER NOT NULL CHECK (valid = 1) + ) + `) + yield* tx.run(` + INSERT INTO provider_turn_identity_preflight(valid) + SELECT CASE WHEN EXISTS ( + SELECT 1 + FROM session_provider_owner_lease owner + WHERE owner.registered_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + OR owner.heartbeat_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + OR owner.lease_expires_at - owner.heartbeat_at > 31536000000 + OR owner.released_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + ) OR EXISTS ( + SELECT 1 + FROM session_tool_request_receipt receipt + LEFT JOIN session_provider_attempt attempt + ON attempt.attempt_id = receipt.provider_attempt_id + LEFT JOIN session_context_selection selection + ON selection.selection_id = attempt.selection_id + WHERE receipt.provider_attempt_id IS NOT NULL + AND ( + attempt.attempt_id IS NULL OR + attempt.session_id IS NOT receipt.session_id OR + attempt.owner_token IS NOT receipt.owner_token OR + attempt.selection_id IS NOT receipt.context_selection_id OR + attempt.activity_id IS NOT selection.activity_id OR + attempt.session_id IS NOT selection.session_id OR + attempt.projection_hash IS NOT selection.projection_hash OR + attempt.provider_id IS NOT receipt.provider_id OR + attempt.request_hash IS NOT receipt.request_input_hash + ) + ) THEN 0 ELSE 1 END + `) + yield* tx.run("DROP TABLE provider_turn_identity_preflight") + yield* tx.run(` + UPDATE session_provider_owner_lease + SET released_at = CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + WHERE released_at IS NULL + `) + yield* tx.run(` + CREATE UNIQUE INDEX session_tool_request_receipt_provider_attempt_idx + ON session_tool_request_receipt(provider_attempt_id) + WHERE provider_attempt_id IS NOT NULL + `) + yield* tx.run(` + CREATE TRIGGER session_provider_owner_lease_insert_clock_guard + BEFORE INSERT ON session_provider_owner_lease + WHEN NEW.registered_at != CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + OR NEW.heartbeat_at != NEW.registered_at + OR NEW.lease_expires_at <= NEW.heartbeat_at + OR NEW.lease_expires_at - NEW.heartbeat_at > 31536000000 + OR NEW.released_at IS NOT NULL + BEGIN + SELECT RAISE(ABORT, 'provider owner lease requires database observed time'); + END + `) + yield* tx.run("DROP TRIGGER IF EXISTS session_provider_owner_lease_legal_update") + yield* tx.run(` + CREATE TRIGGER session_provider_owner_lease_legal_update + BEFORE UPDATE ON session_provider_owner_lease + WHEN NEW.owner_token != OLD.owner_token + OR NEW.registered_at != OLD.registered_at + OR NOT ( + ( + OLD.released_at IS NULL AND NEW.released_at IS NULL + AND NEW.heartbeat_at = CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + AND OLD.heartbeat_at <= NEW.heartbeat_at + AND NEW.heartbeat_at < OLD.lease_expires_at + AND NEW.lease_expires_at >= OLD.lease_expires_at + AND NEW.lease_expires_at > NEW.heartbeat_at + AND NEW.lease_expires_at - NEW.heartbeat_at <= 31536000000 + ) OR ( + OLD.released_at IS NULL AND NEW.released_at IS NOT NULL + AND NEW.released_at = CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + AND NEW.heartbeat_at = OLD.heartbeat_at + AND NEW.lease_expires_at = OLD.lease_expires_at + ) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal session_provider_owner_lease update'); + END + `) + yield* tx.run("DROP TRIGGER IF EXISTS session_provider_attempt_owner_insert_guard") + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_owner_insert_guard + BEFORE INSERT ON session_provider_attempt + WHEN NEW.owner_token IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM session_provider_owner_lease owner + WHERE owner.owner_token = NEW.owner_token + AND owner.released_at IS NULL + AND owner.lease_expires_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + ) + BEGIN + SELECT RAISE(ABORT, 'new provider attempt requires a live exact owner lease at database time'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_owner_update_guard + BEFORE UPDATE ON session_provider_attempt + WHEN ( + (OLD.state = 'prepared' AND NEW.state IN ('prepared', 'dispatching')) OR + ( + OLD.state = 'prepared' AND NEW.state = 'failed' + AND NEW.error_code NOT IN ( + 'owner_lease_lost_before_dispatch', + 'legacy_wire_identity_missing' + ) + ) OR + (OLD.state = 'dispatching' AND NEW.state = 'streaming') OR + (OLD.state IN ('dispatching', 'streaming') AND NEW.state IN ('settled', 'failed')) + ) AND NOT EXISTS ( + SELECT 1 + FROM session_provider_owner_lease owner + WHERE owner.owner_token = OLD.owner_token + AND owner.released_at IS NULL + AND owner.lease_expires_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + ) + BEGIN + SELECT RAISE(ABORT, 'provider attempt transition requires its live exact owner at database time'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_recovery_owner_guard + BEFORE UPDATE ON session_provider_attempt + WHEN ( + ( + OLD.state IN ('prepared', 'dispatching', 'streaming') + AND NEW.state = 'indeterminate_after_crash' + AND NEW.error_code = 'process_recovery' + ) OR ( + OLD.state = 'prepared' AND NEW.state = 'failed' + AND NEW.error_code = 'owner_lease_lost_before_dispatch' + ) + ) + AND ( + EXISTS ( + SELECT 1 + FROM session_provider_owner_lease stale + WHERE stale.owner_token = OLD.owner_token + AND stale.released_at IS NULL + AND stale.lease_expires_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + ) OR NOT EXISTS ( + SELECT 1 + FROM session_provider_owner_lease recovery + WHERE recovery.owner_token IS NOT OLD.owner_token + AND recovery.released_at IS NULL + AND recovery.lease_expires_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + ) + ) + BEGIN + SELECT RAISE(ABORT, 'provider attempt recovery requires stale old owner and a live recovery owner'); + END + `) + yield* tx.run("DROP TRIGGER IF EXISTS session_tool_request_receipt_attempt_owner_insert_guard") + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_attempt_owner_insert_guard + BEFORE INSERT ON session_tool_request_receipt + WHEN NEW.owner_token IS NULL OR + NOT EXISTS ( + SELECT 1 + FROM session_provider_owner_lease owner + WHERE owner.owner_token = NEW.owner_token + AND owner.released_at IS NULL + AND owner.lease_expires_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + ) OR + (NEW.provider_attempt_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 + FROM session_provider_attempt attempt + WHERE attempt.attempt_id = NEW.provider_attempt_id + AND attempt.session_id = NEW.session_id + AND attempt.owner_token = NEW.owner_token + )) + BEGIN + SELECT RAISE(ABORT, 'provider receipt requires the attempt live exact owner at database time'); + END + `) + + yield* tx.run("DROP TRIGGER IF EXISTS session_provider_attempt_legal_update") + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_legal_update + BEFORE UPDATE ON session_provider_attempt + WHEN NEW.attempt_id != OLD.attempt_id + OR NEW.session_id != OLD.session_id + OR NEW.activity_id != OLD.activity_id + OR NEW.provider_turn_seq != OLD.provider_turn_seq + OR NEW.selection_id != OLD.selection_id + OR NEW.projection_hash != OLD.projection_hash + OR NEW.request_hash != OLD.request_hash + OR NEW.provider_id != OLD.provider_id + OR NEW.owner_token IS NOT OLD.owner_token + OR NEW.parent_attempt_id IS NOT OLD.parent_attempt_id + OR NEW.idempotency_key IS NOT OLD.idempotency_key + OR NEW.created_at != OLD.created_at + OR (NEW.first_event_at IS NOT OLD.first_event_at AND NOT (OLD.state = 'dispatching' AND NEW.state = 'streaming' AND OLD.first_event_at IS NULL AND NEW.first_event_at IS NOT NULL)) + OR (NEW.error_code IS NOT OLD.error_code AND NEW.state NOT IN ('failed', 'indeterminate_after_crash')) + OR NOT ( + ( + OLD.state = 'prepared' AND NEW.state = 'prepared' + AND OLD.prepared_turn_hash IS NULL AND OLD.wire_request_hash IS NULL + AND NEW.prepared_turn_hash IS NOT NULL AND NEW.wire_request_hash IS NOT NULL + AND NEW.first_event_at IS OLD.first_event_at + AND NEW.settled_at IS OLD.settled_at + AND NEW.error_code IS OLD.error_code + ) OR + ( + OLD.state = 'prepared' AND NEW.state = 'prepared' + AND OLD.prepared_turn_hash IS NOT NULL AND OLD.wire_request_hash IS NOT NULL + AND NEW.prepared_turn_hash IS OLD.prepared_turn_hash + AND NEW.wire_request_hash IS OLD.wire_request_hash + AND NEW.first_event_at IS OLD.first_event_at + AND NEW.settled_at IS OLD.settled_at + AND NEW.error_code IS OLD.error_code + ) OR + ( + OLD.state = 'prepared' AND NEW.state = 'dispatching' + AND NEW.prepared_turn_hash IS NOT NULL AND NEW.wire_request_hash IS NOT NULL + AND NEW.prepared_turn_hash IS OLD.prepared_turn_hash + AND NEW.wire_request_hash IS OLD.wire_request_hash + AND NEW.first_event_at IS NULL AND NEW.settled_at IS NULL AND NEW.error_code IS NULL + ) OR + ( + OLD.state = 'prepared' AND NEW.state = 'failed' + AND NEW.prepared_turn_hash IS OLD.prepared_turn_hash + AND NEW.wire_request_hash IS OLD.wire_request_hash + AND NEW.first_event_at IS NULL AND NEW.settled_at IS NOT NULL + AND length(trim(COALESCE(NEW.error_code, ''))) > 0 + ) OR + ( + OLD.state = 'dispatching' AND NEW.state = 'streaming' + AND NEW.prepared_turn_hash IS OLD.prepared_turn_hash + AND NEW.wire_request_hash IS OLD.wire_request_hash + AND NEW.first_event_at IS NOT NULL AND NEW.settled_at IS NULL + ) OR + ( + OLD.state IN ('dispatching', 'streaming') AND NEW.state IN ('settled', 'failed') + AND NEW.prepared_turn_hash IS OLD.prepared_turn_hash + AND NEW.wire_request_hash IS OLD.wire_request_hash + AND NEW.settled_at IS NOT NULL + ) OR + ( + OLD.state = 'prepared' AND NEW.state = 'indeterminate_after_crash' + AND NEW.prepared_turn_hash IS OLD.prepared_turn_hash + AND NEW.wire_request_hash IS OLD.wire_request_hash + AND NEW.first_event_at IS OLD.first_event_at + AND NEW.settled_at IS NULL + AND NEW.error_code = 'process_recovery' + ) OR + ( + OLD.state IN ('dispatching', 'streaming') AND NEW.state = 'indeterminate_after_crash' + AND NEW.prepared_turn_hash IS OLD.prepared_turn_hash + AND NEW.wire_request_hash IS OLD.wire_request_hash + AND NEW.settled_at IS NULL AND NEW.error_code = 'process_recovery' + ) OR + ( + OLD.state = 'indeterminate_after_crash' + AND NEW.state IN ('resolved_abandoned', 'resolved_settled', 'resolved_replayed') + AND NEW.prepared_turn_hash IS OLD.prepared_turn_hash + AND NEW.wire_request_hash IS OLD.wire_request_hash + AND NEW.settled_at IS NOT NULL + ) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal session_provider_attempt transition'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_wire_identity_insert_guard + BEFORE INSERT ON session_provider_attempt + WHEN NEW.prepared_turn_hash IS NOT NULL OR NEW.wire_request_hash IS NOT NULL + BEGIN + SELECT RAISE(ABORT, 'provider attempt wire identity must be sealed after admission'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_provider_attempt_wire_identity_immutable + BEFORE UPDATE OF prepared_turn_hash, wire_request_hash ON session_provider_attempt + WHEN ( + NEW.prepared_turn_hash IS NOT OLD.prepared_turn_hash OR + NEW.wire_request_hash IS NOT OLD.wire_request_hash + ) AND NOT ( + OLD.state = 'prepared' AND NEW.state = 'prepared' + AND OLD.prepared_turn_hash IS NULL AND OLD.wire_request_hash IS NULL + AND NEW.prepared_turn_hash IS NOT NULL AND NEW.wire_request_hash IS NOT NULL + ) + BEGIN + SELECT RAISE(ABORT, 'provider attempt wire identity may only be sealed once'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_attempt_identity_insert_guard + BEFORE INSERT ON session_tool_request_receipt + WHEN NEW.provider_attempt_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 + FROM session_provider_attempt attempt + JOIN session_context_selection selection + ON selection.selection_id = attempt.selection_id + WHERE attempt.attempt_id = NEW.provider_attempt_id + AND attempt.session_id = NEW.session_id + AND attempt.owner_token = NEW.owner_token + AND attempt.selection_id = NEW.context_selection_id + AND attempt.activity_id = selection.activity_id + AND attempt.session_id = selection.session_id + AND attempt.projection_hash = selection.projection_hash + AND attempt.provider_id = NEW.provider_id + AND attempt.request_hash = NEW.request_input_hash + ) + BEGIN + SELECT RAISE(ABORT, 'provider receipt attempt identity mismatch'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_attempt_wire_guard + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_attempt_id IS NOT NULL + AND NEW.provider_state IN ('prepared', 'dispatching') + AND NOT EXISTS ( + SELECT 1 + FROM session_provider_attempt attempt + JOIN session_context_selection selection + ON selection.selection_id = attempt.selection_id + WHERE attempt.attempt_id = NEW.provider_attempt_id + AND attempt.session_id = NEW.session_id + AND attempt.owner_token = NEW.owner_token + AND attempt.selection_id = NEW.context_selection_id + AND attempt.activity_id = selection.activity_id + AND attempt.session_id = selection.session_id + AND attempt.projection_hash = selection.projection_hash + AND attempt.provider_id = NEW.provider_id + AND attempt.request_hash = NEW.request_input_hash + AND attempt.prepared_turn_hash = NEW.prepared_turn_hash + AND attempt.wire_request_hash = NEW.wire_request_hash + AND ( + (NEW.provider_state = 'prepared' AND attempt.state = 'prepared') OR + (NEW.provider_state = 'dispatching' AND attempt.state = 'dispatching') + ) + ) + BEGIN + SELECT RAISE(ABORT, 'provider receipt requires exact attempt wire identity'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812060000_activity_permission_authority_hardening.ts b/packages/core/src/database/migration/20260812060000_activity_permission_authority_hardening.ts new file mode 100644 index 000000000..8292770d2 --- /dev/null +++ b/packages/core/src/database/migration/20260812060000_activity_permission_authority_hardening.ts @@ -0,0 +1,82 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812060000_activity_permission_authority_hardening", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TEMP TABLE activity_permission_authority_preflight ( + valid INTEGER NOT NULL CHECK (valid = 1) + ) + `) + yield* tx.run(` + INSERT INTO activity_permission_authority_preflight(valid) + SELECT CASE WHEN EXISTS ( + SELECT 1 + FROM session_activity_permission_request request + JOIN session ON session.id = request.session_id + WHERE request.workspace_id IS NOT session.workspace_id + ) OR EXISTS ( + SELECT 1 + FROM session_activity_permission_owner_lease owner + WHERE owner.heartbeat_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + OR owner.lease_expires_at <= owner.heartbeat_at + OR owner.lease_expires_at - owner.heartbeat_at > 31536000000 + ) THEN 0 ELSE 1 END + `) + yield* tx.run("DROP TABLE activity_permission_authority_preflight") + yield* tx.run(` + CREATE TRIGGER session_activity_permission_request_workspace_insert_guard + BEFORE INSERT ON session_activity_permission_request + WHEN NOT EXISTS ( + SELECT 1 + FROM session + WHERE session.id = NEW.session_id + AND session.workspace_id IS NEW.workspace_id + ) + BEGIN + SELECT RAISE(ABORT, 'permission request workspace identity mismatch'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_workspace_pending_permission_guard + BEFORE UPDATE OF workspace_id ON session + WHEN NEW.workspace_id IS NOT OLD.workspace_id + AND EXISTS ( + SELECT 1 + FROM session_activity_permission_request request + WHERE request.session_id = OLD.id + AND request.state = 'pending' + ) + BEGIN + SELECT RAISE(ABORT, 'session workspace cannot change with pending permission authority'); + END + `) + yield* tx.run("DROP TRIGGER session_activity_permission_owner_lease_legal_update") + yield* tx.run(` + CREATE TRIGGER session_activity_permission_owner_lease_insert_guard + BEFORE INSERT ON session_activity_permission_owner_lease + WHEN NEW.heartbeat_at != CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + OR NEW.lease_expires_at <= NEW.heartbeat_at + OR NEW.lease_expires_at - NEW.heartbeat_at > 31536000000 + BEGIN + SELECT RAISE(ABORT, 'permission owner lease requires database observed time'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_owner_lease_legal_update + BEFORE UPDATE ON session_activity_permission_owner_lease + WHEN NEW.owner_id != OLD.owner_id + OR NEW.heartbeat_at != CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + OR NEW.heartbeat_at < OLD.heartbeat_at + OR NEW.heartbeat_at >= OLD.lease_expires_at + OR NEW.lease_expires_at <= NEW.heartbeat_at + OR NEW.lease_expires_at - NEW.heartbeat_at > 31536000000 + BEGIN + SELECT RAISE(ABORT, 'illegal session_activity_permission_owner_lease update'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812061000_provider_cross_state_recovery.ts b/packages/core/src/database/migration/20260812061000_provider_cross_state_recovery.ts new file mode 100644 index 000000000..03c5d93ec --- /dev/null +++ b/packages/core/src/database/migration/20260812061000_provider_cross_state_recovery.ts @@ -0,0 +1,193 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812061000_provider_cross_state_recovery", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TEMP TABLE provider_cross_state_recovery_preflight ( + valid INTEGER NOT NULL CHECK (valid = 1) + ) + `) + yield* tx.run(` + INSERT INTO provider_cross_state_recovery_preflight(valid) + SELECT CASE WHEN EXISTS ( + SELECT 1 + FROM session_tool_request_receipt receipt + LEFT JOIN session_provider_attempt attempt + ON attempt.attempt_id = receipt.provider_attempt_id + LEFT JOIN session_context_selection selection + ON selection.selection_id = attempt.selection_id + WHERE receipt.provider_attempt_id IS NOT NULL + AND ( + attempt.attempt_id IS NULL OR + attempt.session_id IS NOT receipt.session_id OR + attempt.owner_token IS NOT receipt.owner_token OR + attempt.selection_id IS NOT receipt.context_selection_id OR + attempt.activity_id IS NOT selection.activity_id OR + attempt.session_id IS NOT selection.session_id OR + attempt.projection_hash IS NOT selection.projection_hash OR + attempt.provider_id IS NOT receipt.provider_id OR + attempt.request_hash IS NOT receipt.request_input_hash OR + attempt.prepared_turn_hash IS NOT receipt.prepared_turn_hash OR + attempt.wire_request_hash IS NOT receipt.wire_request_hash + ) + ) OR EXISTS ( + SELECT 1 + FROM compaction_run continuation + JOIN session_tool_request_receipt receipt + ON receipt.receipt_id = continuation.continuation_receipt_id + JOIN session_provider_attempt attempt + ON attempt.attempt_id = receipt.provider_attempt_id + WHERE receipt.provider_state IN ('preparing', 'prepared') + AND attempt.state != 'prepared' + AND ( + continuation.session_id IS NOT receipt.session_id OR + continuation.state != 'committed' OR + continuation.continuation_state NOT IN ('admitted', 'dispatching', 'indeterminate') + ) + ) THEN 0 ELSE 1 END + `) + yield* tx.run("DROP TABLE provider_cross_state_recovery_preflight") + yield* tx.run("DROP TRIGGER IF EXISTS session_tool_request_receipt_provider_transition") + + // Keep the attempt as the physical-dispatch evidence. Only its stale receipt, + // continuation, and Session history authorities are moved into fail-closed recovery. + yield* tx.run(` + UPDATE compaction_run + SET continuation_state = 'dispatching', + continuation_dispatching_at = COALESCE( + continuation_dispatching_at, + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + ), + continuation_error_code = 'provider_attempt_started_before_receipt_state_recovery' + WHERE state = 'committed' + AND continuation_state = 'admitted' + AND continuation_receipt_id IN ( + SELECT receipt.receipt_id + FROM session_tool_request_receipt receipt + JOIN session_provider_attempt attempt + ON attempt.attempt_id = receipt.provider_attempt_id + WHERE receipt.provider_state IN ('preparing', 'prepared') + AND attempt.state != 'prepared' + ) + `) + yield* tx.run(` + UPDATE compaction_run + SET continuation_state = 'indeterminate', + continuation_terminal_at = COALESCE( + continuation_terminal_at, + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + ), + continuation_error_code = 'provider_started_outcome_unknown_after_process_restart' + WHERE state = 'committed' + AND continuation_state = 'dispatching' + AND continuation_receipt_id IN ( + SELECT receipt.receipt_id + FROM session_tool_request_receipt receipt + JOIN session_provider_attempt attempt + ON attempt.attempt_id = receipt.provider_attempt_id + WHERE receipt.provider_state IN ('preparing', 'prepared') + AND attempt.state != 'prepared' + ) + `) + yield* tx.run(` + UPDATE session_prompt_epoch + SET authority_state = 'recovery_required', + recovery_reason = 'provider outcome is unknown after process restart' + WHERE state = 'active' + AND session_id IN ( + SELECT receipt.session_id + FROM session_tool_request_receipt receipt + JOIN session_provider_attempt attempt + ON attempt.attempt_id = receipt.provider_attempt_id + WHERE receipt.provider_state IN ('preparing', 'prepared') + AND attempt.state != 'prepared' + ) + `) + yield* tx.run(` + INSERT INTO session_history_state ( + session_id, state, reason, time_created, time_updated + ) + SELECT DISTINCT + receipt.session_id, + 'recovery_required', + 'provider outcome is unknown after process restart', + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + FROM session_tool_request_receipt receipt + JOIN session_provider_attempt attempt + ON attempt.attempt_id = receipt.provider_attempt_id + WHERE receipt.provider_state IN ('preparing', 'prepared') + AND attempt.state != 'prepared' + ON CONFLICT(session_id) DO UPDATE SET + state = 'recovery_required', + reason = 'provider outcome is unknown after process restart', + time_updated = excluded.time_updated + `) + yield* tx.run(` + UPDATE session_tool_request_receipt + SET provider_state = 'indeterminate_after_crash', + terminal_at = CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + request_error_code = 'provider_started_outcome_unknown_after_process_restart' + WHERE provider_state IN ('preparing', 'prepared') + AND EXISTS ( + SELECT 1 + FROM session_provider_attempt attempt + WHERE attempt.attempt_id = session_tool_request_receipt.provider_attempt_id + AND attempt.state != 'prepared' + ) + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_provider_transition + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_state != OLD.provider_state AND NOT ( + (OLD.provider_state = 'preparing' AND NEW.provider_state IN ('prepared', 'failed')) OR + (OLD.provider_state = 'prepared' AND NEW.provider_state IN ('dispatching', 'failed')) OR + ( + OLD.provider_state IN ('preparing', 'prepared') + AND NEW.provider_state = 'indeterminate_after_crash' + ) OR + (OLD.provider_state = 'dispatching' AND NEW.provider_state IN ('streaming', 'settled', 'failed', 'indeterminate_after_crash')) OR + (OLD.provider_state = 'streaming' AND NEW.provider_state IN ('settled', 'failed', 'indeterminate_after_crash')) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal provider receipt transition'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_cross_state_recovery_guard + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN OLD.provider_state IN ('preparing', 'prepared') + AND NEW.provider_state = 'indeterminate_after_crash' + AND ( + NEW.terminal_at IS NULL OR + NEW.request_error_code != 'provider_started_outcome_unknown_after_process_restart' OR + NEW.provider_attempt_id IS NULL OR + NOT EXISTS ( + SELECT 1 + FROM session_provider_attempt attempt + JOIN session_context_selection selection + ON selection.selection_id = attempt.selection_id + WHERE attempt.attempt_id = NEW.provider_attempt_id + AND attempt.session_id = NEW.session_id + AND attempt.owner_token = NEW.owner_token + AND attempt.selection_id = NEW.context_selection_id + AND attempt.activity_id = selection.activity_id + AND attempt.session_id = selection.session_id + AND attempt.projection_hash = selection.projection_hash + AND attempt.provider_id = NEW.provider_id + AND attempt.request_hash = NEW.request_input_hash + AND attempt.prepared_turn_hash = NEW.prepared_turn_hash + AND attempt.wire_request_hash = NEW.wire_request_hash + AND attempt.state != 'prepared' + ) + ) + BEGIN + SELECT RAISE(ABORT, 'provider receipt cross-state recovery requires exact physical-start evidence'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812114412_task_structured_output_receipt_schema.ts b/packages/core/src/database/migration/20260812114412_task_structured_output_receipt_schema.ts new file mode 100644 index 000000000..826f34fae --- /dev/null +++ b/packages/core/src/database/migration/20260812114412_task_structured_output_receipt_schema.ts @@ -0,0 +1,11 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812114412_task_structured_output_receipt_schema", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(`ALTER TABLE \`task_run\` ADD \`structured_output_receipt\` text;`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812114500_task_structured_output_receipt_authority.ts b/packages/core/src/database/migration/20260812114500_task_structured_output_receipt_authority.ts new file mode 100644 index 000000000..97ec8f069 --- /dev/null +++ b/packages/core/src/database/migration/20260812114500_task_structured_output_receipt_authority.ts @@ -0,0 +1,195 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812114500_task_structured_output_receipt_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TEMP TABLE task_structured_output_receipt_preflight ( + valid INTEGER NOT NULL CHECK (valid = 1) + ) + `) + yield* tx.run(` + INSERT INTO task_structured_output_receipt_preflight(valid) + SELECT CASE WHEN EXISTS ( + SELECT 1 + FROM task_run + WHERE phase = 'settled' + AND reason IN ( + 'structured_output_valid', + 'structured_output_text_fallback', + 'structured_output_degraded_text' + ) + AND ( + state != 'completed' OR + attempts NOT IN (1, 2) OR + (reason IN ('structured_output_text_fallback', 'structured_output_degraded_text') AND attempts != 2) OR + (reason IN ('structured_output_valid', 'structured_output_text_fallback') AND + structured_result_message_id IS NULL) OR + (reason = 'structured_output_degraded_text' AND ( + structured_result_message_id IS NOT NULL OR + output IS NULL OR + json_valid(output) != 1 OR + json_extract(output, '$._degraded') IS NOT 1 OR + json_extract(output, '$._attempts') IS NOT attempts OR + COALESCE(json_type(output, '$._reason'), 'missing') != 'text' OR + json_extract(output, '$._reason') NOT IN ( + 'structured_output_missing', + 'structured_output_invalid' + ) + )) + ) + ) THEN 0 ELSE 1 END + `) + yield* tx.run("DROP TABLE task_structured_output_receipt_preflight") + yield* tx.run(` + UPDATE task_run + SET structured_output_receipt = CASE reason + WHEN 'structured_output_text_fallback' + THEN json_object('attempt', attempts, 'transport', 'text_fallback') + WHEN 'structured_output_degraded_text' + THEN json_object( + 'attempt', attempts, + 'transport', 'degraded_text', + 'reason', json_extract(output, '$._reason') + ) + ELSE NULL + END + WHERE phase = 'settled' + AND reason IN ('structured_output_text_fallback', 'structured_output_degraded_text') + `) + yield* tx.run(` + CREATE TRIGGER task_run_structured_output_receipt_insert_guard + BEFORE INSERT ON task_run + WHEN NOT ( + CASE + WHEN NEW.structured_output_receipt IS NULL THEN + NEW.phase != 'settled' OR NEW.reason IS NULL OR NEW.reason NOT IN ( + 'structured_output_valid', + 'structured_output_text_fallback', + 'structured_output_degraded_text' + ) + WHEN NEW.phase != 'settled' OR NEW.state != 'completed' OR NEW.reason IS NULL OR + NEW.reason NOT IN ( + 'structured_output_valid', + 'structured_output_text_fallback', + 'structured_output_degraded_text' + ) THEN 0 + WHEN json_valid(NEW.structured_output_receipt) != 1 OR + json_type(NEW.structured_output_receipt) != 'object' THEN 0 + WHEN json_type(NEW.structured_output_receipt, '$.attempt') != 'integer' OR + json_extract(NEW.structured_output_receipt, '$.attempt') != NEW.attempts OR + NEW.attempts NOT IN (1, 2) OR + json_type(NEW.structured_output_receipt, '$.transport') != 'text' THEN 0 + WHEN NEW.reason = 'structured_output_valid' THEN + json_extract(NEW.structured_output_receipt, '$.transport') = 'structured' AND + json_type(NEW.structured_output_receipt, '$.reason') IS NULL AND + (SELECT count(*) FROM json_each(NEW.structured_output_receipt)) = 2 AND + NEW.structured_result_message_id IS NOT NULL + WHEN NEW.reason = 'structured_output_text_fallback' THEN + json_extract(NEW.structured_output_receipt, '$.transport') = 'text_fallback' AND + NEW.attempts = 2 AND + json_type(NEW.structured_output_receipt, '$.reason') IS NULL AND + (SELECT count(*) FROM json_each(NEW.structured_output_receipt)) = 2 AND + NEW.structured_result_message_id IS NOT NULL + ELSE + json_extract(NEW.structured_output_receipt, '$.transport') = 'degraded_text' AND + NEW.attempts = 2 AND + json_extract(NEW.structured_output_receipt, '$.reason') IN ( + 'structured_output_missing', + 'structured_output_invalid' + ) AND + (SELECT count(*) FROM json_each(NEW.structured_output_receipt)) = 3 AND + NEW.structured_result_message_id IS NULL AND + NEW.output IS NOT NULL AND json_valid(NEW.output) = 1 AND + json_extract(NEW.output, '$._degraded') IS 1 AND + json_extract(NEW.output, '$._attempts') IS NEW.attempts AND + json_extract(NEW.output, '$._reason') IS + json_extract(NEW.structured_output_receipt, '$.reason') + END + ) + BEGIN + SELECT RAISE(ABORT, 'invalid task structured output receipt'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_run_structured_output_receipt_update_guard + BEFORE UPDATE ON task_run + WHEN NOT ( + CASE + WHEN NEW.structured_output_receipt IS NULL THEN + NEW.phase != 'settled' OR NEW.reason IS NULL OR NEW.reason NOT IN ( + 'structured_output_valid', + 'structured_output_text_fallback', + 'structured_output_degraded_text' + ) OR ( + OLD.structured_output_receipt IS NULL AND + OLD.phase = 'settled' AND OLD.state = 'completed' AND + OLD.reason = 'structured_output_valid' AND + NEW.phase IS OLD.phase AND NEW.state IS OLD.state AND + NEW.reason IS OLD.reason AND NEW.attempts IS OLD.attempts AND + NEW.raw_result_message_id IS OLD.raw_result_message_id AND + NEW.structured_result_message_id IS OLD.structured_result_message_id AND + NEW.output IS OLD.output AND NEW.error IS OLD.error AND + NEW.result_hash IS OLD.result_hash AND NEW.usage IS OLD.usage AND + NEW.time_settled IS OLD.time_settled AND + NEW.control_state IS OLD.control_state AND + NEW.finalizer_input_message_id IS OLD.finalizer_input_message_id AND + NEW.execution_spec IS OLD.execution_spec + ) + WHEN NEW.phase != 'settled' OR NEW.state != 'completed' OR NEW.reason IS NULL OR + NEW.reason NOT IN ( + 'structured_output_valid', + 'structured_output_text_fallback', + 'structured_output_degraded_text' + ) THEN 0 + WHEN json_valid(NEW.structured_output_receipt) != 1 OR + json_type(NEW.structured_output_receipt) != 'object' THEN 0 + WHEN json_type(NEW.structured_output_receipt, '$.attempt') != 'integer' OR + json_extract(NEW.structured_output_receipt, '$.attempt') != NEW.attempts OR + NEW.attempts NOT IN (1, 2) OR + json_type(NEW.structured_output_receipt, '$.transport') != 'text' THEN 0 + WHEN NEW.reason = 'structured_output_valid' THEN + json_extract(NEW.structured_output_receipt, '$.transport') = 'structured' AND + json_type(NEW.structured_output_receipt, '$.reason') IS NULL AND + (SELECT count(*) FROM json_each(NEW.structured_output_receipt)) = 2 AND + NEW.structured_result_message_id IS NOT NULL + WHEN NEW.reason = 'structured_output_text_fallback' THEN + json_extract(NEW.structured_output_receipt, '$.transport') = 'text_fallback' AND + NEW.attempts = 2 AND + json_type(NEW.structured_output_receipt, '$.reason') IS NULL AND + (SELECT count(*) FROM json_each(NEW.structured_output_receipt)) = 2 AND + NEW.structured_result_message_id IS NOT NULL + ELSE + json_extract(NEW.structured_output_receipt, '$.transport') = 'degraded_text' AND + NEW.attempts = 2 AND + json_extract(NEW.structured_output_receipt, '$.reason') IN ( + 'structured_output_missing', + 'structured_output_invalid' + ) AND + (SELECT count(*) FROM json_each(NEW.structured_output_receipt)) = 3 AND + NEW.structured_result_message_id IS NULL AND + NEW.output IS NOT NULL AND json_valid(NEW.output) = 1 AND + json_extract(NEW.output, '$._degraded') IS 1 AND + json_extract(NEW.output, '$._attempts') IS NEW.attempts AND + json_extract(NEW.output, '$._reason') IS + json_extract(NEW.structured_output_receipt, '$.reason') + END + ) + BEGIN + SELECT RAISE(ABORT, 'invalid task structured output receipt'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_run_structured_output_receipt_immutable + BEFORE UPDATE OF structured_output_receipt ON task_run + WHEN OLD.structured_output_receipt IS NOT NULL + AND NEW.structured_output_receipt IS NOT OLD.structured_output_receipt + BEGIN + SELECT RAISE(ABORT, 'task structured output receipt is immutable'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812151010_confused_polaris.ts b/packages/core/src/database/migration/20260812151010_confused_polaris.ts new file mode 100644 index 000000000..a32a3a26d --- /dev/null +++ b/packages/core/src/database/migration/20260812151010_confused_polaris.ts @@ -0,0 +1,41 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812151010_confused_polaris", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`session_activity_permission_effect_dispatch\` ( + \`receipt_id\` text PRIMARY KEY, + \`request_id\` text NOT NULL, + \`activity_kind\` text NOT NULL, + \`activity_id\` text NOT NULL, + \`session_id\` text NOT NULL, + \`project_id\` text NOT NULL, + \`workspace_id\` text, + \`tool_message_id\` text NOT NULL, + \`tool_call_id\` text NOT NULL, + \`tool_name\` text NOT NULL, + \`consumer_id\` text NOT NULL, + \`idempotency_key\` text NOT NULL, + \`owner_id\` text NOT NULL, + \`state\` text NOT NULL, + \`version\` integer NOT NULL, + \`outcome\` text, + \`result_json\` text, + \`result_hash\` text, + \`started_at\` integer NOT NULL, + \`settled_at\` integer, + CONSTRAINT \`fk_session_activity_permission_effect_dispatch_request_id_session_activity_permission_request_request_id_fk\` FOREIGN KEY (\`request_id\`) REFERENCES \`session_activity_permission_request\`(\`request_id\`) ON DELETE CASCADE, + CONSTRAINT \`fk_session_activity_permission_effect_dispatch_session_id_session_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session\`(\`id\`) ON DELETE CASCADE, + CONSTRAINT \`fk_session_activity_permission_effect_dispatch_project_id_project_id_fk\` FOREIGN KEY (\`project_id\`) REFERENCES \`project\`(\`id\`) ON DELETE CASCADE + ); + `) + yield* tx.run(`CREATE UNIQUE INDEX \`session_activity_permission_effect_dispatch_request_idx\` ON \`session_activity_permission_effect_dispatch\` (\`request_id\`);`) + yield* tx.run(`CREATE UNIQUE INDEX \`session_activity_permission_effect_dispatch_idempotency_idx\` ON \`session_activity_permission_effect_dispatch\` (\`idempotency_key\`);`) + yield* tx.run(`CREATE INDEX \`session_activity_permission_effect_dispatch_activity_idx\` ON \`session_activity_permission_effect_dispatch\` (\`activity_kind\`,\`activity_id\`,\`state\`,\`started_at\`);`) + yield* tx.run(`CREATE INDEX \`session_activity_permission_effect_dispatch_owner_idx\` ON \`session_activity_permission_effect_dispatch\` (\`owner_id\`,\`state\`,\`started_at\`);`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812154657_task_structured_output_evidence.ts b/packages/core/src/database/migration/20260812154657_task_structured_output_evidence.ts new file mode 100644 index 000000000..9c8a8c009 --- /dev/null +++ b/packages/core/src/database/migration/20260812154657_task_structured_output_evidence.ts @@ -0,0 +1,36 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812154657_task_structured_output_evidence", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`task_structured_output_evidence\` ( + \`run_id\` text PRIMARY KEY, + \`child_session_id\` text NOT NULL, + \`terminal_state\` text NOT NULL, + \`attempts\` integer NOT NULL, + \`contract_json\` text NOT NULL, + \`contract_hash\` text NOT NULL, + \`raw_result_message_id\` text NOT NULL, + \`raw_message_json\` text NOT NULL, + \`raw_parts_json\` text NOT NULL, + \`raw_material_hash\` text NOT NULL, + \`result_message_id\` text, + \`result_message_json\` text, + \`result_parts_json\` text, + \`result_material_hash\` text, + \`output\` text, + \`output_hash\` text, + \`structured_output_receipt\` text, + \`failure_code\` text, + \`created_at\` integer NOT NULL, + CONSTRAINT \`fk_task_structured_output_evidence_run_id_task_run_run_id_fk\` FOREIGN KEY (\`run_id\`) REFERENCES \`task_run\`(\`run_id\`) ON DELETE CASCADE + ); + `) + yield* tx.run(`CREATE INDEX \`task_structured_output_evidence_raw_message_idx\` ON \`task_structured_output_evidence\` (\`raw_result_message_id\`);`) + yield* tx.run(`CREATE INDEX \`task_structured_output_evidence_result_message_idx\` ON \`task_structured_output_evidence\` (\`result_message_id\`);`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812155138_task_structured_output_evidence_parts.ts b/packages/core/src/database/migration/20260812155138_task_structured_output_evidence_parts.ts new file mode 100644 index 000000000..9eae29a8f --- /dev/null +++ b/packages/core/src/database/migration/20260812155138_task_structured_output_evidence_parts.ts @@ -0,0 +1,25 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812155138_task_structured_output_evidence_parts", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`task_structured_output_evidence_part\` ( + \`run_id\` text NOT NULL, + \`role\` text NOT NULL, + \`ordinal\` integer NOT NULL, + \`part_id\` text NOT NULL, + \`message_id\` text NOT NULL, + \`session_id\` text NOT NULL, + \`part_json\` text NOT NULL, + CONSTRAINT \`task_structured_output_evidence_part_pk\` PRIMARY KEY(\`run_id\`, \`role\`, \`part_id\`), + CONSTRAINT \`fk_task_structured_output_evidence_part_run_id_task_structured_output_evidence_run_id_fk\` FOREIGN KEY (\`run_id\`) REFERENCES \`task_structured_output_evidence\`(\`run_id\`) ON DELETE CASCADE + ); + `) + yield* tx.run(`CREATE UNIQUE INDEX \`task_structured_output_evidence_part_ordinal_idx\` ON \`task_structured_output_evidence_part\` (\`run_id\`,\`role\`,\`ordinal\`);`) + yield* tx.run(`CREATE INDEX \`task_structured_output_evidence_part_part_idx\` ON \`task_structured_output_evidence_part\` (\`part_id\`);`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812155242_task_structured_output_evidence_fence.ts b/packages/core/src/database/migration/20260812155242_task_structured_output_evidence_fence.ts new file mode 100644 index 000000000..e17b2baef --- /dev/null +++ b/packages/core/src/database/migration/20260812155242_task_structured_output_evidence_fence.ts @@ -0,0 +1,13 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812155242_task_structured_output_evidence_fence", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(`ALTER TABLE \`task_structured_output_evidence\` ADD \`owner_token\` text NOT NULL;`) + yield* tx.run(`ALTER TABLE \`task_structured_output_evidence\` ADD \`claim_generation\` integer NOT NULL;`) + yield* tx.run(`ALTER TABLE \`task_structured_output_evidence\` ADD \`expected_version\` integer NOT NULL;`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812162033_new_titanium_man.ts b/packages/core/src/database/migration/20260812162033_new_titanium_man.ts new file mode 100644 index 000000000..22799e39f --- /dev/null +++ b/packages/core/src/database/migration/20260812162033_new_titanium_man.ts @@ -0,0 +1,14 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812162033_new_titanium_man", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(`ALTER TABLE \`task_structured_output_evidence\` DROP COLUMN \`contract_hash\`;`) + yield* tx.run(`ALTER TABLE \`task_structured_output_evidence\` DROP COLUMN \`raw_material_hash\`;`) + yield* tx.run(`ALTER TABLE \`task_structured_output_evidence\` DROP COLUMN \`result_material_hash\`;`) + yield* tx.run(`ALTER TABLE \`task_structured_output_evidence\` DROP COLUMN \`output_hash\`;`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812184453_charming_anthem.ts b/packages/core/src/database/migration/20260812184453_charming_anthem.ts new file mode 100644 index 000000000..bf612f495 --- /dev/null +++ b/packages/core/src/database/migration/20260812184453_charming_anthem.ts @@ -0,0 +1,29 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812184453_charming_anthem", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`task_structured_finalizer_response\` ( + \`run_id\` text NOT NULL, + \`attempt\` integer NOT NULL, + \`child_session_id\` text NOT NULL, + \`owner_token\` text NOT NULL, + \`claim_generation\` integer NOT NULL, + \`expected_version\` integer NOT NULL, + \`source_message_id\` text NOT NULL, + \`request_message_id\` text NOT NULL, + \`response_message_id\` text NOT NULL, + \`response_message_json\` text NOT NULL, + \`created_at\` integer NOT NULL, + CONSTRAINT \`task_structured_finalizer_response_pk\` PRIMARY KEY(\`run_id\`, \`attempt\`), + CONSTRAINT \`fk_task_structured_finalizer_response_run_id_task_run_run_id_fk\` FOREIGN KEY (\`run_id\`) REFERENCES \`task_run\`(\`run_id\`) ON DELETE CASCADE + ); + `) + yield* tx.run(`ALTER TABLE \`learning_job\` ADD \`expected_result_ref\` text;`) + yield* tx.run(`CREATE UNIQUE INDEX \`task_structured_finalizer_response_message_idx\` ON \`task_structured_finalizer_response\` (\`response_message_id\`);`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812210000_task_execution_spec_authority.ts b/packages/core/src/database/migration/20260812210000_task_execution_spec_authority.ts new file mode 100644 index 000000000..239a8e425 --- /dev/null +++ b/packages/core/src/database/migration/20260812210000_task_execution_spec_authority.ts @@ -0,0 +1,212 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +const safeExecutionSpec = "CASE WHEN json_valid(execution_spec) = 1 THEN execution_spec ELSE '{}' END" +const safeNewExecutionSpec = "CASE WHEN json_valid(NEW.execution_spec) = 1 THEN NEW.execution_spec ELSE '{}' END" + +const invalidStructuredContract = (value: string) => ` + json_type(${value}, '$.structuredOutput') IS NOT NULL AND ( + json_type(${value}, '$.structuredOutput') != 'object' OR + json_type(${value}, '$.agent') IS NOT 'text' OR + length(trim(COALESCE(json_extract(${value}, '$.agent'), ''))) = 0 OR + json_type(${value}, '$.model') IS NOT 'object' OR + json_type(${value}, '$.model.providerID') IS NOT 'text' OR + length(trim(COALESCE(json_extract(${value}, '$.model.providerID'), ''))) = 0 OR + json_type(${value}, '$.model.modelID') IS NOT 'text' OR + length(trim(COALESCE(json_extract(${value}, '$.model.modelID'), ''))) = 0 OR + json_type(${value}, '$.structuredOutput.schema') != 'object' OR + json_type(${value}, '$.structuredOutput.allowTextFallback') NOT IN ('true', 'false') OR + json_type(${value}, '$.structuredOutput.receiptVersion') != 'integer' OR + json_extract(${value}, '$.structuredOutput.receiptVersion') != 1 OR + json_type(${value}, '$.structuredOutput.maxAttempts') != 'integer' OR + json_extract(${value}, '$.structuredOutput.maxAttempts') != 2 OR + (SELECT count(*) FROM json_each(json_extract(${value}, '$.structuredOutput'))) != 4 + ) +` + +const invalidStructuredCompletion = (value: string, row: "NEW" | "OLD") => ` + json_type(${value}, '$.structuredOutput') = 'object' AND + ${row}.state = 'completed' AND ( + ${row}.phase != 'settled' OR + ${row}.reason NOT IN ( + 'structured_output_valid', + 'structured_output_text_fallback', + 'structured_output_degraded_text' + ) OR + ${row}.structured_output_receipt IS NULL OR + ${row}.raw_result_message_id IS NULL OR + ( + ${row}.reason = 'structured_output_text_fallback' AND + json_extract(${value}, '$.structuredOutput.allowTextFallback') IS NOT 1 + ) + ) +` + +const invalidStructuredEvidence = (row: "NEW" | "task_run") => ` + ${row}.structured_output_receipt IS NOT NULL AND ( + ${row}.raw_result_message_id IS NULL OR + NOT EXISTS ( + SELECT 1 FROM message + WHERE message.id = ${row}.raw_result_message_id + AND message.session_id = ${row}.child_session_id + AND json_valid(message.data) = 1 + AND json_extract(message.data, '$.role') = 'assistant' + ) OR ( + ${row}.reason IN ('structured_output_valid', 'structured_output_text_fallback') AND ( + ${row}.structured_result_message_id IS NULL OR + NOT EXISTS ( + SELECT 1 FROM message + WHERE message.id = ${row}.structured_result_message_id + AND message.session_id = ${row}.child_session_id + AND json_valid(message.data) = 1 + AND json_extract(message.data, '$.role') = 'assistant' + ) + ) + ) + ) +` + +export default { + id: "20260812210000_task_execution_spec_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TEMP TABLE task_execution_spec_preflight ( + valid INTEGER NOT NULL CHECK (valid = 1) + ) + `) + yield* tx.run(` + INSERT INTO task_execution_spec_preflight(valid) + SELECT CASE WHEN EXISTS ( + SELECT 1 + FROM task_run + WHERE execution_spec IS NOT NULL AND ( + json_valid(execution_spec) != 1 OR + json_type(${safeExecutionSpec}) != 'object' OR + (${invalidStructuredContract(safeExecutionSpec)}) + ) + OR (${invalidStructuredEvidence("task_run")}) + ) THEN 0 ELSE 1 END + `) + yield* tx.run("DROP TABLE task_execution_spec_preflight") + yield* tx.run(` + CREATE TRIGGER task_run_execution_spec_insert_guard + BEFORE INSERT ON task_run + WHEN NEW.execution_spec IS NOT NULL AND ( + json_valid(NEW.execution_spec) != 1 OR + json_type(${safeNewExecutionSpec}) != 'object' OR + (${invalidStructuredContract(safeNewExecutionSpec)}) + ) + BEGIN + SELECT RAISE(ABORT, 'invalid task execution spec'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_run_execution_spec_immutable + BEFORE UPDATE OF execution_spec ON task_run + WHEN NEW.execution_spec IS NOT OLD.execution_spec + BEGIN + SELECT RAISE(ABORT, 'task execution spec is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_run_structured_receipt_requires_execution_spec_insert + BEFORE INSERT ON task_run + WHEN ( + NEW.structured_output_receipt IS NOT NULL AND ( + NEW.execution_spec IS NULL OR + json_valid(NEW.execution_spec) != 1 OR + json_type(${safeNewExecutionSpec}, '$.structuredOutput') != 'object' OR + (${invalidStructuredContract(safeNewExecutionSpec)}) + ) + ) OR ( + NEW.execution_spec IS NOT NULL AND + json_valid(NEW.execution_spec) = 1 AND + (${invalidStructuredCompletion(safeNewExecutionSpec, "NEW")}) + ) OR (${invalidStructuredEvidence("NEW")}) + BEGIN + SELECT RAISE(ABORT, 'task structured receipt requires frozen execution spec'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_run_structured_receipt_requires_execution_spec_update + BEFORE UPDATE ON task_run + WHEN ( + OLD.structured_output_receipt IS NULL AND NEW.structured_output_receipt IS NOT NULL AND ( + NEW.execution_spec IS NULL OR + json_valid(NEW.execution_spec) != 1 OR + json_type(${safeNewExecutionSpec}, '$.structuredOutput') != 'object' OR + (${invalidStructuredContract(safeNewExecutionSpec)}) + ) + ) OR ( + ( + NEW.phase IS NOT OLD.phase OR + NEW.state IS NOT OLD.state OR + NEW.reason IS NOT OLD.reason OR + NEW.attempts IS NOT OLD.attempts OR + NEW.structured_result_message_id IS NOT OLD.structured_result_message_id OR + NEW.structured_output_receipt IS NOT OLD.structured_output_receipt + ) AND + NEW.execution_spec IS NOT NULL AND + json_valid(NEW.execution_spec) = 1 AND + (${invalidStructuredCompletion(safeNewExecutionSpec, "NEW")}) + ) OR (${invalidStructuredEvidence("NEW")}) + BEGIN + SELECT RAISE(ABORT, 'task structured receipt requires frozen execution spec'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_run_historical_structured_terminal_immutable + BEFORE UPDATE ON task_run + WHEN OLD.execution_spec IS NOT NULL AND + json_valid(OLD.execution_spec) = 1 AND + (${invalidStructuredCompletion( + "CASE WHEN json_valid(OLD.execution_spec) = 1 THEN OLD.execution_spec ELSE '{}' END", + "OLD", + )}) AND ( + NEW.phase IS NOT OLD.phase OR + NEW.state IS NOT OLD.state OR + NEW.reason IS NOT OLD.reason OR + NEW.attempts IS NOT OLD.attempts OR + NEW.raw_result_message_id IS NOT OLD.raw_result_message_id OR + NEW.structured_result_message_id IS NOT OLD.structured_result_message_id OR + NEW.structured_output_receipt IS NOT OLD.structured_output_receipt OR + NEW.output IS NOT OLD.output OR + NEW.error IS NOT OLD.error OR + NEW.result_hash IS NOT OLD.result_hash OR + NEW.usage IS NOT OLD.usage OR + NEW.time_settled IS NOT OLD.time_settled OR + NEW.control_state IS NOT OLD.control_state OR + NEW.finalizer_input_message_id IS NOT OLD.finalizer_input_message_id + ) + BEGIN + SELECT RAISE(ABORT, 'historical task structured terminal is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_run_structured_terminal_immutable + BEFORE UPDATE ON task_run + WHEN OLD.structured_output_receipt IS NOT NULL AND ( + NEW.phase IS NOT OLD.phase OR + NEW.state IS NOT OLD.state OR + NEW.reason IS NOT OLD.reason OR + NEW.attempts IS NOT OLD.attempts OR + NEW.raw_result_message_id IS NOT OLD.raw_result_message_id OR + NEW.structured_result_message_id IS NOT OLD.structured_result_message_id OR + NEW.structured_output_receipt IS NOT OLD.structured_output_receipt OR + NEW.output IS NOT OLD.output OR + NEW.error IS NOT OLD.error OR + NEW.result_hash IS NOT OLD.result_hash OR + NEW.usage IS NOT OLD.usage OR + NEW.time_settled IS NOT OLD.time_settled OR + NEW.control_state IS NOT OLD.control_state OR + NEW.execution_spec IS NOT OLD.execution_spec OR + NEW.finalizer_input_message_id IS NOT OLD.finalizer_input_message_id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured terminal is immutable'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812213000_permission_effect_dispatch_authority.ts b/packages/core/src/database/migration/20260812213000_permission_effect_dispatch_authority.ts new file mode 100644 index 000000000..9b2a99f58 --- /dev/null +++ b/packages/core/src/database/migration/20260812213000_permission_effect_dispatch_authority.ts @@ -0,0 +1,205 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +const databaseNow = "CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)" + +export default { + id: "20260812213000_permission_effect_dispatch_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TRIGGER session_activity_permission_effect_dispatch_insert_guard + BEFORE INSERT ON session_activity_permission_effect_dispatch + WHEN length(trim(NEW.receipt_id)) = 0 OR + length(trim(NEW.request_id)) = 0 OR + length(trim(NEW.activity_id)) = 0 OR + length(trim(NEW.tool_message_id)) = 0 OR + length(trim(NEW.tool_call_id)) = 0 OR + length(trim(NEW.tool_name)) = 0 OR + length(trim(NEW.consumer_id)) = 0 OR + length(trim(NEW.idempotency_key)) = 0 OR + length(trim(NEW.owner_id)) = 0 OR + NEW.activity_kind NOT IN ('legacy', 'v2') OR + NEW.state != 'started' OR NEW.version != 1 OR + NEW.outcome IS NOT NULL OR NEW.result_json IS NOT NULL OR + NEW.result_hash IS NOT NULL OR NEW.settled_at IS NOT NULL OR + NEW.started_at < ${databaseNow} - 1000 OR NEW.started_at > ${databaseNow} OR + NOT EXISTS ( + SELECT 1 + FROM session_activity_permission_request request + JOIN session_activity_permission_decision decision + ON decision.request_id = request.request_id + JOIN session_activity_objective objective + ON objective.activity_kind = request.activity_kind + AND objective.activity_id = request.activity_id + JOIN session session ON session.id = request.session_id + JOIN session_activity_permission_owner_lease owner + ON owner.owner_id = NEW.owner_id + WHERE request.request_id = NEW.request_id + AND request.request_kind = 'tool' + AND request.state IN ('approved_once', 'approved_always') + AND decision.decision = request.state + AND (request.expires_at IS NULL OR request.expires_at > ${databaseNow}) + AND (decision.expires_at IS NULL OR decision.expires_at > ${databaseNow}) + AND objective.state = 'active' + AND objective.session_id = request.session_id + AND session.project_id = request.project_id + AND owner.lease_expires_at > ${databaseNow} + AND NEW.activity_kind = request.activity_kind + AND NEW.activity_id = request.activity_id + AND NEW.session_id = request.session_id + AND NEW.project_id = request.project_id + AND NEW.workspace_id IS request.workspace_id + AND NEW.tool_message_id = request.tool_message_id + AND NEW.tool_call_id = request.tool_call_id + AND ( + (request.activity_kind = 'legacy' AND EXISTS ( + SELECT 1 FROM session_legacy_activity activity + WHERE activity.activity_id = request.activity_id + AND activity.session_id = request.session_id + AND activity.state = 'active' + )) OR + (request.activity_kind = 'v2' AND EXISTS ( + SELECT 1 FROM session_activity activity + WHERE activity.activity_id = request.activity_id + AND activity.session_id = request.session_id + AND activity.state = 'active' + )) + ) + AND ( + (decision.decision = 'approved_once' AND EXISTS ( + SELECT 1 + FROM session_activity_permission_once_consumption consumption + WHERE consumption.request_id = request.request_id + AND consumption.consumer_id = NEW.consumer_id + AND consumption.consumed_at = NEW.started_at + )) OR + (decision.decision = 'approved_always' AND NOT EXISTS ( + SELECT 1 + FROM session_activity_permission_once_consumption consumption + WHERE consumption.request_id = request.request_id + )) + ) + ) + BEGIN + SELECT RAISE(ABORT, 'invalid permission effect dispatch admission'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_effect_dispatch_update_guard + BEFORE UPDATE ON session_activity_permission_effect_dispatch + WHEN NEW.receipt_id != OLD.receipt_id OR + NEW.request_id != OLD.request_id OR + NEW.activity_kind != OLD.activity_kind OR + NEW.activity_id != OLD.activity_id OR + NEW.session_id != OLD.session_id OR + NEW.project_id != OLD.project_id OR + NEW.workspace_id IS NOT OLD.workspace_id OR + NEW.tool_message_id != OLD.tool_message_id OR + NEW.tool_call_id != OLD.tool_call_id OR + NEW.tool_name != OLD.tool_name OR + NEW.consumer_id != OLD.consumer_id OR + NEW.idempotency_key != OLD.idempotency_key OR + NEW.owner_id != OLD.owner_id OR + NEW.started_at != OLD.started_at OR + OLD.state != 'started' OR NEW.version != OLD.version + 1 OR + NEW.state NOT IN ('settled', 'unknown') OR + NEW.settled_at IS NULL OR NEW.settled_at < OLD.started_at OR + NEW.settled_at > ${databaseNow} OR + ( + NEW.state = 'settled' AND ( + NEW.outcome NOT IN ('success', 'failure') OR + NEW.result_json IS NULL OR json_valid(NEW.result_json) != 1 OR + NEW.result_hash IS NULL OR length(NEW.result_hash) != 64 OR + NEW.result_hash GLOB '*[^0-9a-f]*' OR + NOT EXISTS ( + SELECT 1 + FROM session_activity_permission_owner_lease owner + WHERE owner.owner_id = OLD.owner_id + AND owner.lease_expires_at > ${databaseNow} + ) + ) + ) OR + ( + NEW.state = 'unknown' AND ( + NEW.outcome IS NOT NULL OR NEW.result_json IS NOT NULL OR NEW.result_hash IS NOT NULL OR + EXISTS ( + SELECT 1 + FROM session_activity_permission_owner_lease owner + WHERE owner.owner_id = OLD.owner_id + AND owner.lease_expires_at > ${databaseNow} + ) OR + NOT EXISTS ( + SELECT 1 + FROM session_activity_permission_owner_lease owner + WHERE owner.owner_id != OLD.owner_id + AND owner.lease_expires_at > ${databaseNow} + ) + ) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal permission effect dispatch transition'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_permission_effect_dispatch_delete_guard + BEFORE DELETE ON session_activity_permission_effect_dispatch + WHEN EXISTS (SELECT 1 FROM session WHERE id = OLD.session_id) + BEGIN + SELECT RAISE(ABORT, 'permission effect dispatch is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_activity_objective_permission_effect_terminal_guard + BEFORE UPDATE OF state ON session_activity_objective + WHEN NEW.state IN ('completed', 'interrupted', 'recovery_required') AND EXISTS ( + SELECT 1 + FROM session_activity_permission_effect_dispatch dispatch + WHERE dispatch.activity_kind = OLD.activity_kind + AND dispatch.activity_id = OLD.activity_id + AND ( + dispatch.state = 'started' OR + (NEW.state IN ('completed', 'interrupted') AND dispatch.state = 'unknown') + ) + ) + BEGIN + SELECT RAISE(ABORT, 'activity has unresolved permission effects'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_legacy_activity_permission_effect_terminal_guard + BEFORE UPDATE OF state ON session_legacy_activity + WHEN NEW.state IN ('settled', 'failed', 'interrupted', 'recovery_required') AND EXISTS ( + SELECT 1 + FROM session_activity_permission_effect_dispatch dispatch + WHERE dispatch.activity_kind = 'legacy' + AND dispatch.activity_id = OLD.activity_id + AND ( + dispatch.state = 'started' OR + (NEW.state IN ('settled', 'interrupted') AND dispatch.state = 'unknown') + ) + ) + BEGIN + SELECT RAISE(ABORT, 'activity has unresolved permission effects'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_activity_permission_effect_terminal_guard + BEFORE UPDATE OF state ON session_activity + WHEN NEW.state IN ('settled', 'failed', 'interrupted') AND EXISTS ( + SELECT 1 + FROM session_activity_permission_effect_dispatch dispatch + WHERE dispatch.activity_kind = 'v2' + AND dispatch.activity_id = OLD.activity_id + AND ( + dispatch.state = 'started' OR + (NEW.state IN ('settled', 'interrupted') AND dispatch.state = 'unknown') + ) + ) + BEGIN + SELECT RAISE(ABORT, 'activity has unresolved permission effects'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812220000_task_structured_output_evidence_authority.ts b/packages/core/src/database/migration/20260812220000_task_structured_output_evidence_authority.ts new file mode 100644 index 000000000..d266866a2 --- /dev/null +++ b/packages/core/src/database/migration/20260812220000_task_structured_output_evidence_authority.ts @@ -0,0 +1,337 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +const databaseNow = "CAST(strftime('%s', 'now') AS INTEGER) * 1000" + +export default { + id: "20260812220000_task_structured_output_evidence_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TEMP TABLE task_structured_output_evidence_upgrade_preflight ( + valid INTEGER NOT NULL CHECK(valid = 1) + ) + `) + yield* tx.run(` + INSERT INTO task_structured_output_evidence_upgrade_preflight(valid) + SELECT CASE + WHEN NOT EXISTS (SELECT 1 FROM task_structured_output_evidence) THEN 1 + ELSE 0 + END + `) + yield* tx.run(`DROP TABLE task_structured_output_evidence_upgrade_preflight`) + yield* tx.run(` + CREATE TRIGGER task_structured_output_evidence_insert_guard + BEFORE INSERT ON task_structured_output_evidence + WHEN NEW.terminal_state NOT IN ('completed', 'failed') OR + NEW.attempts NOT BETWEEN 0 AND 2 OR + json_valid(NEW.contract_json) != 1 OR json_type(NEW.contract_json) != 'object' OR + json_valid(NEW.raw_message_json) != 1 OR json_extract(NEW.raw_message_json, '$.role') != 'assistant' OR + json_valid(NEW.raw_parts_json) != 1 OR json_type(NEW.raw_parts_json) != 'array' OR + NOT EXISTS ( + SELECT 1 FROM task_run + WHERE task_run.run_id = NEW.run_id + AND task_run.child_session_id = NEW.child_session_id + AND task_run.execution_owner = NEW.owner_token + AND task_run.claim_generation = NEW.claim_generation + AND task_run.version = NEW.expected_version + AND task_run.state IN ('provisioning', 'running', 'researching', 'finalizing') + AND task_run.lease_expires_at > ${databaseNow} + AND task_run.execution_spec IS NOT NULL + AND json_valid(task_run.execution_spec) = 1 + AND json_type(task_run.execution_spec, '$.structuredOutput') = 'object' + AND json_extract(task_run.execution_spec, '$.structuredOutput') = NEW.contract_json + ) OR + NOT EXISTS ( + SELECT 1 FROM message + WHERE message.id = NEW.raw_result_message_id + AND message.session_id = NEW.child_session_id + AND message.data = NEW.raw_message_json + ) OR ( + NEW.terminal_state = 'completed' AND ( + NEW.attempts NOT BETWEEN 1 AND 2 OR + NEW.output IS NULL OR + NEW.structured_output_receipt IS NULL OR json_valid(NEW.structured_output_receipt) != 1 OR + json_extract(NEW.structured_output_receipt, '$.attempt') != NEW.attempts OR + json_extract(NEW.structured_output_receipt, '$.transport') NOT IN ('structured', 'text_fallback', 'degraded_text') OR + NEW.failure_code IS NOT NULL OR ( + json_extract(NEW.structured_output_receipt, '$.transport') IN ('structured', 'text_fallback') AND ( + NEW.result_message_id IS NULL OR NEW.result_message_json IS NULL OR + NEW.result_parts_json IS NULL OR + json_valid(NEW.result_message_json) != 1 OR + json_extract(NEW.result_message_json, '$.role') != 'assistant' OR + json_valid(NEW.result_parts_json) != 1 OR json_type(NEW.result_parts_json) != 'array' OR + NOT EXISTS ( + SELECT 1 FROM message + WHERE message.id = NEW.result_message_id + AND message.session_id = NEW.child_session_id + AND message.data = NEW.result_message_json + ) + ) + ) OR ( + json_extract(NEW.structured_output_receipt, '$.transport') = 'degraded_text' AND ( + NEW.result_message_id IS NOT NULL OR NEW.result_message_json IS NOT NULL OR + NEW.result_parts_json IS NOT NULL OR + json_extract(NEW.structured_output_receipt, '$.reason') NOT IN ( + 'structured_output_missing', 'structured_output_invalid' + ) + ) + ) + ) + ) OR ( + NEW.terminal_state = 'failed' AND ( + NEW.output IS NOT NULL OR + NEW.structured_output_receipt IS NOT NULL OR + NEW.result_message_id IS NOT NULL OR NEW.result_message_json IS NOT NULL OR + NEW.result_parts_json IS NOT NULL OR + NEW.failure_code NOT LIKE 'structured_finalizer_%' + ) + ) + BEGIN + SELECT RAISE(ABORT, 'invalid task structured output evidence'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_evidence_part_insert_guard + BEFORE INSERT ON task_structured_output_evidence_part + WHEN NEW.role NOT IN ('raw', 'result') OR NEW.ordinal < 0 OR + json_valid(NEW.part_json) != 1 OR + NOT EXISTS ( + SELECT 1 + FROM task_structured_output_evidence evidence + JOIN part ON part.id = NEW.part_id + WHERE evidence.run_id = NEW.run_id + AND part.message_id = NEW.message_id + AND part.session_id = NEW.session_id + AND part.data = NEW.part_json + AND NEW.session_id = evidence.child_session_id + AND NEW.message_id = CASE NEW.role + WHEN 'raw' THEN evidence.raw_result_message_id + ELSE evidence.result_message_id + END + AND NEW.ordinal = ( + SELECT count(*) FROM part prior + WHERE prior.message_id = NEW.message_id + AND prior.session_id = NEW.session_id + AND prior.id < NEW.part_id + ) + ) + BEGIN + SELECT RAISE(ABORT, 'invalid task structured output evidence part'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_run_structured_output_evidence_guard + BEFORE UPDATE ON task_run + WHEN OLD.phase != 'settled' AND NEW.phase = 'settled' AND + OLD.execution_spec IS NOT NULL AND json_valid(OLD.execution_spec) = 1 AND + json_type(OLD.execution_spec, '$.structuredOutput') = 'object' AND ( + NEW.state = 'completed' OR + (NEW.state IN ('failed', 'error') AND json_extract(NEW.error, '$.code') LIKE 'structured_finalizer_%') + ) AND NOT EXISTS ( + SELECT 1 FROM task_structured_output_evidence evidence + WHERE evidence.run_id = NEW.run_id + AND evidence.child_session_id = NEW.child_session_id + AND evidence.owner_token = OLD.execution_owner + AND evidence.claim_generation = OLD.claim_generation + AND evidence.expected_version <= OLD.version + AND NEW.version = OLD.version + 1 + AND evidence.terminal_state = CASE NEW.state WHEN 'completed' THEN 'completed' ELSE 'failed' END + AND evidence.attempts = NEW.attempts + AND evidence.contract_json = json_extract(OLD.execution_spec, '$.structuredOutput') + AND evidence.raw_result_message_id = NEW.raw_result_message_id + AND evidence.result_message_id IS NEW.structured_result_message_id + AND evidence.output IS NEW.output + AND evidence.structured_output_receipt IS NEW.structured_output_receipt + AND evidence.failure_code IS json_extract(NEW.error, '$.code') + AND json(evidence.raw_parts_json) = ( + SELECT COALESCE(json_group_array(json(snapshot)), json('[]')) + FROM ( + SELECT json_object( + 'id', part.id, + 'messageID', part.message_id, + 'sessionID', part.session_id, + 'data', json(part.data) + ) AS snapshot + FROM part + WHERE part.message_id = evidence.raw_result_message_id + AND part.session_id = evidence.child_session_id + ORDER BY part.id + ) + ) + AND ( + evidence.result_message_id IS NULL OR json(evidence.result_parts_json) = ( + SELECT COALESCE(json_group_array(json(snapshot)), json('[]')) + FROM ( + SELECT json_object( + 'id', part.id, + 'messageID', part.message_id, + 'sessionID', part.session_id, + 'data', json(part.data) + ) AS snapshot + FROM part + WHERE part.message_id = evidence.result_message_id + AND part.session_id = evidence.child_session_id + ORDER BY part.id + ) + ) + ) + AND EXISTS ( + SELECT 1 FROM message + WHERE message.id = evidence.raw_result_message_id + AND message.session_id = evidence.child_session_id + AND message.data = evidence.raw_message_json + ) + AND ( + evidence.result_message_id IS NULL OR EXISTS ( + SELECT 1 FROM message + WHERE message.id = evidence.result_message_id + AND message.session_id = evidence.child_session_id + AND message.data = evidence.result_message_json + ) + ) + AND ( + SELECT count(*) FROM part + WHERE part.message_id = evidence.raw_result_message_id + AND part.session_id = evidence.child_session_id + ) = ( + SELECT count(*) FROM task_structured_output_evidence_part sealed + WHERE sealed.run_id = evidence.run_id AND sealed.role = 'raw' + ) + AND NOT EXISTS ( + SELECT 1 FROM part + WHERE part.message_id = evidence.raw_result_message_id + AND part.session_id = evidence.child_session_id + AND NOT EXISTS ( + SELECT 1 FROM task_structured_output_evidence_part sealed + WHERE sealed.run_id = evidence.run_id AND sealed.role = 'raw' + AND sealed.part_id = part.id AND sealed.message_id = part.message_id + AND sealed.session_id = part.session_id AND sealed.part_json = part.data + ) + ) + AND ( + evidence.result_message_id IS NULL OR ( + (SELECT count(*) FROM part + WHERE part.message_id = evidence.result_message_id + AND part.session_id = evidence.child_session_id) = + (SELECT count(*) FROM task_structured_output_evidence_part sealed + WHERE sealed.run_id = evidence.run_id AND sealed.role = 'result') + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM part + WHERE part.message_id = evidence.result_message_id + AND part.session_id = evidence.child_session_id + AND NOT EXISTS ( + SELECT 1 FROM task_structured_output_evidence_part sealed + WHERE sealed.run_id = evidence.run_id AND sealed.role = 'result' + AND sealed.part_id = part.id AND sealed.message_id = part.message_id + AND sealed.session_id = part.session_id AND sealed.part_json = part.data + ) + ) + ) + BEGIN + SELECT RAISE(ABORT, 'task structured terminal is missing exact material evidence'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_run_structured_terminal_insert_requires_history + BEFORE INSERT ON task_run + WHEN NEW.phase = 'settled' AND NEW.execution_spec IS NOT NULL AND + json_valid(NEW.execution_spec) = 1 AND + json_type(NEW.execution_spec, '$.structuredOutput') = 'object' + BEGIN + SELECT RAISE(ABORT, 'new structured task terminals require transition evidence'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_evidence_immutable + BEFORE UPDATE ON task_structured_output_evidence + BEGIN + SELECT RAISE(ABORT, 'task structured output evidence is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_evidence_delete_guard + BEFORE DELETE ON task_structured_output_evidence + WHEN EXISTS (SELECT 1 FROM task_run WHERE task_run.run_id = OLD.run_id) + BEGIN + SELECT RAISE(ABORT, 'task structured output evidence cannot be deleted'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_evidence_part_immutable + BEFORE UPDATE ON task_structured_output_evidence_part + BEGIN + SELECT RAISE(ABORT, 'task structured output evidence part is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_evidence_part_delete_guard + BEFORE DELETE ON task_structured_output_evidence_part + WHEN EXISTS ( + SELECT 1 FROM task_structured_output_evidence + WHERE task_structured_output_evidence.run_id = OLD.run_id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured output evidence part cannot be deleted'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_message_update_guard + BEFORE UPDATE ON message + WHEN NEW.data IS NOT OLD.data AND EXISTS ( + SELECT 1 FROM task_structured_output_evidence + WHERE raw_result_message_id = OLD.id OR result_message_id = OLD.id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured output message material is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_message_delete_guard + BEFORE DELETE ON message + WHEN EXISTS ( + SELECT 1 FROM task_structured_output_evidence + WHERE raw_result_message_id = OLD.id OR result_message_id = OLD.id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured output message material cannot be deleted'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_part_insert_guard + BEFORE INSERT ON part + WHEN EXISTS ( + SELECT 1 FROM task_structured_output_evidence + WHERE raw_result_message_id = NEW.message_id OR result_message_id = NEW.message_id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured output message cannot gain parts'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_part_update_guard + BEFORE UPDATE ON part + WHEN EXISTS ( + SELECT 1 FROM task_structured_output_evidence + WHERE raw_result_message_id = OLD.message_id OR result_message_id = OLD.message_id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured output part material is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_output_part_delete_guard + BEFORE DELETE ON part + WHEN EXISTS ( + SELECT 1 FROM task_structured_output_evidence + WHERE raw_result_message_id = OLD.message_id OR result_message_id = OLD.message_id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured output part material cannot be deleted'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812223000_task_structured_finalizer_response.ts b/packages/core/src/database/migration/20260812223000_task_structured_finalizer_response.ts new file mode 100644 index 000000000..05e7ea1eb --- /dev/null +++ b/packages/core/src/database/migration/20260812223000_task_structured_finalizer_response.ts @@ -0,0 +1,118 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812223000_task_structured_finalizer_response", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TRIGGER task_structured_finalizer_response_insert_guard + BEFORE INSERT ON task_structured_finalizer_response + WHEN NEW.attempt NOT BETWEEN 1 AND 2 OR + json_valid(NEW.response_message_json) != 1 OR + json_extract(NEW.response_message_json, '$.role') != 'assistant' OR + json_extract(NEW.response_message_json, '$.error') IS NOT NULL OR + NOT EXISTS ( + SELECT 1 + FROM task_run run + JOIN message request + ON request.id = NEW.request_message_id + AND request.session_id = NEW.child_session_id + JOIN message response + ON response.id = NEW.response_message_id + AND response.session_id = NEW.child_session_id + WHERE run.run_id = NEW.run_id + AND run.state = 'finalizing' + AND run.control_state = 'open' + AND run.interrupt_requested_at IS NULL + AND run.child_session_id = NEW.child_session_id + AND run.execution_owner = NEW.owner_token + AND run.claim_generation = NEW.claim_generation + AND run.version = NEW.expected_version + AND run.attempts = NEW.attempt + AND run.raw_result_message_id = NEW.source_message_id + AND response.data = NEW.response_message_json + AND json_extract(response.data, '$.parentID') = request.id + AND json_valid(request.data) = 1 + AND json_extract(request.data, '$.role') = 'user' + AND json_extract(request.data, '$.metadata.deepagent.structured_finalizer.run_id') = NEW.run_id + AND json_extract(request.data, '$.metadata.deepagent.structured_finalizer.attempt') = NEW.attempt + AND json_extract(request.data, '$.metadata.deepagent.structured_finalizer.source_message_id') = NEW.source_message_id + ) + BEGIN + SELECT RAISE(ABORT, 'invalid task structured finalizer response'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_finalizer_response_immutable + BEFORE UPDATE ON task_structured_finalizer_response + BEGIN + SELECT RAISE(ABORT, 'task structured finalizer response is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_finalizer_response_delete_guard + BEFORE DELETE ON task_structured_finalizer_response + WHEN EXISTS (SELECT 1 FROM task_run WHERE task_run.run_id = OLD.run_id) + BEGIN + SELECT RAISE(ABORT, 'task structured finalizer response cannot be deleted'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_finalizer_response_message_update_guard + BEFORE UPDATE ON message + WHEN EXISTS ( + SELECT 1 FROM task_structured_finalizer_response + WHERE request_message_id = OLD.id OR response_message_id = OLD.id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured finalizer response messages are immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_finalizer_response_message_delete_guard + BEFORE DELETE ON message + WHEN EXISTS ( + SELECT 1 FROM task_structured_finalizer_response + WHERE request_message_id = OLD.id OR response_message_id = OLD.id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured finalizer response messages cannot be deleted'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_finalizer_response_part_insert_guard + BEFORE INSERT ON part + WHEN EXISTS ( + SELECT 1 FROM task_structured_finalizer_response + WHERE response_message_id = NEW.message_id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured finalizer response cannot gain parts'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_finalizer_response_part_update_guard + BEFORE UPDATE ON part + WHEN EXISTS ( + SELECT 1 FROM task_structured_finalizer_response + WHERE response_message_id = OLD.message_id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured finalizer response parts are immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER task_structured_finalizer_response_part_delete_guard + BEFORE DELETE ON part + WHEN EXISTS ( + SELECT 1 FROM task_structured_finalizer_response + WHERE response_message_id = OLD.message_id + ) + BEGIN + SELECT RAISE(ABORT, 'task structured finalizer response parts cannot be deleted'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260812224000_learning_artifact_phase_receipt.ts b/packages/core/src/database/migration/20260812224000_learning_artifact_phase_receipt.ts new file mode 100644 index 000000000..3577b8f25 --- /dev/null +++ b/packages/core/src/database/migration/20260812224000_learning_artifact_phase_receipt.ts @@ -0,0 +1,68 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260812224000_learning_artifact_phase_receipt", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + UPDATE learning_job + SET state = 'recovery_required', owner = NULL, lease_expires_at = NULL, + version = version + 1, side_effect_state = 'unknown', + side_effect_kind = CASE + WHEN state = 'running' THEN 'extraction' + WHEN state = 'reviewing' THEN 'reviewer' + ELSE 'governance' + END, + expected_result_ref = NULL, error_code = 'legacy_artifact_plan_missing', + error_detail = 'The side effect started before exact artifact planning was durable; automatic replay is forbidden.', + settled_at = updated_at + WHERE (state = 'running' AND side_effect_state <> 'not_started') + OR state IN ('reviewing', 'governance') + `) + yield* tx.run(` + CREATE TRIGGER learning_job_expected_result_insert_guard + BEFORE INSERT ON learning_job + WHEN NOT ( + (NEW.side_effect_state = 'not_started' AND NEW.expected_result_ref IS NULL) OR + (NEW.side_effect_kind IN ('extraction', 'reviewer') AND + NEW.side_effect_state IN ('started', 'settled') AND length(trim(NEW.expected_result_ref)) > 0) OR + (NEW.side_effect_kind = 'governance' AND NEW.expected_result_ref IS NULL) OR + (NEW.side_effect_state = 'unknown' AND NEW.expected_result_ref IS NULL) + ) + BEGIN SELECT RAISE(ABORT, 'learning_job_expected_result_invalid'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_expected_result_update_guard + BEFORE UPDATE ON learning_job + WHEN NOT ( + (NEW.side_effect_state = 'not_started' AND NEW.expected_result_ref IS NULL) OR + (NEW.side_effect_kind IN ('extraction', 'reviewer') AND + NEW.side_effect_state IN ('started', 'settled') AND length(trim(NEW.expected_result_ref)) > 0) OR + (NEW.side_effect_kind = 'governance' AND NEW.expected_result_ref IS NULL) OR + (NEW.side_effect_state = 'unknown' AND NEW.expected_result_ref IS NULL) + ) + BEGIN SELECT RAISE(ABORT, 'learning_job_expected_result_invalid'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_expected_result_settlement_guard + BEFORE UPDATE ON learning_job + WHEN NEW.side_effect_state = 'settled' AND NEW.side_effect_kind IN ('extraction', 'reviewer') + AND NEW.result_ref IS NOT NEW.expected_result_ref + BEGIN SELECT RAISE(ABORT, 'learning_job_expected_result_mismatch'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_expected_result_immutable + BEFORE UPDATE ON learning_job + WHEN OLD.side_effect_state IN ('started', 'settled') AND + OLD.side_effect_kind IN ('extraction', 'reviewer') AND + NOT ( + NEW.expected_result_ref IS OLD.expected_result_ref OR + (OLD.side_effect_state = 'settled' AND NEW.side_effect_state = 'not_started' AND + NEW.expected_result_ref IS NULL) + ) + BEGIN SELECT RAISE(ABORT, 'learning_job_expected_result_immutable'); END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813040301_bug407_final_authorities.ts b/packages/core/src/database/migration/20260813040301_bug407_final_authorities.ts new file mode 100644 index 000000000..41f253275 --- /dev/null +++ b/packages/core/src/database/migration/20260813040301_bug407_final_authorities.ts @@ -0,0 +1,260 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813040301_bug407_final_authorities", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`learning_governance_compensation\` ( + \`compensation_id\` text PRIMARY KEY, + \`plan_id\` text NOT NULL, + \`action_id\` text NOT NULL, + \`sequence\` integer NOT NULL, + \`kind\` text NOT NULL, + \`source_payload_fingerprint\` text NOT NULL, + \`state\` text NOT NULL, + \`owner\` text, + \`lease_expires_at\` integer, + \`version\` integer NOT NULL, + \`result_ref\` text, + \`result_hash\` text, + \`result_fingerprint\` text, + \`error_code\` text, + \`error_detail\` text, + \`created_at\` integer NOT NULL, + \`settled_at\` integer, + \`updated_at\` integer NOT NULL, + CONSTRAINT \`fk_learning_governance_compensation_plan_id_learning_governance_plan_plan_id_fk\` FOREIGN KEY (\`plan_id\`) REFERENCES \`learning_governance_plan\`(\`plan_id\`), + CONSTRAINT \`fk_learning_governance_compensation_action_id_learning_governance_action_action_id_fk\` FOREIGN KEY (\`action_id\`) REFERENCES \`learning_governance_action\`(\`action_id\`), + CONSTRAINT "learning_governance_compensation_sequence_check" CHECK("sequence" >= 0), + CONSTRAINT "learning_governance_compensation_version_check" CHECK("version" >= 0), + CONSTRAINT "learning_governance_compensation_kind_check" CHECK("kind" IN ('document_quarantine', 'memory_inbox_revoke')), + CONSTRAINT "learning_governance_compensation_source_fingerprint_check" CHECK(length("source_payload_fingerprint") = 64 AND "source_payload_fingerprint" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_governance_compensation_result_hash_check" CHECK("result_hash" IS NULL OR (length("result_hash") = 64 AND "result_hash" NOT GLOB '*[^0-9a-f]*')), + CONSTRAINT "learning_governance_compensation_result_fingerprint_check" CHECK("result_fingerprint" IS NULL OR (length("result_fingerprint") = 64 AND "result_fingerprint" NOT GLOB '*[^0-9a-f]*')), + CONSTRAINT "learning_governance_compensation_state_check" CHECK("state" IN ('prepared', 'running', 'settled', 'recovery_required')), + CONSTRAINT "learning_governance_compensation_lifecycle_check" CHECK(("state" = 'prepared' AND "owner" IS NULL AND "lease_expires_at" IS NULL AND "result_ref" IS NULL AND "result_hash" IS NULL AND "result_fingerprint" IS NULL AND "error_code" IS NULL AND "settled_at" IS NULL) OR ("state" = 'running' AND length(trim("owner")) > 0 AND "lease_expires_at" IS NOT NULL AND "result_ref" IS NULL AND "result_hash" IS NULL AND "result_fingerprint" IS NULL AND "error_code" IS NULL AND "settled_at" IS NULL) OR ("state" = 'settled' AND "owner" IS NULL AND "lease_expires_at" IS NULL AND length(trim("result_ref")) > 0 AND "result_hash" IS NOT NULL AND "result_fingerprint" IS NOT NULL AND "error_code" IS NULL AND "settled_at" IS NOT NULL) OR ("state" = 'recovery_required' AND "owner" IS NULL AND "lease_expires_at" IS NULL AND length(trim("error_code")) > 0 AND "settled_at" IS NOT NULL)) + ); + `) + yield* tx.run(` + CREATE TABLE \`learning_lifecycle_trigger_receipt\` ( + \`receipt_id\` text PRIMARY KEY, + \`trigger\` text NOT NULL, + \`boundary_key\` text NOT NULL, + \`session_id\` text NOT NULL, + \`run_id\` text NOT NULL, + \`source_admission_hash\` text NOT NULL, + \`source_terminal_hash\` text NOT NULL, + \`artifact_path\` text NOT NULL, + \`artifact_hash\` text NOT NULL, + \`artifact_json\` text NOT NULL, + \`admission_fingerprint\` text NOT NULL, + \`admission_json\` text NOT NULL, + \`state\` text NOT NULL, + \`error_detail\` text, + \`created_at\` integer NOT NULL, + \`settled_at\` integer, + \`updated_at\` integer NOT NULL, + CONSTRAINT "learning_lifecycle_trigger_kind_check" CHECK("trigger" IN ('idle', 'pause', 'project_switch')), + CONSTRAINT "learning_lifecycle_trigger_state_check" CHECK("state" IN ('prepared', 'admitted')), + CONSTRAINT "learning_lifecycle_trigger_source_admission_hash_check" CHECK(length("source_admission_hash") = 64 AND "source_admission_hash" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_lifecycle_trigger_source_terminal_hash_check" CHECK(length("source_terminal_hash") = 64 AND "source_terminal_hash" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_lifecycle_trigger_artifact_hash_check" CHECK(length("artifact_hash") = 64 AND "artifact_hash" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_lifecycle_trigger_admission_fingerprint_check" CHECK(length("admission_fingerprint") = 64 AND "admission_fingerprint" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_lifecycle_trigger_admission_json_check" CHECK(json_valid("admission_json") AND json_type("admission_json") = 'object'), + CONSTRAINT "learning_lifecycle_trigger_artifact_json_check" CHECK(json_valid("artifact_json") AND json_type("artifact_json") = 'object'), + CONSTRAINT "learning_lifecycle_trigger_settlement_check" CHECK(("state" = 'prepared' AND "settled_at" IS NULL) OR ("state" = 'admitted' AND "settled_at" IS NOT NULL AND "error_detail" IS NULL)) + ); + `) + yield* tx.run(` + CREATE TABLE \`learning_reviewer_attempt\` ( + \`attempt_id\` text PRIMARY KEY, + \`job_id\` text NOT NULL, + \`state\` text NOT NULL, + \`version\` integer NOT NULL, + \`owner\` text, + \`review_session_id\` text NOT NULL, + \`request_ref\` text NOT NULL, + \`request_hash\` text NOT NULL, + \`source_candidate_ids_json\` text NOT NULL, + \`source_candidate_set_hash\` text NOT NULL, + \`provider_id\` text NOT NULL, + \`model_id\` text NOT NULL, + \`policy_hash\` text NOT NULL, + \`response_ref\` text, + \`response_hash\` text, + \`verdict\` text, + \`selected_candidate_ids_json\` text, + \`selected_subset_hash\` text, + \`error_code\` text, + \`error_detail\` text, + \`created_at\` integer NOT NULL, + \`dispatched_at\` integer, + \`settled_at\` integer, + \`updated_at\` integer NOT NULL, + CONSTRAINT \`fk_learning_reviewer_attempt_job_id_learning_job_job_id_fk\` FOREIGN KEY (\`job_id\`) REFERENCES \`learning_job\`(\`job_id\`), + CONSTRAINT "learning_reviewer_attempt_state_check" CHECK("state" IN ('prepared', 'dispatching', 'settled', 'failed', 'recovery_required')), + CONSTRAINT "learning_reviewer_attempt_version_check" CHECK("version" >= 0), + CONSTRAINT "learning_reviewer_attempt_hash_check" CHECK(length("request_hash") = 64 AND "request_hash" NOT GLOB '*[^0-9a-f]*' AND length("source_candidate_set_hash") = 64 AND "source_candidate_set_hash" NOT GLOB '*[^0-9a-f]*' AND length("policy_hash") = 64 AND "policy_hash" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_reviewer_attempt_terminal_check" CHECK(("state" IN ('prepared', 'dispatching') AND "settled_at" IS NULL) OR ("state" IN ('settled', 'failed', 'recovery_required') AND "settled_at" IS NOT NULL)), + CONSTRAINT "learning_reviewer_attempt_response_check" CHECK(("state" <> 'settled' AND "response_ref" IS NULL AND "response_hash" IS NULL AND "verdict" IS NULL AND "selected_candidate_ids_json" IS NULL AND "selected_subset_hash" IS NULL) OR ("state" = 'settled' AND length(trim("response_ref")) > 0 AND length("response_hash") = 64 AND "response_hash" NOT GLOB '*[^0-9a-f]*' AND "verdict" IN ('approve', 'reject', 'manual_review') AND length(trim("selected_candidate_ids_json")) > 0 AND length("selected_subset_hash") = 64 AND "selected_subset_hash" NOT GLOB '*[^0-9a-f]*')), + CONSTRAINT "learning_reviewer_attempt_error_check" CHECK("state" NOT IN ('failed', 'recovery_required') OR length(trim("error_code")) > 0) + ); + `) + yield* tx.run(` + CREATE TABLE \`session_v2_provider_parity_receipt\` ( + \`campaign_id\` text NOT NULL, + \`case_name\` text NOT NULL, + \`legacy_receipt_id\` text NOT NULL, + \`core_v2_receipt_id\` text NOT NULL, + \`legacy_request_hash\` text NOT NULL, + \`core_v2_request_hash\` text NOT NULL, + \`legacy_outcome_hash\` text NOT NULL, + \`core_v2_outcome_hash\` text NOT NULL, + \`legacy_prepared_turn\` text NOT NULL, + \`core_v2_prepared_turn\` text NOT NULL, + \`diff_artifact\` text NOT NULL, + \`allowlist_version\` text NOT NULL, + \`allowlisted_differences\` text NOT NULL, + \`disallowed_differences\` text NOT NULL, + \`evidence\` text NOT NULL, + \`verified\` integer NOT NULL, + \`receipt_hash\` text NOT NULL, + \`created_at\` integer NOT NULL, + CONSTRAINT \`session_v2_provider_parity_receipt_pk\` PRIMARY KEY(\`campaign_id\`, \`case_name\`), + CONSTRAINT \`fk_session_v2_provider_parity_receipt_core_v2_receipt_id_session_v2_provider_turn_receipt_receipt_id_fk\` FOREIGN KEY (\`core_v2_receipt_id\`) REFERENCES \`session_v2_provider_turn_receipt\`(\`receipt_id\`) + ); + `) + yield* tx.run(` + CREATE TABLE \`session_v2_provider_turn_receipt\` ( + \`receipt_id\` text PRIMARY KEY, + \`session_id\` text NOT NULL, + \`request_ordinal\` integer NOT NULL, + \`user_message_id\` text NOT NULL, + \`history_prompt_epoch\` integer NOT NULL, + \`history_source_end_message_id\` text, + \`request_input_hash\` text NOT NULL, + \`provider_id\` text NOT NULL, + \`model_id\` text NOT NULL, + \`protocol\` text NOT NULL, + \`owner_mode\` text NOT NULL, + \`owner_token\` text NOT NULL, + \`state\` text NOT NULL, + \`prepared_turn_hash\` text, + \`wire_request_hash\` text, + \`prepared_turn\` text, + \`outcome_hash\` text, + \`error_code\` text, + \`created_at\` integer NOT NULL, + \`dispatching_at\` integer, + \`first_event_at\` integer, + \`terminal_at\` integer, + CONSTRAINT \`fk_session_v2_provider_turn_receipt_session_id_session_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session\`(\`id\`) ON DELETE CASCADE, + CONSTRAINT \`fk_session_v2_provider_turn_receipt_owner_token_session_provider_owner_lease_owner_token_fk\` FOREIGN KEY (\`owner_token\`) REFERENCES \`session_provider_owner_lease\`(\`owner_token\`) + ); + `) + /* Drizzle proposed rebuilding learning_job only to restate checks already installed by + 20260812224000. Rebuilding here would drop its cross-table authority triggers. */ + yield* Effect.void + /* + CREATE TABLE \`__new_learning_job\` ( + \`job_id\` text PRIMARY KEY, + \`project_id\` text NOT NULL, + \`session_id\` text, + \`run_id\` text, + \`trigger\` text NOT NULL, + \`dedupe_key\` text NOT NULL, + \`candidate_input_ref\` text NOT NULL, + \`policy\` text NOT NULL, + \`max_attempts\` integer NOT NULL, + \`admission_fingerprint\` text NOT NULL, + \`state\` text NOT NULL, + \`attempts\` integer NOT NULL, + \`owner\` text, + \`lease_expires_at\` integer, + \`version\` integer NOT NULL, + \`side_effect_state\` text NOT NULL, + \`side_effect_kind\` text, + \`expected_result_ref\` text, + \`review_job_id\` text, + \`result_ref\` text, + \`error_code\` text, + \`error_detail\` text, + \`settlement_fingerprint\` text, + \`next_attempt_at\` integer NOT NULL, + \`created_at\` integer NOT NULL, + \`started_at\` integer, + \`settled_at\` integer, + \`updated_at\` integer NOT NULL, + CONSTRAINT \`fk_learning_job_project_id_project_id_fk\` FOREIGN KEY (\`project_id\`) REFERENCES \`project\`(\`id\`), + CONSTRAINT \`fk_learning_job_session_id_session_id_fk\` FOREIGN KEY (\`session_id\`) REFERENCES \`session\`(\`id\`), + CONSTRAINT "learning_job_trigger_check" CHECK("trigger" IN ('idle', 'pause', 'project_switch', 'session_finalization')), + CONSTRAINT "learning_job_policy_check" CHECK("policy" IN ('auto_merge_safe_project', 'manual_review')), + CONSTRAINT "learning_job_state_check" CHECK("state" IN ('queued', 'running', 'reviewing', 'governance', 'completed', 'failed', 'cancelled', 'recovery_required')), + CONSTRAINT "learning_job_attempts_check" CHECK("attempts" >= 0), + CONSTRAINT "learning_job_max_attempts_check" CHECK("max_attempts" > 0), + CONSTRAINT "learning_job_version_check" CHECK("version" >= 0), + CONSTRAINT "learning_job_admission_fingerprint_check" CHECK(length("admission_fingerprint") = 64 AND "admission_fingerprint" NOT GLOB '*[^0-9a-f]*'), + CONSTRAINT "learning_job_side_effect_check" CHECK(("side_effect_state" = 'not_started' AND "side_effect_kind" IS NULL) OR ("side_effect_state" IN ('started', 'settled', 'unknown') AND "side_effect_kind" IS NOT NULL)), + CONSTRAINT "learning_job_expected_result_check" CHECK(("side_effect_state" = 'not_started' AND "expected_result_ref" IS NULL) OR ("side_effect_kind" IN ('extraction', 'reviewer') AND "side_effect_state" IN ('started', 'settled') AND length(trim("expected_result_ref")) > 0) OR ("side_effect_kind" = 'governance' AND "expected_result_ref" IS NULL) OR ("side_effect_state" = 'unknown' AND "expected_result_ref" IS NULL)), + CONSTRAINT "learning_job_settled_result_check" CHECK("side_effect_state" <> 'settled' OR (length(trim("result_ref")) > 0 AND ("expected_result_ref" IS NULL OR "side_effect_kind" = 'reviewer' OR "result_ref" = "expected_result_ref"))), + CONSTRAINT "learning_job_active_phase_kind_check" CHECK("state" NOT IN ('running', 'reviewing', 'governance') OR "side_effect_state" = 'not_started' OR ("state" = 'running' AND "side_effect_kind" = 'extraction') OR ("state" = 'reviewing' AND "side_effect_kind" = 'reviewer') OR ("state" = 'governance' AND "side_effect_kind" = 'governance')), + CONSTRAINT "learning_job_ownership_check" CHECK(("state" = 'queued' AND "owner" IS NULL AND "lease_expires_at" IS NULL AND "side_effect_state" = 'not_started') OR ("state" IN ('running', 'reviewing', 'governance') AND "started_at" IS NOT NULL AND "settled_at" IS NULL AND ((length(trim("owner")) > 0 AND "lease_expires_at" IS NOT NULL) OR ("state" IN ('reviewing', 'governance') AND "owner" IS NULL AND "lease_expires_at" IS NULL AND "side_effect_state" = 'not_started'))) OR ("state" IN ('completed', 'failed', 'cancelled', 'recovery_required') AND "owner" IS NULL AND "lease_expires_at" IS NULL AND "settled_at" IS NOT NULL)), + CONSTRAINT "learning_job_recovery_check" CHECK("state" <> 'recovery_required' OR ("side_effect_state" <> 'not_started' AND "error_code" IS NOT NULL)), + CONSTRAINT "learning_job_completed_check" CHECK("state" <> 'completed' OR ("side_effect_state" = 'settled' AND length(trim("result_ref")) > 0)), + CONSTRAINT "learning_job_failed_check" CHECK("state" <> 'failed' OR length(trim("error_code")) > 0), + CONSTRAINT "learning_job_terminal_side_effect_check" CHECK("state" NOT IN ('completed', 'failed', 'cancelled') OR "side_effect_state" IN ('not_started', 'settled')), + CONSTRAINT "learning_job_settlement_fingerprint_check" CHECK("settlement_fingerprint" IS NULL OR (length("settlement_fingerprint") = 64 AND "settlement_fingerprint" NOT GLOB '*[^0-9a-f]*')) + ); + `) + yield* tx.run(`INSERT INTO \`__new_learning_job\`(\`job_id\`, \`project_id\`, \`session_id\`, \`run_id\`, \`trigger\`, \`dedupe_key\`, \`candidate_input_ref\`, \`policy\`, \`max_attempts\`, \`admission_fingerprint\`, \`state\`, \`attempts\`, \`owner\`, \`lease_expires_at\`, \`version\`, \`side_effect_state\`, \`side_effect_kind\`, \`expected_result_ref\`, \`review_job_id\`, \`result_ref\`, \`error_code\`, \`error_detail\`, \`settlement_fingerprint\`, \`next_attempt_at\`, \`created_at\`, \`started_at\`, \`settled_at\`, \`updated_at\`) SELECT \`job_id\`, \`project_id\`, \`session_id\`, \`run_id\`, \`trigger\`, \`dedupe_key\`, \`candidate_input_ref\`, \`policy\`, \`max_attempts\`, \`admission_fingerprint\`, \`state\`, \`attempts\`, \`owner\`, \`lease_expires_at\`, \`version\`, \`side_effect_state\`, \`side_effect_kind\`, \`expected_result_ref\`, \`review_job_id\`, \`result_ref\`, \`error_code\`, \`error_detail\`, \`settlement_fingerprint\`, \`next_attempt_at\`, \`created_at\`, \`started_at\`, \`settled_at\`, \`updated_at\` FROM \`learning_job\`;`) + yield* tx.run(`DROP TABLE \`learning_job\`;`) + yield* tx.run(`ALTER TABLE \`__new_learning_job\` RENAME TO \`learning_job\`;`) + yield* tx.run(`PRAGMA foreign_keys=ON;`) + yield* tx.run(`CREATE UNIQUE INDEX \`learning_job_dedupe_idx\` ON \`learning_job\` (\`dedupe_key\`);`) + yield* tx.run(`CREATE INDEX \`learning_job_due_idx\` ON \`learning_job\` (\`state\`,\`next_attempt_at\`,\`created_at\`);`) + yield* tx.run(`CREATE INDEX \`learning_job_project_created_idx\` ON \`learning_job\` (\`project_id\`,\`created_at\`);`) + yield* tx.run(`CREATE INDEX \`learning_job_owner_lease_idx\` ON \`learning_job\` (\`owner\`,\`lease_expires_at\`) WHERE "learning_job"."owner" IS NOT NULL;`) */ + yield* tx.run( + `CREATE UNIQUE INDEX \`learning_governance_compensation_action_idx\` ON \`learning_governance_compensation\` (\`action_id\`);`, + ) + yield* tx.run( + `CREATE UNIQUE INDEX \`learning_governance_compensation_plan_sequence_idx\` ON \`learning_governance_compensation\` (\`plan_id\`,\`sequence\`);`, + ) + yield* tx.run( + `CREATE INDEX \`learning_governance_compensation_claim_idx\` ON \`learning_governance_compensation\` (\`state\`,\`created_at\`);`, + ) + yield* tx.run( + `CREATE UNIQUE INDEX \`learning_lifecycle_trigger_identity_idx\` ON \`learning_lifecycle_trigger_receipt\` (\`trigger\`,\`session_id\`,\`run_id\`);`, + ) + yield* tx.run( + `CREATE INDEX \`learning_lifecycle_trigger_pending_idx\` ON \`learning_lifecycle_trigger_receipt\` (\`created_at\`,\`receipt_id\`) WHERE "learning_lifecycle_trigger_receipt"."state" = 'prepared';`, + ) + yield* tx.run( + `CREATE UNIQUE INDEX \`learning_reviewer_attempt_job_idx\` ON \`learning_reviewer_attempt\` (\`job_id\`);`, + ) + yield* tx.run( + `CREATE UNIQUE INDEX \`learning_reviewer_attempt_session_idx\` ON \`learning_reviewer_attempt\` (\`review_session_id\`);`, + ) + yield* tx.run( + `CREATE INDEX \`learning_reviewer_attempt_state_idx\` ON \`learning_reviewer_attempt\` (\`state\`,\`updated_at\`);`, + ) + yield* tx.run( + `CREATE UNIQUE INDEX \`session_v2_provider_parity_receipt_hash_idx\` ON \`session_v2_provider_parity_receipt\` (\`receipt_hash\`);`, + ) + yield* tx.run( + `CREATE INDEX \`session_v2_provider_parity_receipt_campaign_idx\` ON \`session_v2_provider_parity_receipt\` (\`campaign_id\`,\`verified\`);`, + ) + yield* tx.run( + `CREATE UNIQUE INDEX \`session_v2_provider_turn_receipt_ordinal_idx\` ON \`session_v2_provider_turn_receipt\` (\`session_id\`,\`request_ordinal\`);`, + ) + yield* tx.run( + `CREATE INDEX \`session_v2_provider_turn_receipt_input_idx\` ON \`session_v2_provider_turn_receipt\` (\`session_id\`,\`user_message_id\`,\`history_prompt_epoch\`,\`request_input_hash\`);`, + ) + yield* tx.run( + `CREATE INDEX \`session_v2_provider_turn_receipt_owner_state_idx\` ON \`session_v2_provider_turn_receipt\` (\`owner_token\`,\`state\`,\`created_at\`);`, + ) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813041000_learning_reviewer_attempt_authority.ts b/packages/core/src/database/migration/20260813041000_learning_reviewer_attempt_authority.ts new file mode 100644 index 000000000..016a52021 --- /dev/null +++ b/packages/core/src/database/migration/20260813041000_learning_reviewer_attempt_authority.ts @@ -0,0 +1,78 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813041000_learning_reviewer_attempt_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TRIGGER learning_reviewer_attempt_identity_immutable + BEFORE UPDATE ON learning_reviewer_attempt + WHEN NEW.attempt_id != OLD.attempt_id OR NEW.job_id != OLD.job_id OR + NEW.review_session_id != OLD.review_session_id OR NEW.request_ref != OLD.request_ref OR + NEW.request_hash != OLD.request_hash OR NEW.source_candidate_ids_json != OLD.source_candidate_ids_json OR + NEW.source_candidate_set_hash != OLD.source_candidate_set_hash OR NEW.provider_id != OLD.provider_id OR + NEW.model_id != OLD.model_id OR NEW.policy_hash != OLD.policy_hash OR NEW.created_at != OLD.created_at + BEGIN SELECT RAISE(ABORT, 'learning_reviewer_attempt_identity_immutable'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_reviewer_attempt_version_fence + BEFORE UPDATE ON learning_reviewer_attempt + WHEN NEW.version != OLD.version + 1 + BEGIN SELECT RAISE(ABORT, 'learning_reviewer_attempt_version_must_advance_once'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_reviewer_attempt_terminal_immutable + BEFORE UPDATE ON learning_reviewer_attempt + WHEN OLD.state IN ('settled', 'failed', 'recovery_required') + BEGIN SELECT RAISE(ABORT, 'learning_reviewer_attempt_terminal_immutable'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_reviewer_attempt_transition_guard + BEFORE UPDATE OF state ON learning_reviewer_attempt + WHEN NOT ( + (OLD.state = 'prepared' AND NEW.state IN ('prepared', 'dispatching', 'failed')) OR + (OLD.state = 'dispatching' AND NEW.state IN ('settled', 'failed', 'recovery_required')) + ) + BEGIN SELECT RAISE(ABORT, 'learning_reviewer_attempt_transition_invalid'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_reviewer_attempt_delete_guard + BEFORE DELETE ON learning_reviewer_attempt + BEGIN SELECT RAISE(ABORT, 'learning_reviewer_attempt_delete_forbidden'); END + `) + yield* tx.run("DROP TRIGGER IF EXISTS learning_job_expected_result_settlement_guard") + yield* tx.run(` + CREATE TRIGGER learning_job_expected_result_settlement_guard + BEFORE UPDATE ON learning_job + WHEN NEW.side_effect_state = 'settled' AND NEW.side_effect_kind = 'extraction' + AND NEW.result_ref IS NOT NEW.expected_result_ref + BEGIN SELECT RAISE(ABORT, 'learning_job_expected_result_mismatch'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_job_reviewer_result_guard + BEFORE UPDATE ON learning_job + WHEN NEW.side_effect_state = 'settled' AND NEW.side_effect_kind = 'reviewer' + AND NEW.review_job_id GLOB 'review:*' AND NOT EXISTS ( + SELECT 1 FROM learning_reviewer_attempt attempt + WHERE attempt.job_id = NEW.job_id AND attempt.attempt_id = NEW.review_job_id + AND attempt.state = 'settled' AND attempt.request_ref = NEW.expected_result_ref + AND attempt.response_ref = NEW.result_ref + ) + BEGIN SELECT RAISE(ABORT, 'learning_job_reviewer_receipt_required'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_reviewer_attempt_prepare_guard + BEFORE INSERT ON learning_reviewer_attempt + WHEN NOT EXISTS ( + SELECT 1 FROM learning_job job + WHERE job.job_id = NEW.job_id AND job.state = 'reviewing' AND job.side_effect_state = 'started' + AND job.side_effect_kind = 'reviewer' AND job.review_job_id = NEW.attempt_id + AND job.expected_result_ref = NEW.request_ref AND job.owner = NEW.owner + AND job.lease_expires_at > NEW.created_at + ) + BEGIN SELECT RAISE(ABORT, 'learning_reviewer_attempt_job_receipt_required'); END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813041100_learning_lifecycle_trigger_authority.ts b/packages/core/src/database/migration/20260813041100_learning_lifecycle_trigger_authority.ts new file mode 100644 index 000000000..94d348090 --- /dev/null +++ b/packages/core/src/database/migration/20260813041100_learning_lifecycle_trigger_authority.ts @@ -0,0 +1,40 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813041100_learning_lifecycle_trigger_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TRIGGER learning_lifecycle_trigger_identity_immutable + BEFORE UPDATE ON learning_lifecycle_trigger_receipt + WHEN NEW.receipt_id != OLD.receipt_id + OR NEW.trigger != OLD.trigger + OR NEW.boundary_key != OLD.boundary_key + OR NEW.session_id != OLD.session_id + OR NEW.run_id != OLD.run_id + OR NEW.source_admission_hash != OLD.source_admission_hash + OR NEW.source_terminal_hash != OLD.source_terminal_hash + OR NEW.artifact_path != OLD.artifact_path + OR NEW.artifact_hash != OLD.artifact_hash + OR NEW.artifact_json != OLD.artifact_json + OR NEW.admission_fingerprint != OLD.admission_fingerprint + OR NEW.admission_json != OLD.admission_json + OR NEW.created_at != OLD.created_at + BEGIN SELECT RAISE(ABORT, 'learning_lifecycle_trigger_identity_immutable'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_lifecycle_trigger_transition_guard + BEFORE UPDATE OF state ON learning_lifecycle_trigger_receipt + WHEN NOT (OLD.state = 'prepared' AND NEW.state = 'admitted') + BEGIN SELECT RAISE(ABORT, 'learning_lifecycle_trigger_transition_invalid'); END + `) + yield* tx.run(` + CREATE TRIGGER learning_lifecycle_trigger_terminal_immutable + BEFORE UPDATE ON learning_lifecycle_trigger_receipt + WHEN OLD.state = 'admitted' + BEGIN SELECT RAISE(ABORT, 'learning_lifecycle_trigger_terminal_immutable'); END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813041200_v2_provider_shadow_authority.ts b/packages/core/src/database/migration/20260813041200_v2_provider_shadow_authority.ts new file mode 100644 index 000000000..281ce67ba --- /dev/null +++ b/packages/core/src/database/migration/20260813041200_v2_provider_shadow_authority.ts @@ -0,0 +1,194 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813041200_v2_provider_shadow_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TRIGGER session_v2_provider_turn_receipt_insert_guard + BEFORE INSERT ON session_v2_provider_turn_receipt + WHEN NEW.state != 'preparing' + OR NEW.prepared_turn_hash IS NOT NULL + OR NEW.wire_request_hash IS NOT NULL + OR NEW.prepared_turn IS NOT NULL + OR NEW.outcome_hash IS NOT NULL + OR NEW.error_code IS NOT NULL + OR NEW.dispatching_at IS NOT NULL + OR NEW.first_event_at IS NOT NULL + OR NEW.terminal_at IS NOT NULL + OR NOT EXISTS ( + SELECT 1 FROM session_provider_owner_lease owner + WHERE owner.owner_token = NEW.owner_token + AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.created_at + ) + BEGIN + SELECT RAISE(ABORT, 'v2 provider receipt admission requires a live owner and preparing state'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_turn_receipt_identity_immutable + BEFORE UPDATE ON session_v2_provider_turn_receipt + WHEN NEW.receipt_id != OLD.receipt_id + OR NEW.session_id != OLD.session_id + OR NEW.request_ordinal != OLD.request_ordinal + OR NEW.user_message_id != OLD.user_message_id + OR NEW.history_prompt_epoch != OLD.history_prompt_epoch + OR NEW.history_source_end_message_id IS NOT OLD.history_source_end_message_id + OR NEW.request_input_hash != OLD.request_input_hash + OR NEW.provider_id != OLD.provider_id + OR NEW.model_id != OLD.model_id + OR NEW.protocol != OLD.protocol + OR NEW.owner_mode != OLD.owner_mode + OR NEW.owner_token != OLD.owner_token + OR NEW.created_at != OLD.created_at + OR (OLD.prepared_turn_hash IS NOT NULL AND NEW.prepared_turn_hash IS NOT OLD.prepared_turn_hash) + OR (OLD.wire_request_hash IS NOT NULL AND NEW.wire_request_hash IS NOT OLD.wire_request_hash) + OR (OLD.prepared_turn IS NOT NULL AND NEW.prepared_turn IS NOT OLD.prepared_turn) + OR (OLD.outcome_hash IS NOT NULL AND NEW.outcome_hash IS NOT OLD.outcome_hash) + OR (OLD.error_code IS NOT NULL AND NEW.error_code IS NOT OLD.error_code) + OR (OLD.dispatching_at IS NOT NULL AND NEW.dispatching_at IS NOT OLD.dispatching_at) + OR (OLD.first_event_at IS NOT NULL AND NEW.first_event_at IS NOT OLD.first_event_at) + OR (OLD.terminal_at IS NOT NULL AND NEW.terminal_at IS NOT OLD.terminal_at) + BEGIN + SELECT RAISE(ABORT, 'v2 provider receipt identity is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_turn_receipt_transition_guard + BEFORE UPDATE OF state ON session_v2_provider_turn_receipt + WHEN NOT ( + (OLD.state = 'preparing' AND NEW.state = 'dispatching' + AND NEW.prepared_turn_hash IS NOT NULL + AND NEW.wire_request_hash IS NOT NULL + AND NEW.prepared_turn IS NOT NULL + AND json_extract(NEW.prepared_turn, '$.request_hash') = NEW.prepared_turn_hash + AND json_extract(NEW.prepared_turn, '$.wire_request_hash') = NEW.wire_request_hash + AND NEW.dispatching_at IS NOT NULL + AND EXISTS ( + SELECT 1 FROM session_provider_owner_lease owner + WHERE owner.owner_token = NEW.owner_token + AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.dispatching_at + )) + OR (OLD.state = 'dispatching' AND NEW.state = 'streaming' + AND NEW.first_event_at IS NOT NULL) + OR (OLD.state IN ('dispatching', 'streaming') AND NEW.state IN ('settled', 'failed') + AND NEW.outcome_hash IS NOT NULL AND NEW.terminal_at IS NOT NULL) + OR (OLD.state = 'preparing' AND NEW.state = 'failed' + AND NEW.error_code IS NOT NULL AND NEW.terminal_at IS NOT NULL + AND ( + NEW.error_code != 'owner_lost_before_dispatch' + OR ( + NOT EXISTS ( + SELECT 1 FROM session_provider_owner_lease owner + WHERE owner.owner_token = OLD.owner_token AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.terminal_at + ) + AND EXISTS ( + SELECT 1 FROM session_provider_owner_lease recovery + WHERE recovery.owner_token != OLD.owner_token AND recovery.released_at IS NULL + AND recovery.lease_expires_at > NEW.terminal_at + ) + ) + )) + OR (OLD.state IN ('dispatching', 'streaming') AND NEW.state = 'indeterminate_after_crash' + AND NEW.error_code IN ('owner_lost_after_dispatch', 'consumer_cancelled_after_dispatch') + AND NEW.terminal_at IS NOT NULL + AND ( + (NEW.error_code = 'consumer_cancelled_after_dispatch' AND EXISTS ( + SELECT 1 FROM session_provider_owner_lease owner + WHERE owner.owner_token = OLD.owner_token AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.terminal_at + )) + OR (NEW.error_code = 'owner_lost_after_dispatch' + AND NOT EXISTS ( + SELECT 1 FROM session_provider_owner_lease owner + WHERE owner.owner_token = OLD.owner_token AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.terminal_at + ) + AND EXISTS ( + SELECT 1 FROM session_provider_owner_lease recovery + WHERE recovery.owner_token != OLD.owner_token AND recovery.released_at IS NULL + AND recovery.lease_expires_at > NEW.terminal_at + )) + )) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal v2 provider receipt transition'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_turn_receipt_delete_guard + BEFORE DELETE ON session_v2_provider_turn_receipt + BEGIN + SELECT RAISE(ABORT, 'v2 provider receipts are append only'); + END + `) + + yield* tx.run(` + CREATE TRIGGER session_v2_provider_parity_receipt_authority_guard + BEFORE INSERT ON session_v2_provider_parity_receipt + WHEN NOT EXISTS ( + SELECT 1 FROM session_tool_request_receipt legacy + WHERE legacy.receipt_id = NEW.legacy_receipt_id + AND legacy.provider_state = 'settled' + AND legacy.prepared_turn_hash = NEW.legacy_request_hash + AND legacy.response_fingerprint = NEW.legacy_outcome_hash + AND json_extract(NEW.legacy_prepared_turn, '$.receipt_id') = legacy.receipt_id + AND json_extract(NEW.legacy_prepared_turn, '$.request_ordinal') = legacy.request_ordinal + AND json_extract(NEW.legacy_prepared_turn, '$.session_id') = legacy.session_id + AND json_extract(NEW.legacy_prepared_turn, '$.user_message_id') = legacy.user_message_id + AND json_extract(NEW.legacy_prepared_turn, '$.request_hash') = legacy.prepared_turn_hash + AND json_extract(NEW.legacy_prepared_turn, '$.wire_request_hash') = legacy.wire_request_hash + ) OR NOT EXISTS ( + SELECT 1 FROM session_v2_provider_turn_receipt core + WHERE core.receipt_id = NEW.core_v2_receipt_id + AND core.state = 'settled' + AND core.prepared_turn_hash = NEW.core_v2_request_hash + AND core.outcome_hash = NEW.core_v2_outcome_hash + AND NEW.core_v2_prepared_turn = core.prepared_turn + AND json_extract(NEW.core_v2_prepared_turn, '$.request_hash') = core.prepared_turn_hash + ) OR EXISTS ( + SELECT 1 FROM json_each(NEW.allowlisted_differences) + WHERE value NOT IN ( + 'owner', 'receipt_id', 'provider_attempt_id', 'assistant_message_id', 'prepared_at' + ) + ) OR EXISTS ( + SELECT 1 FROM json_each(NEW.evidence) + WHERE value NOT IN ('shadow_snapshot', 'recorded_provider', 'real_session_replay') + ) OR NEW.verified NOT IN (0, 1) + OR (NEW.verified = 1 AND ( + NEW.legacy_request_hash != NEW.core_v2_request_hash + OR NEW.legacy_outcome_hash != NEW.core_v2_outcome_hash + OR json_array_length(NEW.disallowed_differences) != 0 + OR json_remove( + NEW.legacy_prepared_turn, + '$.owner', '$.receipt_id', '$.provider_attempt_id', '$.assistant_message_id', '$.prepared_at' + ) != json_remove( + NEW.core_v2_prepared_turn, + '$.owner', '$.receipt_id', '$.provider_attempt_id', '$.assistant_message_id', '$.prepared_at' + ) + )) + BEGIN + SELECT RAISE(ABORT, 'v2 provider parity receipt lacks exact terminal authority'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_parity_receipt_delete_guard + BEFORE DELETE ON session_v2_provider_parity_receipt + BEGIN + SELECT RAISE(ABORT, 'v2 provider parity receipts are append only'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_parity_receipt_update_guard + BEFORE UPDATE ON session_v2_provider_parity_receipt + BEGIN + SELECT RAISE(ABORT, 'v2 provider parity receipts are immutable'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813041300_learning_governance_compensation_authority.ts b/packages/core/src/database/migration/20260813041300_learning_governance_compensation_authority.ts new file mode 100644 index 000000000..8c13bb2c6 --- /dev/null +++ b/packages/core/src/database/migration/20260813041300_learning_governance_compensation_authority.ts @@ -0,0 +1,96 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813041300_learning_governance_compensation_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TRIGGER learning_governance_compensation_identity_immutable + BEFORE UPDATE ON learning_governance_compensation + WHEN NEW.compensation_id != OLD.compensation_id + OR NEW.plan_id != OLD.plan_id + OR NEW.action_id != OLD.action_id + OR NEW.sequence != OLD.sequence + OR NEW.kind != OLD.kind + OR NEW.source_payload_fingerprint != OLD.source_payload_fingerprint + OR NEW.created_at != OLD.created_at + BEGIN + SELECT RAISE(ABORT, 'learning_governance_compensation_identity_immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_compensation_insert_guard + BEFORE INSERT ON learning_governance_compensation + WHEN NOT EXISTS ( + SELECT 1 + FROM learning_governance_action action + JOIN learning_governance_plan plan ON plan.plan_id = action.plan_id + WHERE action.action_id = NEW.action_id + AND action.plan_id = NEW.plan_id + AND ( + action.state = 'settled' OR ( + action.state = 'recovery_required' + AND action.result_ref IS NOT NULL + AND action.result_hash IS NOT NULL + AND action.result_fingerprint IS NOT NULL + ) + ) + AND action.payload_fingerprint = NEW.source_payload_fingerprint + AND NEW.kind = CASE action.kind + WHEN 'document_stage' THEN 'document_quarantine' + WHEN 'memory_inbox' THEN 'memory_inbox_revoke' + END + AND plan.state = 'recovery_required' + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_compensation_source_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_compensation_version_fence + BEFORE UPDATE ON learning_governance_compensation + WHEN NEW.version != OLD.version + 1 + BEGIN + SELECT RAISE(ABORT, 'learning_governance_compensation_version_must_advance_once'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_compensation_transition_guard + BEFORE UPDATE OF state ON learning_governance_compensation + WHEN NOT ( + (OLD.state = 'prepared' AND NEW.state = 'running' + AND NEW.lease_expires_at > NEW.updated_at) OR + (OLD.state = 'running' AND NEW.state = 'running' + AND OLD.lease_expires_at <= NEW.updated_at + AND NEW.lease_expires_at > NEW.updated_at) OR + (OLD.state = 'running' AND NEW.state IN ('settled', 'recovery_required') + AND NEW.owner IS NULL) + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_compensation_transition_invalid'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_compensation_sequence_guard + BEFORE UPDATE OF state ON learning_governance_compensation + WHEN NEW.state = 'running' AND NEW.sequence > 0 AND NOT EXISTS ( + SELECT 1 FROM learning_governance_compensation predecessor + WHERE predecessor.plan_id = NEW.plan_id + AND predecessor.sequence = NEW.sequence - 1 + AND predecessor.state = 'settled' + ) + BEGIN + SELECT RAISE(ABORT, 'learning_governance_compensation_predecessor_unsettled'); + END + `) + yield* tx.run(` + CREATE TRIGGER learning_governance_compensation_delete_guard + BEFORE DELETE ON learning_governance_compensation + BEGIN + SELECT RAISE(ABORT, 'learning_governance_compensation_delete_forbidden'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813041400_context_activation_semantic_authority.ts b/packages/core/src/database/migration/20260813041400_context_activation_semantic_authority.ts new file mode 100644 index 000000000..4cdb5bf34 --- /dev/null +++ b/packages/core/src/database/migration/20260813041400_context_activation_semantic_authority.ts @@ -0,0 +1,92 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813041400_context_activation_semantic_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run("DROP TRIGGER IF EXISTS session_tool_request_receipt_context_activation_semantic_guard") + yield* tx.run(` + CREATE TRIGGER session_tool_request_receipt_context_activation_semantic_guard + BEFORE UPDATE OF provider_state ON session_tool_request_receipt + WHEN NEW.provider_state IN ('prepared', 'dispatching') AND ${invalidActivation("NEW")} + BEGIN + SELECT RAISE(ABORT, 'provider receipt context activation semantics are invalid'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration + +function invalidActivation(row: string) { + const activation = `${row}.context_activation` + const eligibility = `${row}.context_eligibility` + const readiness = `${row}.context_readiness` + return `( + json_extract(${activation}, '$.schemaVersion') IS NOT 1 OR + COALESCE(json_type(${activation}, '$.recordedAt'), 'missing') NOT IN ('integer', 'real') OR + COALESCE(json_type(${activation}, '$.readinessAgeMs'), 'missing') NOT IN ('integer', 'real') OR + COALESCE(json_type(${activation}, '$.readinessExpiresInMs'), 'missing') NOT IN ('integer', 'real') OR + COALESCE(json_type(${readiness}, '$.observedAt'), 'missing') NOT IN ('integer', 'real') OR + COALESCE(json_type(${readiness}, '$.expiresAt'), 'missing') NOT IN ('integer', 'real') OR + json_extract(${activation}, '$.readinessAgeMs') != MAX( + 0, + json_extract(${activation}, '$.recordedAt') - json_extract(${readiness}, '$.observedAt') + ) OR + json_extract(${activation}, '$.readinessExpiresInMs') != + json_extract(${readiness}, '$.expiresAt') - json_extract(${activation}, '$.recordedAt') OR + COALESCE(json_extract(${activation}, '$.outcome'), '') NOT IN + ('active', 'shadow_only', 'fallback', 'not_requested') OR + json_type(${activation}, '$.enabledCapabilities') IS NOT 'array' OR + json_type(${activation}, '$.fallbackReasons') IS NOT 'array' OR + EXISTS ( + SELECT 1 + FROM json_each(${activation}, '$.enabledCapabilities') capability + WHERE capability.type != 'text' OR capability.value NOT IN + ('context_projection_v2', 'context_query_tools_v2') + ) OR + (SELECT count(*) FROM json_each(${activation}, '$.enabledCapabilities')) != + (SELECT count(DISTINCT capability.value) + FROM json_each(${activation}, '$.enabledCapabilities') capability) OR + EXISTS ( + SELECT 1 + FROM json_each(${activation}, '$.fallbackReasons') reason + WHERE reason.type != 'text' OR reason.value = '' + ) OR + (json_array_length(json_extract(${activation}, '$.enabledCapabilities')) > 0) != + (json_extract(${activation}, '$.outcome') = 'active') OR + (EXISTS ( + SELECT 1 FROM json_each(${activation}, '$.enabledCapabilities') + WHERE value = 'context_projection_v2' + ) AND json_extract(${activation}, '$.decision.enabled.contextProjectionV2') IS NOT 1) OR + (EXISTS ( + SELECT 1 FROM json_each(${activation}, '$.enabledCapabilities') + WHERE value = 'context_query_tools_v2' + ) AND json_extract(${activation}, '$.decision.enabled.contextQueryToolsV2') IS NOT 1) OR + json_extract(${activation}, '$.decision.requested') IS NOT json_extract(${eligibility}, '$.requested') OR + json_extract(${activation}, '$.decision.project') IS NOT json_extract(${eligibility}, '$.project') OR + (json_extract(${activation}, '$.decision.enabled.contextProjectionV2') = 1 AND + json_extract(${eligibility}, '$.enabled.contextProjectionV2') IS NOT 1) OR + (json_extract(${activation}, '$.decision.enabled.contextQueryToolsV2') = 1 AND + json_extract(${eligibility}, '$.enabled.contextQueryToolsV2') IS NOT 1) OR + (json_extract(${activation}, '$.decision.enabled.coreV2ExecutionOwner') = 1 AND + json_extract(${eligibility}, '$.enabled.coreV2ExecutionOwner') IS NOT 1) OR + (json_extract(${readiness}, '$.expiresAt') <= json_extract(${activation}, '$.recordedAt') AND ( + json_array_length(json_extract(${activation}, '$.enabledCapabilities')) != 0 OR + json_extract(${activation}, '$.decision.enabled.contextProjectionV2') IS NOT 0 OR + json_extract(${activation}, '$.decision.enabled.contextQueryToolsV2') IS NOT 0 OR + json_extract(${activation}, '$.decision.enabled.coreV2ExecutionOwner') IS NOT 0 + )) OR + (${row}.context_selection_id IS NULL AND json_type(${activation}, '$.selection') IS NOT NULL) OR + (${row}.context_selection_id IS NOT NULL AND ( + json_extract(${activation}, '$.selection.selectionId') IS NOT ${row}.context_selection_id OR + NOT EXISTS ( + SELECT 1 + FROM session_context_selection selection + WHERE selection.selection_id = ${row}.context_selection_id + AND selection.session_id = ${row}.session_id + AND selection.projection_hash = json_extract(${activation}, '$.selection.projectionHash') + ) + )) + )` +} diff --git a/packages/core/src/database/migration/20260813120346_v2_provider_parity_campaign.ts b/packages/core/src/database/migration/20260813120346_v2_provider_parity_campaign.ts new file mode 100644 index 000000000..a404297c5 --- /dev/null +++ b/packages/core/src/database/migration/20260813120346_v2_provider_parity_campaign.ts @@ -0,0 +1,29 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813120346_v2_provider_parity_campaign", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(` + CREATE TABLE \`session_v2_provider_parity_baseline\` ( + \`campaign_id\` text NOT NULL, + \`case_name\` text NOT NULL, + \`legacy_receipt_id\` text NOT NULL UNIQUE, + \`state\` text NOT NULL, + \`prepared_turn\` text NOT NULL, + \`outcome_hash\` text, + \`outcome_artifact\` text, + \`evidence\` text NOT NULL, + \`receipt_hash\` text NOT NULL, + \`created_at\` integer NOT NULL, + \`settled_at\` integer, + CONSTRAINT \`session_v2_provider_parity_baseline_pk\` PRIMARY KEY(\`campaign_id\`, \`case_name\`) + ); + `) + yield* tx.run(`ALTER TABLE \`session_v2_provider_turn_receipt\` ADD \`outcome_artifact\` text;`) + yield* tx.run(`CREATE UNIQUE INDEX \`session_v2_provider_parity_baseline_hash_idx\` ON \`session_v2_provider_parity_baseline\` (\`receipt_hash\`);`) + yield* tx.run(`CREATE INDEX \`session_v2_provider_parity_baseline_campaign_idx\` ON \`session_v2_provider_parity_baseline\` (\`campaign_id\`,\`state\`);`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813121129_v2_provider_parity_response_fingerprint.ts b/packages/core/src/database/migration/20260813121129_v2_provider_parity_response_fingerprint.ts new file mode 100644 index 000000000..49e8a4715 --- /dev/null +++ b/packages/core/src/database/migration/20260813121129_v2_provider_parity_response_fingerprint.ts @@ -0,0 +1,11 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +export default { + id: "20260813121129_v2_provider_parity_response_fingerprint", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(`ALTER TABLE \`session_v2_provider_parity_baseline\` ADD \`legacy_response_fingerprint\` text;`) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/database/migration/20260813121200_v2_provider_parity_campaign_authority.ts b/packages/core/src/database/migration/20260813121200_v2_provider_parity_campaign_authority.ts new file mode 100644 index 000000000..fbd7066c7 --- /dev/null +++ b/packages/core/src/database/migration/20260813121200_v2_provider_parity_campaign_authority.ts @@ -0,0 +1,251 @@ +import { Effect } from "effect" +import type { DatabaseMigration } from "../migration" + +const cases = [ + "admission_activity", + "batched_steer_queue", + "exact_retry_interrupt", + "stable_prefix_runtime_tail_cache", + "tool_registry_permissions_question", + "attachments_location_agent_model", + "compaction_overflow_provider_errors_tokens", + "tool_settlement_cancellation", + "task_goal_finalizer_im", + "status_events_telemetry", + "provider_contract_replay", +] + +export default { + id: "20260813121200_v2_provider_parity_campaign_authority", + up(tx) { + return Effect.gen(function* () { + yield* tx.run(`DROP TRIGGER session_v2_provider_turn_receipt_transition_guard`) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_turn_receipt_transition_guard + BEFORE UPDATE OF state ON session_v2_provider_turn_receipt + WHEN NOT ( + (OLD.state = 'preparing' AND NEW.state = 'dispatching' + AND NEW.prepared_turn_hash IS NOT NULL + AND NEW.wire_request_hash IS NOT NULL + AND NEW.prepared_turn IS NOT NULL + AND json_extract(NEW.prepared_turn, '$.request_hash') = NEW.prepared_turn_hash + AND json_extract(NEW.prepared_turn, '$.wire_request_hash') = NEW.wire_request_hash + AND NEW.dispatching_at IS NOT NULL + AND EXISTS ( + SELECT 1 FROM session_provider_owner_lease owner + WHERE owner.owner_token = NEW.owner_token + AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.dispatching_at + )) + OR (OLD.state = 'dispatching' AND NEW.state = 'streaming' + AND NEW.first_event_at IS NOT NULL) + OR (OLD.state IN ('dispatching', 'streaming') AND NEW.state IN ('settled', 'failed') + AND length(NEW.outcome_hash) = 64 + AND NEW.outcome_hash NOT GLOB '*[^0-9a-f]*' + AND json_valid(NEW.outcome_artifact) + AND json_type(NEW.outcome_artifact) = 'array' + AND NEW.terminal_at IS NOT NULL) + OR (OLD.state = 'preparing' AND NEW.state = 'failed' + AND NEW.outcome_artifact IS NULL + AND NEW.error_code IS NOT NULL AND NEW.terminal_at IS NOT NULL + AND ( + NEW.error_code != 'owner_lost_before_dispatch' + OR ( + NOT EXISTS ( + SELECT 1 FROM session_provider_owner_lease owner + WHERE owner.owner_token = OLD.owner_token AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.terminal_at + ) + AND EXISTS ( + SELECT 1 FROM session_provider_owner_lease recovery + WHERE recovery.owner_token != OLD.owner_token AND recovery.released_at IS NULL + AND recovery.lease_expires_at > NEW.terminal_at + ) + ) + )) + OR (OLD.state IN ('dispatching', 'streaming') AND NEW.state = 'indeterminate_after_crash' + AND NEW.outcome_artifact IS NULL + AND NEW.error_code IN ('owner_lost_after_dispatch', 'consumer_cancelled_after_dispatch') + AND NEW.terminal_at IS NOT NULL + AND ( + (NEW.error_code = 'consumer_cancelled_after_dispatch' AND EXISTS ( + SELECT 1 FROM session_provider_owner_lease owner + WHERE owner.owner_token = OLD.owner_token AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.terminal_at + )) + OR (NEW.error_code = 'owner_lost_after_dispatch' + AND NOT EXISTS ( + SELECT 1 FROM session_provider_owner_lease owner + WHERE owner.owner_token = OLD.owner_token AND owner.released_at IS NULL + AND owner.lease_expires_at > NEW.terminal_at + ) + AND EXISTS ( + SELECT 1 FROM session_provider_owner_lease recovery + WHERE recovery.owner_token != OLD.owner_token AND recovery.released_at IS NULL + AND recovery.lease_expires_at > NEW.terminal_at + )) + )) + ) + BEGIN + SELECT RAISE(ABORT, 'illegal v2 provider receipt transition'); + END + `) + yield* tx.run(`DROP TRIGGER session_v2_provider_turn_receipt_identity_immutable`) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_turn_receipt_identity_immutable + BEFORE UPDATE ON session_v2_provider_turn_receipt + WHEN NEW.receipt_id != OLD.receipt_id + OR NEW.session_id != OLD.session_id + OR NEW.user_message_id != OLD.user_message_id + OR NEW.request_ordinal != OLD.request_ordinal + OR NEW.owner_token != OLD.owner_token + OR NEW.owner_mode != OLD.owner_mode + OR NEW.history_prompt_epoch != OLD.history_prompt_epoch + OR NEW.history_source_end_message_id IS NOT OLD.history_source_end_message_id + OR NEW.request_input_hash != OLD.request_input_hash + OR NEW.provider_id != OLD.provider_id + OR NEW.model_id != OLD.model_id + OR NEW.protocol != OLD.protocol + OR (OLD.prepared_turn_hash IS NOT NULL AND NEW.prepared_turn_hash IS NOT OLD.prepared_turn_hash) + OR (OLD.wire_request_hash IS NOT NULL AND NEW.wire_request_hash IS NOT OLD.wire_request_hash) + OR (OLD.prepared_turn IS NOT NULL AND NEW.prepared_turn IS NOT OLD.prepared_turn) + OR (OLD.outcome_hash IS NOT NULL AND NEW.outcome_hash IS NOT OLD.outcome_hash) + OR (OLD.outcome_artifact IS NOT NULL AND NEW.outcome_artifact IS NOT OLD.outcome_artifact) + OR (OLD.error_code IS NOT NULL AND NEW.error_code IS NOT OLD.error_code) + OR (OLD.dispatching_at IS NOT NULL AND NEW.dispatching_at IS NOT OLD.dispatching_at) + OR (OLD.first_event_at IS NOT NULL AND NEW.first_event_at IS NOT OLD.first_event_at) + OR (OLD.terminal_at IS NOT NULL AND NEW.terminal_at IS NOT OLD.terminal_at) + OR NEW.created_at != OLD.created_at + BEGIN + SELECT RAISE(ABORT, 'v2 provider receipt identity is immutable'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_parity_baseline_insert_guard + BEFORE INSERT ON session_v2_provider_parity_baseline + WHEN length(trim(NEW.campaign_id)) = 0 + OR length(NEW.campaign_id) > 128 + OR NEW.case_name NOT IN (${cases.map((item) => `'${item}'`).join(", ")}) + OR NEW.state != 'prepared' + OR NEW.outcome_hash IS NOT NULL + OR NEW.outcome_artifact IS NOT NULL + OR NEW.legacy_response_fingerprint IS NOT NULL + OR NEW.settled_at IS NOT NULL + OR length(NEW.receipt_hash) != 64 + OR NEW.receipt_hash GLOB '*[^0-9a-f]*' + OR NOT json_valid(NEW.prepared_turn) + OR json_type(NEW.prepared_turn) != 'object' + OR NOT json_valid(NEW.evidence) + OR json_type(NEW.evidence) != 'array' + OR json_array_length(NEW.evidence) != 2 + OR NOT EXISTS (SELECT 1 FROM json_each(NEW.evidence) WHERE value = 'shadow_snapshot') + OR NOT EXISTS (SELECT 1 FROM json_each(NEW.evidence) WHERE value = 'recorded_provider') + OR EXISTS (SELECT 1 FROM json_each(NEW.evidence) WHERE value NOT IN ('shadow_snapshot', 'recorded_provider')) + OR NOT EXISTS ( + SELECT 1 FROM session_tool_request_receipt legacy + WHERE legacy.receipt_id = NEW.legacy_receipt_id + AND legacy.provider_state = 'prepared' + AND legacy.prepared_turn_hash = json_extract(NEW.prepared_turn, '$.request_hash') + AND legacy.wire_request_hash = json_extract(NEW.prepared_turn, '$.wire_request_hash') + AND legacy.request_ordinal = json_extract(NEW.prepared_turn, '$.request_ordinal') + AND legacy.session_id = json_extract(NEW.prepared_turn, '$.session_id') + AND legacy.user_message_id = json_extract(NEW.prepared_turn, '$.user_message_id') + ) + BEGIN + SELECT RAISE(ABORT, 'v2 parity baseline lacks exact prepared legacy authority'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_parity_baseline_update_guard + BEFORE UPDATE ON session_v2_provider_parity_baseline + WHEN NEW.campaign_id != OLD.campaign_id + OR NEW.case_name != OLD.case_name + OR NEW.legacy_receipt_id != OLD.legacy_receipt_id + OR NEW.prepared_turn != OLD.prepared_turn + OR NEW.evidence != OLD.evidence + OR NEW.receipt_hash != OLD.receipt_hash + OR NEW.created_at != OLD.created_at + OR NOT (OLD.state = 'prepared' AND NEW.state = 'settled') + OR length(NEW.outcome_hash) != 64 + OR NEW.outcome_hash GLOB '*[^0-9a-f]*' + OR NOT json_valid(NEW.outcome_artifact) + OR json_type(NEW.outcome_artifact) != 'array' + OR length(NEW.legacy_response_fingerprint) != 64 + OR NEW.legacy_response_fingerprint GLOB '*[^0-9a-f]*' + OR NEW.settled_at IS NULL + OR NOT EXISTS ( + SELECT 1 FROM session_tool_request_receipt legacy + WHERE legacy.receipt_id = NEW.legacy_receipt_id + AND legacy.provider_state = 'settled' + AND legacy.prepared_turn_hash = json_extract(NEW.prepared_turn, '$.request_hash') + AND legacy.wire_request_hash = json_extract(NEW.prepared_turn, '$.wire_request_hash') + AND legacy.response_fingerprint = NEW.legacy_response_fingerprint + ) + BEGIN + SELECT RAISE(ABORT, 'v2 parity baseline settlement lacks exact legacy terminal authority'); + END + `) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_parity_baseline_delete_guard + BEFORE DELETE ON session_v2_provider_parity_baseline + BEGIN + SELECT RAISE(ABORT, 'v2 parity baselines are append only'); + END + `) + yield* tx.run(`DROP TRIGGER session_v2_provider_parity_receipt_authority_guard`) + yield* tx.run(` + CREATE TRIGGER session_v2_provider_parity_receipt_authority_guard + BEFORE INSERT ON session_v2_provider_parity_receipt + WHEN NOT EXISTS ( + SELECT 1 FROM session_v2_provider_parity_baseline baseline + WHERE baseline.campaign_id = NEW.campaign_id + AND baseline.case_name = NEW.case_name + AND baseline.legacy_receipt_id = NEW.legacy_receipt_id + AND baseline.state = 'settled' + AND baseline.prepared_turn = NEW.legacy_prepared_turn + AND baseline.outcome_hash = NEW.legacy_outcome_hash + AND baseline.outcome_artifact IS NOT NULL + ) OR NOT EXISTS ( + SELECT 1 FROM session_v2_provider_turn_receipt core + WHERE core.receipt_id = NEW.core_v2_receipt_id + AND core.state = 'settled' + AND core.prepared_turn_hash = NEW.core_v2_request_hash + AND core.outcome_hash = NEW.core_v2_outcome_hash + AND core.outcome_artifact IS NOT NULL + AND NEW.core_v2_prepared_turn = core.prepared_turn + AND json_extract(NEW.core_v2_prepared_turn, '$.request_hash') = core.prepared_turn_hash + ) OR (NEW.verified = 1 AND NOT EXISTS ( + SELECT 1 FROM session_v2_provider_parity_baseline baseline + JOIN session_v2_provider_turn_receipt core ON core.receipt_id = NEW.core_v2_receipt_id + WHERE baseline.campaign_id = NEW.campaign_id + AND baseline.case_name = NEW.case_name + AND baseline.outcome_artifact = core.outcome_artifact + )) OR EXISTS ( + SELECT 1 FROM json_each(NEW.allowlisted_differences) + WHERE value NOT IN ('owner', 'receipt_id', 'provider_attempt_id', 'assistant_message_id', 'prepared_at') + ) OR NOT EXISTS (SELECT 1 FROM json_each(NEW.evidence) WHERE value = 'shadow_snapshot') + OR NOT EXISTS (SELECT 1 FROM json_each(NEW.evidence) WHERE value = 'recorded_provider') + OR NOT EXISTS (SELECT 1 FROM json_each(NEW.evidence) WHERE value = 'real_session_replay') + OR EXISTS ( + SELECT 1 FROM json_each(NEW.evidence) + WHERE value NOT IN ('shadow_snapshot', 'recorded_provider', 'real_session_replay') + ) OR NEW.verified NOT IN (0, 1) + OR (NEW.verified = 1 AND ( + NEW.legacy_request_hash != NEW.core_v2_request_hash + OR NEW.legacy_outcome_hash != NEW.core_v2_outcome_hash + OR json_array_length(NEW.disallowed_differences) != 0 + OR json_remove( + NEW.legacy_prepared_turn, + '$.owner', '$.receipt_id', '$.provider_attempt_id', '$.assistant_message_id', '$.prepared_at' + ) != json_remove( + NEW.core_v2_prepared_turn, + '$.owner', '$.receipt_id', '$.provider_attempt_id', '$.assistant_message_id', '$.prepared_at' + ) + )) + BEGIN + SELECT RAISE(ABORT, 'v2 provider parity receipt lacks exact campaign authority'); + END + `) + }) + }, +} satisfies DatabaseMigration.Migration diff --git a/packages/core/src/deepagent/activity-authority.sql.ts b/packages/core/src/deepagent/activity-authority.sql.ts new file mode 100644 index 000000000..f522b63c8 --- /dev/null +++ b/packages/core/src/deepagent/activity-authority.sql.ts @@ -0,0 +1,250 @@ +import { sql } from "drizzle-orm" +import { index, integer, primaryKey, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { ProjectTable } from "../project/sql" +import { ProjectV2 } from "../project" +import { SessionTable } from "../session/sql" +import type { SessionSchema } from "../session/schema" +import type { CompletionCriterion } from "./goal-loop" + +export type ActivityKind = "legacy" | "v2" +export type ActivityObjectiveState = "active" | "completed" | "needs_human" | "interrupted" | "recovery_required" +export type PermissionRequestState = + | "pending" + | "approved_once" + | "approved_always" + | "denied" + | "expired" + | "interrupted" +export type PermissionEffectDispatchState = "started" | "settled" | "unknown" + +export const SessionActivityObjectiveTable = sqliteTable( + "session_activity_objective", + { + activity_kind: text().$type().notNull(), + activity_id: text().notNull(), + session_id: text() + .$type() + .notNull() + .references(() => SessionTable.id, { onDelete: "cascade" }), + version: integer().notNull(), + admission_fingerprint: text().notNull(), + objective_fingerprint: text(), + objective_text: text(), + completion_criteria: text({ mode: "json" }).$type().notNull(), + enforcement_state: text().$type<"disabled" | "monitoring">().notNull(), + stall_threshold: integer(), + state: text().$type().notNull(), + no_progress_count: integer().notNull(), + latest_observation_revision: integer().notNull(), + latest_vector_hash: text(), + next_action: text(), + terminal_reason: text(), + created_at: integer().notNull(), + updated_at: integer().notNull(), + settled_at: integer(), + }, + (table) => [ + primaryKey({ columns: [table.activity_kind, table.activity_id] }), + index("session_activity_objective_session_idx").on(table.session_id, table.state, table.updated_at), + ], +) + +export const SessionActivityEvidenceTable = sqliteTable( + "session_activity_evidence", + { + activity_kind: text().$type().notNull(), + activity_id: text().notNull(), + evidence_fingerprint: text().notNull(), + evidence_kind: text().notNull(), + source_receipt_id: text(), + first_observation_revision: integer().notNull(), + created_at: integer().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.activity_kind, table.activity_id, table.evidence_fingerprint] }), + index("session_activity_evidence_activity_idx").on( + table.activity_kind, + table.activity_id, + table.first_observation_revision, + ), + ], +) + +export const SessionActivityEffectReceiptTable = sqliteTable( + "session_activity_effect_receipt", + { + activity_kind: text().$type().notNull(), + activity_id: text().notNull(), + receipt_id: text().notNull(), + effect_fingerprint: text().notNull(), + first_observation_revision: integer().notNull(), + created_at: integer().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.activity_kind, table.activity_id, table.receipt_id] }), + index("session_activity_effect_receipt_activity_idx").on( + table.activity_kind, + table.activity_id, + table.first_observation_revision, + ), + ], +) + +export const SessionActivityProgressObservationTable = sqliteTable( + "session_activity_progress_observation", + { + activity_kind: text().$type().notNull(), + activity_id: text().notNull(), + revision: integer().notNull(), + idempotency_key: text().notNull(), + observation_fingerprint: text().notNull(), + expected_objective_version: integer().notNull(), + workspace_revision: text(), + plan_version: integer(), + validation_fingerprint: text(), + evidence_set_hash: text().notNull(), + effect_receipt_set_hash: text().notNull(), + vector_hash: text().notNull(), + next_action: text(), + changed: integer({ mode: "boolean" }).notNull(), + no_progress_count: integer().notNull(), + observed_at: integer().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.activity_kind, table.activity_id, table.revision] }), + uniqueIndex("session_activity_progress_observation_idempotency_idx").on(table.idempotency_key), + index("session_activity_progress_observation_latest_idx").on( + table.activity_kind, + table.activity_id, + table.observed_at, + ), + ], +) + +export const SessionActivityPermissionRequestTable = sqliteTable( + "session_activity_permission_request", + { + request_id: text().primaryKey(), + activity_kind: text().$type().notNull(), + activity_id: text().notNull(), + session_id: text() + .$type() + .notNull() + .references(() => SessionTable.id, { onDelete: "cascade" }), + project_id: text() + .$type() + .notNull() + .references(() => ProjectTable.id, { onDelete: "cascade" }), + workspace_id: text(), + request_kind: text().$type<"tool" | "no_progress">().notNull(), + idempotency_key: text().notNull(), + permission: text().notNull(), + patterns: text({ mode: "json" }).$type().notNull(), + always_patterns: text({ mode: "json" }).$type().notNull(), + metadata_hash: text().notNull(), + tool_message_id: text(), + tool_call_id: text(), + state: text().$type().notNull(), + authority_epoch: integer().notNull(), + requested_scope: text().$type<"once" | "project">().notNull(), + owner_type: text().$type<"runtime" | "system">().notNull(), + owner_id: text().notNull(), + created_at: integer().notNull(), + expires_at: integer(), + decided_at: integer(), + }, + (table) => [ + uniqueIndex("session_activity_permission_request_idempotency_idx").on(table.idempotency_key), + uniqueIndex("session_activity_permission_request_pending_no_progress_idx") + .on(table.activity_kind, table.activity_id) + .where(sql`${table.state} = 'pending' AND ${table.request_kind} = 'no_progress'`), + index("session_activity_permission_request_pending_idx").on(table.session_id, table.state, table.created_at), + ], +) + +export const SessionActivityPermissionOwnerLeaseTable = sqliteTable( + "session_activity_permission_owner_lease", + { + owner_id: text().primaryKey().notNull(), + lease_expires_at: integer().notNull(), + heartbeat_at: integer().notNull(), + }, + (table) => [index("session_activity_permission_owner_lease_expiry_idx").on(table.lease_expires_at, table.owner_id)], +) + +export const SessionActivityPermissionDecisionTable = sqliteTable( + "session_activity_permission_decision", + { + decision_id: text().primaryKey(), + request_id: text().notNull(), + idempotency_key: text().notNull(), + decision: text().$type<"approved_once" | "approved_always" | "denied" | "expired" | "interrupted">().notNull(), + actor_type: text().$type<"user" | "administrator" | "system">().notNull(), + actor_id: text().notNull(), + scope: text().$type<"once" | "project">().notNull(), + authority_epoch: integer().notNull(), + decided_at: integer().notNull(), + expires_at: integer(), + feedback: text(), + }, + (table) => [ + uniqueIndex("session_activity_permission_decision_request_idx").on(table.request_id), + uniqueIndex("session_activity_permission_decision_idempotency_idx").on(table.idempotency_key), + ], +) + +export const SessionActivityPermissionOnceConsumptionTable = sqliteTable( + "session_activity_permission_once_consumption", + { + request_id: text().primaryKey(), + consumer_id: text().notNull(), + idempotency_key: text().notNull(), + consumed_at: integer().notNull(), + }, + (table) => [uniqueIndex("session_activity_permission_once_consumption_idempotency_idx").on(table.idempotency_key)], +) + +export const SessionActivityPermissionEffectDispatchTable = sqliteTable( + "session_activity_permission_effect_dispatch", + { + receipt_id: text().primaryKey(), + request_id: text() + .notNull() + .references(() => SessionActivityPermissionRequestTable.request_id, { onDelete: "cascade" }), + activity_kind: text().$type().notNull(), + activity_id: text().notNull(), + session_id: text() + .$type() + .notNull() + .references(() => SessionTable.id, { onDelete: "cascade" }), + project_id: text() + .$type() + .notNull() + .references(() => ProjectTable.id, { onDelete: "cascade" }), + workspace_id: text(), + tool_message_id: text().notNull(), + tool_call_id: text().notNull(), + tool_name: text().notNull(), + consumer_id: text().notNull(), + idempotency_key: text().notNull(), + owner_id: text().notNull(), + state: text().$type().notNull(), + version: integer().notNull(), + outcome: text().$type<"success" | "failure">(), + result_json: text({ mode: "json" }).$type(), + result_hash: text(), + started_at: integer().notNull(), + settled_at: integer(), + }, + (table) => [ + uniqueIndex("session_activity_permission_effect_dispatch_request_idx").on(table.request_id), + uniqueIndex("session_activity_permission_effect_dispatch_idempotency_idx").on(table.idempotency_key), + index("session_activity_permission_effect_dispatch_activity_idx").on( + table.activity_kind, + table.activity_id, + table.state, + table.started_at, + ), + index("session_activity_permission_effect_dispatch_owner_idx").on(table.owner_id, table.state, table.started_at), + ], +) diff --git a/packages/core/src/deepagent/learning-admission-outbox.sql.ts b/packages/core/src/deepagent/learning-admission-outbox.sql.ts new file mode 100644 index 000000000..39a779e00 --- /dev/null +++ b/packages/core/src/deepagent/learning-admission-outbox.sql.ts @@ -0,0 +1,47 @@ +import { sql } from "drizzle-orm" +import { check, index, integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { LearningJobTable } from "./learning-job.sql" + +export const LearningAdmissionOutboxTable = sqliteTable( + "learning_admission_outbox", + { + intent_id: text().primaryKey(), + session_id: text().notNull(), + run_id: text().notNull(), + trigger: text().$type<"idle" | "pause" | "project_switch" | "session_finalization">().notNull(), + dedupe_key: text().notNull(), + payload_json: text().notNull(), + payload_fingerprint: text().notNull(), + state: text().$type<"pending" | "admitted" | "rejected">().notNull(), + job_id: text().references(() => LearningJobTable.job_id), + candidate_input_ref: text(), + rejection_code: text(), + rejection_detail: text(), + created_at: integer().notNull(), + settled_at: integer(), + updated_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("learning_admission_outbox_dedupe_idx").on(table.dedupe_key), + index("learning_admission_outbox_pending_idx") + .on(table.created_at) + .where(sql`${table.state} = 'pending'`), + check( + "learning_admission_outbox_trigger_check", + sql`${table.trigger} IN ('idle', 'pause', 'project_switch', 'session_finalization')`, + ), + check( + "learning_admission_outbox_payload_json_check", + sql`json_valid(${table.payload_json}) AND json_type(${table.payload_json}) = 'object'`, + ), + check( + "learning_admission_outbox_payload_fingerprint_check", + sql`length(${table.payload_fingerprint}) = 64 AND ${table.payload_fingerprint} NOT GLOB '*[^0-9a-f]*'`, + ), + check("learning_admission_outbox_state_check", sql`${table.state} IN ('pending', 'admitted', 'rejected')`), + check( + "learning_admission_outbox_settlement_check", + sql`(${table.state} = 'pending' AND ${table.job_id} IS NULL AND ${table.candidate_input_ref} IS NULL AND ${table.rejection_code} IS NULL AND ${table.rejection_detail} IS NULL AND ${table.settled_at} IS NULL) OR (${table.state} = 'admitted' AND ${table.job_id} IS NOT NULL AND length(trim(${table.candidate_input_ref})) > 0 AND ${table.rejection_code} IS NULL AND ${table.rejection_detail} IS NULL AND ${table.settled_at} IS NOT NULL) OR (${table.state} = 'rejected' AND ${table.job_id} IS NULL AND length(trim(${table.rejection_code})) > 0 AND length(trim(${table.rejection_detail})) > 0 AND ${table.settled_at} IS NOT NULL)`, + ), + ], +) diff --git a/packages/core/src/deepagent/learning-governance.sql.ts b/packages/core/src/deepagent/learning-governance.sql.ts new file mode 100644 index 000000000..09b960385 --- /dev/null +++ b/packages/core/src/deepagent/learning-governance.sql.ts @@ -0,0 +1,172 @@ +import { sql } from "drizzle-orm" +import { check, index, integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { LearningJobTable } from "./learning-job.sql" + +export const LearningGovernancePlanTable = sqliteTable( + "learning_governance_plan", + { + plan_id: text().primaryKey(), + job_id: text() + .notNull() + .references(() => LearningJobTable.job_id), + policy: text().$type<"manual_review">().notNull(), + payload_json: text().notNull(), + payload_fingerprint: text().notNull(), + action_count: integer().notNull(), + job_owner: text().notNull(), + source_job_version: integer().notNull(), + job_started_version: integer().notNull(), + state: text().$type<"prepared" | "settled" | "recovery_required">().notNull(), + version: integer().notNull(), + result_ref: text(), + result_hash: text(), + result_fingerprint: text(), + error_code: text(), + error_detail: text(), + created_at: integer().notNull(), + settled_at: integer(), + updated_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("learning_governance_plan_job_idx").on(table.job_id), + index("learning_governance_plan_state_idx").on(table.state, table.created_at), + check("learning_governance_plan_policy_check", sql`${table.policy} = 'manual_review'`), + check("learning_governance_plan_action_count_check", sql`${table.action_count} >= 0`), + check("learning_governance_plan_version_check", sql`${table.version} >= 0`), + check("learning_governance_plan_payload_json_check", sql`json_valid(${table.payload_json})`), + check( + "learning_governance_plan_payload_fingerprint_check", + sql`length(${table.payload_fingerprint}) = 64 AND ${table.payload_fingerprint} NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "learning_governance_plan_result_hash_check", + sql`${table.result_hash} IS NULL OR (length(${table.result_hash}) = 64 AND ${table.result_hash} NOT GLOB '*[^0-9a-f]*')`, + ), + check( + "learning_governance_plan_result_fingerprint_check", + sql`${table.result_fingerprint} IS NULL OR (length(${table.result_fingerprint}) = 64 AND ${table.result_fingerprint} NOT GLOB '*[^0-9a-f]*')`, + ), + check("learning_governance_plan_state_check", sql`${table.state} IN ('prepared', 'settled', 'recovery_required')`), + check( + "learning_governance_plan_settlement_check", + sql`(${table.state} = 'prepared' AND ${table.result_ref} IS NULL AND ${table.result_hash} IS NULL AND ${table.result_fingerprint} IS NULL AND ${table.error_code} IS NULL AND ${table.settled_at} IS NULL) OR (${table.state} = 'settled' AND length(trim(${table.result_ref})) > 0 AND ${table.result_hash} IS NOT NULL AND ${table.result_fingerprint} IS NOT NULL AND ${table.error_code} IS NULL AND ${table.settled_at} IS NOT NULL) OR (${table.state} = 'recovery_required' AND length(trim(${table.error_code})) > 0 AND ${table.settled_at} IS NOT NULL)`, + ), + ], +) + +export const LearningGovernanceActionTable = sqliteTable( + "learning_governance_action", + { + action_id: text().primaryKey(), + plan_id: text() + .notNull() + .references(() => LearningGovernancePlanTable.plan_id), + candidate_id: text().notNull(), + sequence: integer().notNull(), + kind: text().$type<"document_stage" | "memory_inbox">().notNull(), + predecessor_action_id: text(), + payload_json: text().notNull(), + payload_fingerprint: text().notNull(), + state: text().$type<"prepared" | "running" | "settled" | "recovery_required">().notNull(), + owner: text(), + lease_expires_at: integer(), + version: integer().notNull(), + result_ref: text(), + result_hash: text(), + result_fingerprint: text(), + error_code: text(), + error_detail: text(), + created_at: integer().notNull(), + settled_at: integer(), + updated_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("learning_governance_action_plan_sequence_idx").on(table.plan_id, table.sequence), + uniqueIndex("learning_governance_action_plan_candidate_kind_idx").on(table.plan_id, table.candidate_id, table.kind), + index("learning_governance_action_claim_idx").on(table.plan_id, table.state, table.sequence), + check("learning_governance_action_sequence_check", sql`${table.sequence} >= 0`), + check("learning_governance_action_version_check", sql`${table.version} >= 0`), + check("learning_governance_action_candidate_check", sql`length(trim(${table.candidate_id})) > 0`), + check("learning_governance_action_payload_json_check", sql`json_valid(${table.payload_json})`), + check("learning_governance_action_kind_check", sql`${table.kind} IN ('document_stage', 'memory_inbox')`), + check( + "learning_governance_action_payload_fingerprint_check", + sql`length(${table.payload_fingerprint}) = 64 AND ${table.payload_fingerprint} NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "learning_governance_action_result_hash_check", + sql`${table.result_hash} IS NULL OR (length(${table.result_hash}) = 64 AND ${table.result_hash} NOT GLOB '*[^0-9a-f]*')`, + ), + check( + "learning_governance_action_result_fingerprint_check", + sql`${table.result_fingerprint} IS NULL OR (length(${table.result_fingerprint}) = 64 AND ${table.result_fingerprint} NOT GLOB '*[^0-9a-f]*')`, + ), + check( + "learning_governance_action_state_check", + sql`${table.state} IN ('prepared', 'running', 'settled', 'recovery_required')`, + ), + check( + "learning_governance_action_lifecycle_check", + sql`(${table.state} = 'prepared' AND ${table.owner} IS NULL AND ${table.lease_expires_at} IS NULL AND ${table.result_ref} IS NULL AND ${table.result_hash} IS NULL AND ${table.result_fingerprint} IS NULL AND ${table.error_code} IS NULL AND ${table.settled_at} IS NULL) OR (${table.state} = 'running' AND length(trim(${table.owner})) > 0 AND ${table.lease_expires_at} IS NOT NULL AND ${table.result_ref} IS NULL AND ${table.result_hash} IS NULL AND ${table.result_fingerprint} IS NULL AND ${table.error_code} IS NULL AND ${table.settled_at} IS NULL) OR (${table.state} = 'settled' AND ${table.owner} IS NULL AND ${table.lease_expires_at} IS NULL AND length(trim(${table.result_ref})) > 0 AND ${table.result_hash} IS NOT NULL AND ${table.result_fingerprint} IS NOT NULL AND ${table.error_code} IS NULL AND ${table.settled_at} IS NOT NULL) OR (${table.state} = 'recovery_required' AND ${table.owner} IS NULL AND ${table.lease_expires_at} IS NULL AND length(trim(${table.error_code})) > 0 AND ${table.settled_at} IS NOT NULL)`, + ), + ], +) + +export const LearningGovernanceCompensationTable = sqliteTable( + "learning_governance_compensation", + { + compensation_id: text().primaryKey(), + plan_id: text() + .notNull() + .references(() => LearningGovernancePlanTable.plan_id), + action_id: text() + .notNull() + .references(() => LearningGovernanceActionTable.action_id), + sequence: integer().notNull(), + kind: text().$type<"document_quarantine" | "memory_inbox_revoke">().notNull(), + source_payload_fingerprint: text().notNull(), + state: text().$type<"prepared" | "running" | "settled" | "recovery_required">().notNull(), + owner: text(), + lease_expires_at: integer(), + version: integer().notNull(), + result_ref: text(), + result_hash: text(), + result_fingerprint: text(), + error_code: text(), + error_detail: text(), + created_at: integer().notNull(), + settled_at: integer(), + updated_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("learning_governance_compensation_action_idx").on(table.action_id), + uniqueIndex("learning_governance_compensation_plan_sequence_idx").on(table.plan_id, table.sequence), + index("learning_governance_compensation_claim_idx").on(table.state, table.created_at), + check("learning_governance_compensation_sequence_check", sql`${table.sequence} >= 0`), + check("learning_governance_compensation_version_check", sql`${table.version} >= 0`), + check( + "learning_governance_compensation_kind_check", + sql`${table.kind} IN ('document_quarantine', 'memory_inbox_revoke')`, + ), + check( + "learning_governance_compensation_source_fingerprint_check", + sql`length(${table.source_payload_fingerprint}) = 64 AND ${table.source_payload_fingerprint} NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "learning_governance_compensation_result_hash_check", + sql`${table.result_hash} IS NULL OR (length(${table.result_hash}) = 64 AND ${table.result_hash} NOT GLOB '*[^0-9a-f]*')`, + ), + check( + "learning_governance_compensation_result_fingerprint_check", + sql`${table.result_fingerprint} IS NULL OR (length(${table.result_fingerprint}) = 64 AND ${table.result_fingerprint} NOT GLOB '*[^0-9a-f]*')`, + ), + check( + "learning_governance_compensation_state_check", + sql`${table.state} IN ('prepared', 'running', 'settled', 'recovery_required')`, + ), + check( + "learning_governance_compensation_lifecycle_check", + sql`(${table.state} = 'prepared' AND ${table.owner} IS NULL AND ${table.lease_expires_at} IS NULL AND ${table.result_ref} IS NULL AND ${table.result_hash} IS NULL AND ${table.result_fingerprint} IS NULL AND ${table.error_code} IS NULL AND ${table.settled_at} IS NULL) OR (${table.state} = 'running' AND length(trim(${table.owner})) > 0 AND ${table.lease_expires_at} IS NOT NULL AND ${table.result_ref} IS NULL AND ${table.result_hash} IS NULL AND ${table.result_fingerprint} IS NULL AND ${table.error_code} IS NULL AND ${table.settled_at} IS NULL) OR (${table.state} = 'settled' AND ${table.owner} IS NULL AND ${table.lease_expires_at} IS NULL AND length(trim(${table.result_ref})) > 0 AND ${table.result_hash} IS NOT NULL AND ${table.result_fingerprint} IS NOT NULL AND ${table.error_code} IS NULL AND ${table.settled_at} IS NOT NULL) OR (${table.state} = 'recovery_required' AND ${table.owner} IS NULL AND ${table.lease_expires_at} IS NULL AND length(trim(${table.error_code})) > 0 AND ${table.settled_at} IS NOT NULL)`, + ), + ], +) diff --git a/packages/core/src/deepagent/learning-job.sql.ts b/packages/core/src/deepagent/learning-job.sql.ts new file mode 100644 index 000000000..963147a32 --- /dev/null +++ b/packages/core/src/deepagent/learning-job.sql.ts @@ -0,0 +1,105 @@ +import { sql } from "drizzle-orm" +import { check, index, integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { ProjectTable } from "../project/sql" +import { SessionTable } from "../session/sql" + +export const LearningJobTable = sqliteTable( + "learning_job", + { + job_id: text().primaryKey(), + project_id: text() + .notNull() + .references(() => ProjectTable.id), + session_id: text().references(() => SessionTable.id), + run_id: text(), + trigger: text().$type<"idle" | "pause" | "project_switch" | "session_finalization">().notNull(), + dedupe_key: text().notNull(), + candidate_input_ref: text().notNull(), + policy: text().$type<"auto_merge_safe_project" | "manual_review">().notNull(), + max_attempts: integer().notNull(), + admission_fingerprint: text().notNull(), + state: text() + .$type< + "queued" | "running" | "reviewing" | "governance" | "completed" | "failed" | "cancelled" | "recovery_required" + >() + .notNull(), + attempts: integer().notNull(), + owner: text(), + lease_expires_at: integer(), + version: integer().notNull(), + side_effect_state: text().$type<"not_started" | "started" | "settled" | "unknown">().notNull(), + side_effect_kind: text().$type<"extraction" | "reviewer" | "governance">(), + expected_result_ref: text(), + review_job_id: text(), + result_ref: text(), + error_code: text(), + error_detail: text(), + settlement_fingerprint: text(), + next_attempt_at: integer().notNull(), + created_at: integer().notNull(), + started_at: integer(), + settled_at: integer(), + updated_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("learning_job_dedupe_idx").on(table.dedupe_key), + index("learning_job_due_idx").on(table.state, table.next_attempt_at, table.created_at), + index("learning_job_project_created_idx").on(table.project_id, table.created_at), + index("learning_job_owner_lease_idx") + .on(table.owner, table.lease_expires_at) + .where(sql`${table.owner} IS NOT NULL`), + check( + "learning_job_trigger_check", + sql`${table.trigger} IN ('idle', 'pause', 'project_switch', 'session_finalization')`, + ), + check("learning_job_policy_check", sql`${table.policy} IN ('auto_merge_safe_project', 'manual_review')`), + check( + "learning_job_state_check", + sql`${table.state} IN ('queued', 'running', 'reviewing', 'governance', 'completed', 'failed', 'cancelled', 'recovery_required')`, + ), + check("learning_job_attempts_check", sql`${table.attempts} >= 0`), + check("learning_job_max_attempts_check", sql`${table.max_attempts} > 0`), + check("learning_job_version_check", sql`${table.version} >= 0`), + check( + "learning_job_admission_fingerprint_check", + sql`length(${table.admission_fingerprint}) = 64 AND ${table.admission_fingerprint} NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "learning_job_side_effect_check", + sql`(${table.side_effect_state} = 'not_started' AND ${table.side_effect_kind} IS NULL) OR (${table.side_effect_state} IN ('started', 'settled', 'unknown') AND ${table.side_effect_kind} IS NOT NULL)`, + ), + check( + "learning_job_expected_result_check", + sql`(${table.side_effect_state} = 'not_started' AND ${table.expected_result_ref} IS NULL) OR (${table.side_effect_kind} IN ('extraction', 'reviewer') AND ${table.side_effect_state} IN ('started', 'settled') AND length(trim(${table.expected_result_ref})) > 0) OR (${table.side_effect_kind} = 'governance' AND ${table.expected_result_ref} IS NULL) OR (${table.side_effect_state} = 'unknown' AND ${table.expected_result_ref} IS NULL)`, + ), + check( + "learning_job_settled_result_check", + sql`${table.side_effect_state} <> 'settled' OR (length(trim(${table.result_ref})) > 0 AND (${table.expected_result_ref} IS NULL OR ${table.side_effect_kind} = 'reviewer' OR ${table.result_ref} = ${table.expected_result_ref}))`, + ), + check( + "learning_job_active_phase_kind_check", + sql`${table.state} NOT IN ('running', 'reviewing', 'governance') OR ${table.side_effect_state} = 'not_started' OR (${table.state} = 'running' AND ${table.side_effect_kind} = 'extraction') OR (${table.state} = 'reviewing' AND ${table.side_effect_kind} = 'reviewer') OR (${table.state} = 'governance' AND ${table.side_effect_kind} = 'governance')`, + ), + check( + "learning_job_ownership_check", + sql`(${table.state} = 'queued' AND ${table.owner} IS NULL AND ${table.lease_expires_at} IS NULL AND ${table.side_effect_state} = 'not_started') OR (${table.state} IN ('running', 'reviewing', 'governance') AND ${table.started_at} IS NOT NULL AND ${table.settled_at} IS NULL AND ((length(trim(${table.owner})) > 0 AND ${table.lease_expires_at} IS NOT NULL) OR (${table.state} IN ('reviewing', 'governance') AND ${table.owner} IS NULL AND ${table.lease_expires_at} IS NULL AND ${table.side_effect_state} = 'not_started'))) OR (${table.state} IN ('completed', 'failed', 'cancelled', 'recovery_required') AND ${table.owner} IS NULL AND ${table.lease_expires_at} IS NULL AND ${table.settled_at} IS NOT NULL)`, + ), + check( + "learning_job_recovery_check", + sql`${table.state} <> 'recovery_required' OR (${table.side_effect_state} <> 'not_started' AND ${table.error_code} IS NOT NULL)`, + ), + check( + "learning_job_completed_check", + sql`${table.state} <> 'completed' OR (${table.side_effect_state} = 'settled' AND length(trim(${table.result_ref})) > 0)`, + ), + check("learning_job_failed_check", sql`${table.state} <> 'failed' OR length(trim(${table.error_code})) > 0`), + check( + "learning_job_terminal_side_effect_check", + sql`${table.state} NOT IN ('completed', 'failed', 'cancelled') OR ${table.side_effect_state} IN ('not_started', 'settled')`, + ), + check( + "learning_job_settlement_fingerprint_check", + sql`${table.settlement_fingerprint} IS NULL OR (length(${table.settlement_fingerprint}) = 64 AND ${table.settlement_fingerprint} NOT GLOB '*[^0-9a-f]*')`, + ), + ], +) diff --git a/packages/core/src/deepagent/learning-lifecycle-trigger.sql.ts b/packages/core/src/deepagent/learning-lifecycle-trigger.sql.ts new file mode 100644 index 000000000..7850b64d6 --- /dev/null +++ b/packages/core/src/deepagent/learning-lifecycle-trigger.sql.ts @@ -0,0 +1,61 @@ +import { sql } from "drizzle-orm" +import { check, index, integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" + +export const LearningLifecycleTriggerTable = sqliteTable( + "learning_lifecycle_trigger_receipt", + { + receipt_id: text().primaryKey(), + trigger: text().$type<"idle" | "pause" | "project_switch">().notNull(), + boundary_key: text().notNull(), + session_id: text().notNull(), + run_id: text().notNull(), + source_admission_hash: text().notNull(), + source_terminal_hash: text().notNull(), + artifact_path: text().notNull(), + artifact_hash: text().notNull(), + artifact_json: text().notNull(), + admission_fingerprint: text().notNull(), + admission_json: text().notNull(), + state: text().$type<"prepared" | "admitted">().notNull(), + error_detail: text(), + created_at: integer().notNull(), + settled_at: integer(), + updated_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("learning_lifecycle_trigger_identity_idx").on(table.trigger, table.session_id, table.run_id), + index("learning_lifecycle_trigger_pending_idx") + .on(table.created_at, table.receipt_id) + .where(sql`${table.state} = 'prepared'`), + check("learning_lifecycle_trigger_kind_check", sql`${table.trigger} IN ('idle', 'pause', 'project_switch')`), + check("learning_lifecycle_trigger_state_check", sql`${table.state} IN ('prepared', 'admitted')`), + check( + "learning_lifecycle_trigger_source_admission_hash_check", + sql`length(${table.source_admission_hash}) = 64 AND ${table.source_admission_hash} NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "learning_lifecycle_trigger_source_terminal_hash_check", + sql`length(${table.source_terminal_hash}) = 64 AND ${table.source_terminal_hash} NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "learning_lifecycle_trigger_artifact_hash_check", + sql`length(${table.artifact_hash}) = 64 AND ${table.artifact_hash} NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "learning_lifecycle_trigger_admission_fingerprint_check", + sql`length(${table.admission_fingerprint}) = 64 AND ${table.admission_fingerprint} NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "learning_lifecycle_trigger_admission_json_check", + sql`json_valid(${table.admission_json}) AND json_type(${table.admission_json}) = 'object'`, + ), + check( + "learning_lifecycle_trigger_artifact_json_check", + sql`json_valid(${table.artifact_json}) AND json_type(${table.artifact_json}) = 'object'`, + ), + check( + "learning_lifecycle_trigger_settlement_check", + sql`(${table.state} = 'prepared' AND ${table.settled_at} IS NULL) OR (${table.state} = 'admitted' AND ${table.settled_at} IS NOT NULL AND ${table.error_detail} IS NULL)`, + ), + ], +) diff --git a/packages/core/src/deepagent/learning-reviewer-attempt.sql.ts b/packages/core/src/deepagent/learning-reviewer-attempt.sql.ts new file mode 100644 index 000000000..965c3c974 --- /dev/null +++ b/packages/core/src/deepagent/learning-reviewer-attempt.sql.ts @@ -0,0 +1,61 @@ +import { sql } from "drizzle-orm" +import { check, index, integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { LearningJobTable } from "./learning-job.sql" + +export const LearningReviewerAttemptTable = sqliteTable( + "learning_reviewer_attempt", + { + attempt_id: text().primaryKey(), + job_id: text() + .notNull() + .references(() => LearningJobTable.job_id), + state: text().$type<"prepared" | "dispatching" | "settled" | "failed" | "recovery_required">().notNull(), + version: integer().notNull(), + owner: text(), + review_session_id: text().notNull(), + request_ref: text().notNull(), + request_hash: text().notNull(), + source_candidate_ids_json: text().notNull(), + source_candidate_set_hash: text().notNull(), + provider_id: text().notNull(), + model_id: text().notNull(), + policy_hash: text().notNull(), + response_ref: text(), + response_hash: text(), + verdict: text().$type<"approve" | "reject" | "manual_review">(), + selected_candidate_ids_json: text(), + selected_subset_hash: text(), + error_code: text(), + error_detail: text(), + created_at: integer().notNull(), + dispatched_at: integer(), + settled_at: integer(), + updated_at: integer().notNull(), + }, + (table) => [ + uniqueIndex("learning_reviewer_attempt_job_idx").on(table.job_id), + uniqueIndex("learning_reviewer_attempt_session_idx").on(table.review_session_id), + index("learning_reviewer_attempt_state_idx").on(table.state, table.updated_at), + check( + "learning_reviewer_attempt_state_check", + sql`${table.state} IN ('prepared', 'dispatching', 'settled', 'failed', 'recovery_required')`, + ), + check("learning_reviewer_attempt_version_check", sql`${table.version} >= 0`), + check( + "learning_reviewer_attempt_hash_check", + sql`length(${table.request_hash}) = 64 AND ${table.request_hash} NOT GLOB '*[^0-9a-f]*' AND length(${table.source_candidate_set_hash}) = 64 AND ${table.source_candidate_set_hash} NOT GLOB '*[^0-9a-f]*' AND length(${table.policy_hash}) = 64 AND ${table.policy_hash} NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "learning_reviewer_attempt_terminal_check", + sql`(${table.state} IN ('prepared', 'dispatching') AND ${table.settled_at} IS NULL) OR (${table.state} IN ('settled', 'failed', 'recovery_required') AND ${table.settled_at} IS NOT NULL)`, + ), + check( + "learning_reviewer_attempt_response_check", + sql`(${table.state} <> 'settled' AND ${table.response_ref} IS NULL AND ${table.response_hash} IS NULL AND ${table.verdict} IS NULL AND ${table.selected_candidate_ids_json} IS NULL AND ${table.selected_subset_hash} IS NULL) OR (${table.state} = 'settled' AND length(trim(${table.response_ref})) > 0 AND length(${table.response_hash}) = 64 AND ${table.response_hash} NOT GLOB '*[^0-9a-f]*' AND ${table.verdict} IN ('approve', 'reject', 'manual_review') AND length(trim(${table.selected_candidate_ids_json})) > 0 AND length(${table.selected_subset_hash}) = 64 AND ${table.selected_subset_hash} NOT GLOB '*[^0-9a-f]*')`, + ), + check( + "learning_reviewer_attempt_error_check", + sql`${table.state} NOT IN ('failed', 'recovery_required') OR length(trim(${table.error_code})) > 0`, + ), + ], +) diff --git a/packages/core/src/deepagent/released-snapshot.sql.ts b/packages/core/src/deepagent/released-snapshot.sql.ts new file mode 100644 index 000000000..ba015d0a1 --- /dev/null +++ b/packages/core/src/deepagent/released-snapshot.sql.ts @@ -0,0 +1,188 @@ +import { sql } from "drizzle-orm" +import { check, foreignKey, index, integer, primaryKey, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { ProjectScopeIdentityTable, SecurityNamespaceTable } from "../context-federation/sql" + +export const ReleasedKnowledgeEvaluationTable = sqliteTable( + "released_knowledge_evaluation", + { + evaluation_id: text().primaryKey(), + security_namespace_id: text() + .notNull() + .references(() => SecurityNamespaceTable.id), + project_scope_key: text().notNull(), + matrix_hash: text().notNull(), + matrix_json: text().notNull(), + document_manifest_json: text().notNull(), + baseline_ref: text().notNull(), + repetitions: integer().notNull(), + evaluator_type: text().$type<"human" | "agent" | "system">().notNull(), + evaluator_id: text().notNull(), + created_at: integer().notNull(), + }, + (table) => [ + foreignKey({ + columns: [table.security_namespace_id, table.project_scope_key], + foreignColumns: [ProjectScopeIdentityTable.security_namespace_id, ProjectScopeIdentityTable.project_scope_key], + }), + uniqueIndex("released_knowledge_evaluation_scope_identity_idx").on( + table.security_namespace_id, + table.project_scope_key, + table.evaluation_id, + ), + index("released_knowledge_evaluation_matrix_idx").on( + table.security_namespace_id, + table.project_scope_key, + table.matrix_hash, + ), + check( + "released_knowledge_evaluation_matrix_hash_check", + sql`length(${table.matrix_hash}) = 64 AND ${table.matrix_hash} NOT GLOB '*[^0-9a-f]*'`, + ), + check("released_knowledge_evaluation_matrix_json_check", sql`json_valid(${table.matrix_json})`), + check( + "released_knowledge_evaluation_document_manifest_json_check", + sql`json_valid(${table.document_manifest_json}) AND json_type(${table.document_manifest_json}) = 'array'`, + ), + check("released_knowledge_evaluation_repetitions_check", sql`${table.repetitions} > 0`), + check( + "released_knowledge_evaluation_actor_type_check", + sql`${table.evaluator_type} IN ('human', 'agent', 'system')`, + ), + ], +) + +export const ReleasedKnowledgeSnapshotTable = sqliteTable( + "released_knowledge_snapshot", + { + snapshot_id: text().primaryKey(), + security_namespace_id: text() + .notNull() + .references(() => SecurityNamespaceTable.id), + project_scope_key: text().notNull(), + legacy_project_id: text().notNull(), + parent_snapshot_id: text(), + evaluation_id: text().notNull(), + release_kind: text().$type<"legacy_baseline" | "evaluated" | "rollback">().notNull(), + document_count: integer().notNull(), + published_generation: integer().notNull(), + verdict: text().$type<"passed" | "failed">().notNull(), + failure_reason: text(), + actor_type: text().$type<"human" | "agent" | "system">().notNull(), + actor_id: text().notNull(), + created_at: integer().notNull(), + finalized_at: integer(), + }, + (table) => [ + foreignKey({ + columns: [table.security_namespace_id, table.project_scope_key], + foreignColumns: [ProjectScopeIdentityTable.security_namespace_id, ProjectScopeIdentityTable.project_scope_key], + }), + foreignKey({ + columns: [table.security_namespace_id, table.project_scope_key, table.parent_snapshot_id], + foreignColumns: [table.security_namespace_id, table.project_scope_key, table.snapshot_id], + }), + foreignKey({ + columns: [table.security_namespace_id, table.project_scope_key, table.evaluation_id], + foreignColumns: [ + ReleasedKnowledgeEvaluationTable.security_namespace_id, + ReleasedKnowledgeEvaluationTable.project_scope_key, + ReleasedKnowledgeEvaluationTable.evaluation_id, + ], + }), + uniqueIndex("released_knowledge_snapshot_scope_identity_idx").on( + table.security_namespace_id, + table.project_scope_key, + table.snapshot_id, + ), + index("released_knowledge_snapshot_parent_idx").on(table.parent_snapshot_id), + check( + "released_knowledge_snapshot_release_kind_check", + sql`${table.release_kind} IN ('legacy_baseline', 'evaluated', 'rollback')`, + ), + check("released_knowledge_snapshot_document_count_check", sql`${table.document_count} >= 0`), + check( + "released_knowledge_snapshot_published_generation_check", + sql`(${table.verdict} = 'passed' AND ${table.published_generation} > 0) OR (${table.verdict} = 'failed' AND ${table.published_generation} >= 0)`, + ), + check("released_knowledge_snapshot_verdict_check", sql`${table.verdict} IN ('passed', 'failed')`), + check( + "released_knowledge_snapshot_failure_reason_check", + sql`(${table.verdict} = 'passed' AND ${table.failure_reason} IS NULL) OR (${table.verdict} = 'failed' AND length(trim(${table.failure_reason})) > 0)`, + ), + check( + "released_knowledge_snapshot_release_chain_check", + sql`(${table.release_kind} = 'legacy_baseline' AND ${table.parent_snapshot_id} IS NULL AND ${table.verdict} = 'passed') OR (${table.release_kind} <> 'legacy_baseline' AND ${table.parent_snapshot_id} IS NOT NULL)`, + ), + check( + "released_knowledge_snapshot_evaluated_membership_check", + sql`${table.release_kind} <> 'evaluated' OR ${table.verdict} = 'failed' OR ${table.document_count} > 0`, + ), + check("released_knowledge_snapshot_actor_type_check", sql`${table.actor_type} IN ('human', 'agent', 'system')`), + ], +) + +export const ReleasedKnowledgeSnapshotDocumentTable = sqliteTable( + "released_knowledge_snapshot_document", + { + snapshot_id: text() + .notNull() + .references(() => ReleasedKnowledgeSnapshotTable.snapshot_id, { onDelete: "cascade" }), + ordinal: integer().notNull(), + source_store: text().$type<"user_global" | "project">().notNull(), + doc_id: text().notNull(), + doc_version: integer().notNull(), + doc_hash: text().notNull(), + doc_type: text().notNull(), + doc_scope: text().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.snapshot_id, table.source_store, table.doc_id] }), + uniqueIndex("released_knowledge_snapshot_document_ordinal_idx").on(table.snapshot_id, table.ordinal), + check("released_knowledge_snapshot_document_ordinal_check", sql`${table.ordinal} >= 0`), + check( + "released_knowledge_snapshot_document_source_store_check", + sql`${table.source_store} IN ('user_global', 'project')`, + ), + check("released_knowledge_snapshot_document_version_check", sql`${table.doc_version} > 0`), + check( + "released_knowledge_snapshot_document_hash_check", + sql`length(${table.doc_hash}) = 71 AND substr(${table.doc_hash}, 1, 7) = 'sha256:' AND substr(${table.doc_hash}, 8) NOT GLOB '*[^0-9a-f]*'`, + ), + check( + "released_knowledge_snapshot_document_type_check", + sql`${table.doc_type} IN ('knowledge', 'strategy', 'methodology', 'memory', 'skill')`, + ), + ], +) + +export const ReleasedKnowledgeSnapshotHeadTable = sqliteTable( + "released_knowledge_snapshot_head", + { + security_namespace_id: text() + .notNull() + .references(() => SecurityNamespaceTable.id), + project_scope_key: text().notNull(), + snapshot_id: text(), + generation: integer().notNull(), + updated_at: integer().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.security_namespace_id, table.project_scope_key] }), + foreignKey({ + columns: [table.security_namespace_id, table.project_scope_key], + foreignColumns: [ProjectScopeIdentityTable.security_namespace_id, ProjectScopeIdentityTable.project_scope_key], + }), + foreignKey({ + columns: [table.security_namespace_id, table.project_scope_key, table.snapshot_id], + foreignColumns: [ + ReleasedKnowledgeSnapshotTable.security_namespace_id, + ReleasedKnowledgeSnapshotTable.project_scope_key, + ReleasedKnowledgeSnapshotTable.snapshot_id, + ], + }), + check( + "released_knowledge_snapshot_head_generation_check", + sql`(${table.snapshot_id} IS NULL AND ${table.generation} = 0) OR (${table.snapshot_id} IS NOT NULL AND ${table.generation} > 0)`, + ), + ], +) diff --git a/packages/core/src/permission/sql.ts b/packages/core/src/permission/sql.ts index c395555d7..e05dbdf07 100644 --- a/packages/core/src/permission/sql.ts +++ b/packages/core/src/permission/sql.ts @@ -1,4 +1,4 @@ -import { sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { integer, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" import { Timestamps } from "../database/schema.sql" import { ProjectV2 } from "../project" import { ProjectTable } from "../project/sql" @@ -18,3 +18,12 @@ export const PermissionTable = sqliteTable( }, (table) => [uniqueIndex("permission_project_action_resource_idx").on(table.project_id, table.action, table.resource)], ) + +export const PermissionSavedEpochTable = sqliteTable("permission_saved_epoch", { + project_id: text() + .$type() + .primaryKey() + .references(() => ProjectTable.id, { onDelete: "cascade" }), + epoch: integer().notNull(), + updated_at: integer().notNull(), +}) diff --git a/packages/core/src/session/runner/v2-provider-turn.sql.ts b/packages/core/src/session/runner/v2-provider-turn.sql.ts new file mode 100644 index 000000000..759450963 --- /dev/null +++ b/packages/core/src/session/runner/v2-provider-turn.sql.ts @@ -0,0 +1,105 @@ +import { index, integer, primaryKey, sqliteTable, text, uniqueIndex } from "drizzle-orm/sqlite-core" +import { SessionProviderOwnerLeaseTable } from "../../context-federation/session-sql" +import { SessionTable } from "../sql" +import type { SessionSchema } from "../schema" +import type { PreparedProviderTurn } from "./prepared-provider-turn" + +export const V2ProviderTurnReceiptTable = sqliteTable( + "session_v2_provider_turn_receipt", + { + receipt_id: text().primaryKey(), + session_id: text() + .$type() + .notNull() + .references(() => SessionTable.id, { onDelete: "cascade" }), + request_ordinal: integer().notNull(), + user_message_id: text().notNull(), + history_prompt_epoch: integer().notNull(), + history_source_end_message_id: text(), + request_input_hash: text().notNull(), + provider_id: text().notNull(), + model_id: text().notNull(), + protocol: text().notNull(), + owner_mode: text().$type<"shadow_v2" | "v2">().notNull(), + owner_token: text() + .notNull() + .references(() => SessionProviderOwnerLeaseTable.owner_token), + state: text() + .$type<"preparing" | "dispatching" | "streaming" | "settled" | "failed" | "indeterminate_after_crash">() + .notNull(), + prepared_turn_hash: text(), + wire_request_hash: text(), + prepared_turn: text({ mode: "json" }).$type(), + outcome_hash: text(), + outcome_artifact: text({ mode: "json" }).$type(), + error_code: text(), + created_at: integer().notNull(), + dispatching_at: integer(), + first_event_at: integer(), + terminal_at: integer(), + }, + (table) => [ + uniqueIndex("session_v2_provider_turn_receipt_ordinal_idx").on(table.session_id, table.request_ordinal), + index("session_v2_provider_turn_receipt_input_idx").on( + table.session_id, + table.user_message_id, + table.history_prompt_epoch, + table.request_input_hash, + ), + index("session_v2_provider_turn_receipt_owner_state_idx").on(table.owner_token, table.state, table.created_at), + ], +) + +export const V2ProviderParityBaselineTable = sqliteTable( + "session_v2_provider_parity_baseline", + { + campaign_id: text().notNull(), + case_name: text().notNull(), + legacy_receipt_id: text().notNull().unique(), + state: text().$type<"prepared" | "settled">().notNull(), + prepared_turn: text({ mode: "json" }).$type().notNull(), + outcome_hash: text(), + outcome_artifact: text({ mode: "json" }).$type(), + legacy_response_fingerprint: text(), + evidence: text({ mode: "json" }).$type().notNull(), + receipt_hash: text().notNull(), + created_at: integer().notNull(), + settled_at: integer(), + }, + (table) => [ + primaryKey({ columns: [table.campaign_id, table.case_name] }), + uniqueIndex("session_v2_provider_parity_baseline_hash_idx").on(table.receipt_hash), + index("session_v2_provider_parity_baseline_campaign_idx").on(table.campaign_id, table.state), + ], +) + +export const V2ProviderParityReceiptTable = sqliteTable( + "session_v2_provider_parity_receipt", + { + campaign_id: text().notNull(), + case_name: text().notNull(), + legacy_receipt_id: text().notNull(), + core_v2_receipt_id: text() + .notNull() + .references(() => V2ProviderTurnReceiptTable.receipt_id), + legacy_request_hash: text().notNull(), + core_v2_request_hash: text().notNull(), + legacy_outcome_hash: text().notNull(), + core_v2_outcome_hash: text().notNull(), + legacy_prepared_turn: text({ mode: "json" }).$type().notNull(), + core_v2_prepared_turn: text({ mode: "json" }).$type().notNull(), + diff_artifact: text({ mode: "json" }).$type().notNull(), + allowlist_version: text().notNull(), + allowlisted_differences: text({ mode: "json" }).$type().notNull(), + disallowed_differences: text({ mode: "json" }).$type().notNull(), + evidence: text({ mode: "json" }).$type().notNull(), + verified: integer({ mode: "boolean" }).notNull(), + receipt_hash: text().notNull(), + created_at: integer().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.campaign_id, table.case_name] }), + uniqueIndex("session_v2_provider_parity_receipt_hash_idx").on(table.receipt_hash), + index("session_v2_provider_parity_receipt_campaign_idx").on(table.campaign_id, table.verified), + ], +) diff --git a/packages/core/src/session/sql.ts b/packages/core/src/session/sql.ts index 105eec510..8a3bb575f 100644 --- a/packages/core/src/session/sql.ts +++ b/packages/core/src/session/sql.ts @@ -501,6 +501,16 @@ export const SessionContextEpochTable = sqliteTable("session_context_epoch", { revision: integer().notNull().default(0), }) +export type TaskStructuredOutputReceipt = { + readonly attempt: number +} & ( + | { readonly transport: "structured" | "text_fallback" } + | { + readonly transport: "degraded_text" + readonly reason: "structured_output_missing" | "structured_output_invalid" + } +) + export const TaskRunTable = sqliteTable( "task_run", { @@ -541,6 +551,7 @@ export const TaskRunTable = sqliteTable( lease_expires_at: integer(), raw_result_message_id: text().$type(), structured_result_message_id: text().$type(), + structured_output_receipt: text({ mode: "json" }).$type(), output: text(), error: text({ mode: "json" }).$type<{ code: string; message: string; data?: Record }>(), time_created: integer().notNull(), @@ -642,6 +653,79 @@ export const TaskRunTable = sqliteTable( ], ) +export const TaskStructuredOutputEvidenceTable = sqliteTable( + "task_structured_output_evidence", + { + run_id: text() + .primaryKey() + .references(() => TaskRunTable.run_id, { onDelete: "cascade" }), + child_session_id: text().$type().notNull(), + owner_token: text().notNull(), + claim_generation: integer().notNull(), + expected_version: integer().notNull(), + terminal_state: text().$type<"completed" | "failed">().notNull(), + attempts: integer().notNull(), + contract_json: text().notNull(), + raw_result_message_id: text().$type().notNull(), + raw_message_json: text().notNull(), + raw_parts_json: text().notNull(), + result_message_id: text().$type(), + result_message_json: text(), + result_parts_json: text(), + output: text(), + structured_output_receipt: text({ mode: "json" }).$type(), + failure_code: text(), + created_at: integer().notNull(), + }, + (table) => [ + index("task_structured_output_evidence_raw_message_idx").on(table.raw_result_message_id), + index("task_structured_output_evidence_result_message_idx").on(table.result_message_id), + ], +) + +export const TaskStructuredOutputEvidencePartTable = sqliteTable( + "task_structured_output_evidence_part", + { + run_id: text() + .notNull() + .references(() => TaskStructuredOutputEvidenceTable.run_id, { onDelete: "cascade" }), + role: text().$type<"raw" | "result">().notNull(), + ordinal: integer().notNull(), + part_id: text().$type().notNull(), + message_id: text().$type().notNull(), + session_id: text().$type().notNull(), + part_json: text().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.run_id, table.role, table.part_id] }), + uniqueIndex("task_structured_output_evidence_part_ordinal_idx").on(table.run_id, table.role, table.ordinal), + index("task_structured_output_evidence_part_part_idx").on(table.part_id), + ], +) + +export const TaskStructuredFinalizerResponseTable = sqliteTable( + "task_structured_finalizer_response", + { + run_id: text() + .notNull() + .references(() => TaskRunTable.run_id, { onDelete: "cascade" }), + attempt: integer().notNull(), + child_session_id: text().$type().notNull(), + owner_token: text().notNull(), + claim_generation: integer().notNull(), + expected_version: integer().notNull(), + source_message_id: text().$type().notNull(), + request_message_id: text().$type().notNull(), + response_message_id: text().$type().notNull(), + response_message_json: text().notNull(), + created_at: integer().notNull(), + }, + (table) => [ + primaryKey({ columns: [table.run_id, table.attempt] }), + uniqueIndex("task_structured_finalizer_response_message_idx").on(table.response_message_id), + ], +) + export const TaskAdmissionTable = sqliteTable( "task_admission", { diff --git a/packages/core/test/database-migration.test.ts b/packages/core/test/database-migration.test.ts index ebaed90d0..55349407a 100644 --- a/packages/core/test/database-migration.test.ts +++ b/packages/core/test/database-migration.test.ts @@ -33,6 +33,18 @@ import filePartArtifactMigration from "@deepagent-code/core/database/migration/2 import eventSnapshotChunksMigration from "@deepagent-code/core/database/migration/20260813131000_event_snapshot_chunks" import eventImmutableMigration from "@deepagent-code/core/database/migration/20260813135000_event_immutable" import eventSidecarCompactionMigration from "@deepagent-code/core/database/migration/20260813140000_event_sidecar_compaction" +import preparedProviderTurnReceiptMigration from "@deepagent-code/core/database/migration/20260811223000_prepared_provider_turn_receipt" +import learningAdmissionOutboxMigration from "@deepagent-code/core/database/migration/20260812005647_learning_admission_outbox" +import activityPermissionRouteFeedbackMigration from "@deepagent-code/core/database/migration/20260812014934_activity_permission_route_feedback" +import releasedKnowledgeSnapshotMigration from "@deepagent-code/core/database/migration/20260811185417_released_knowledge_snapshot_authority" +import providerAttemptPreDispatchTerminalMigration from "@deepagent-code/core/database/migration/20260812043000_provider_attempt_pre_dispatch_terminal" +import providerOwnerAuthorityMigration from "@deepagent-code/core/database/migration/20260812050000_provider_owner_authority" +import providerTurnIdentityAuthorityMigration from "@deepagent-code/core/database/migration/20260812053000_provider_turn_identity_authority" +import taskStructuredOutputReceiptSchemaMigration from "@deepagent-code/core/database/migration/20260812114412_task_structured_output_receipt_schema" +import taskStructuredOutputReceiptMigration from "@deepagent-code/core/database/migration/20260812114500_task_structured_output_receipt_authority" +import taskExecutionSpecAuthorityMigration from "@deepagent-code/core/database/migration/20260812210000_task_execution_spec_authority" +import taskStructuredOutputEvidenceAuthorityMigration from "@deepagent-code/core/database/migration/20260812220000_task_structured_output_evidence_authority" +import providerCrossStateRecoveryMigration from "@deepagent-code/core/database/migration/20260812061000_provider_cross_state_recovery" import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" import { Database } from "@deepagent-code/core/database/database" import { tmpdir } from "./fixture/tmpdir" @@ -43,6 +55,47 @@ const run = (effect: Effect.Effect) => ) const makeDb = EffectDrizzleSqlite.makeWithDefaults() +const receiptNamespace = "sec_receipt_migration" +const receiptProjectScope = "prjctx_receipt_migration" +const emptyReleasedRefsFingerprint = "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" +const preparedTurnHash = "b".repeat(64) +const systemStableHash = "c".repeat(64) +const systemVolatileHash = "d".repeat(64) +const wireRequestHash = "e".repeat(64) +const contextEligibility = { + requested: { + contextFederationShadow: false, + locationIndexesV2Shadow: false, + contextProjectionV2: false, + contextQueryToolsV2: false, + coreV2ExecutionOwner: false, + }, + enabled: { + contextFederationShadow: false, + locationIndexesV2Shadow: false, + contextProjectionV2: false, + contextQueryToolsV2: false, + coreV2ExecutionOwner: false, + }, + blocked: {}, + project: { + projectScopeKey: receiptProjectScope, + stage: "all", + bucket: 0, + selected: true, + killSwitch: false, + }, +} +const contextReadiness = { + revision: "context-readiness-migration", + state: "ready", + identityBound: true, + indexAvailable: true, + storageHealthy: true, + reasons: [], + observedAt: 0, + expiresAt: 100, +} describe("DatabaseMigration", () => { test("rejects duplicate migration IDs before changing the database", async () => { @@ -508,6 +561,7 @@ describe("DatabaseMigration", () => { Effect.gen(function* () { const db = yield* makeDb yield* DatabaseMigration.apply(db) + yield* seedReceiptScope(db) expect(yield* db.get(sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'session'`)).toEqual({ name: "session", @@ -583,14 +637,28 @@ describe("DatabaseMigration", () => { sql`SELECT name, dflt_value FROM pragma_table_info('session_tool_argument_receipt') WHERE name = 'validation_outcome'`, ), ).toEqual({ name: "validation_outcome", dflt_value: "'not_evaluated'" }) + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ( + 'owner-constraint-test', + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + 10000 + ) + `) yield* db.run(sql` INSERT INTO session_tool_request_receipt ( receipt_id, request_ordinal, session_id, user_message_id, provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, final_offered_tool_ids, call_ids, - request_state, created_at + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, owner_token, request_state, created_at ) VALUES ( 'receipt-constraint-test', 1, 'session-constraint-test', 'message-constraint-test', - 'provider-test', 'model-test', '[]', '[]', '[]', '[]', 'dispatched', 1 + 'provider-test', 'model-test', '[]', '[]', '[]', '[]', + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', ${emptyReleasedRefsFingerprint}, + 'owner-constraint-test', 'dispatched', 1 ) `) yield* db.run(sql` @@ -681,6 +749,36 @@ describe("DatabaseMigration", () => { ) }) + test("upgrades a pre-snapshot database without inventing a released head", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const migrationIndex = migrations.findIndex( + (migration) => migration.id === releasedKnowledgeSnapshotMigration.id, + ) + expect(migrationIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, migrationIndex)) + yield* seedReceiptScope(db) + + expect( + yield* db.get( + sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'released_knowledge_snapshot_head'`, + ), + ).toBeUndefined() + + yield* DatabaseMigration.applyOnly(db, [releasedKnowledgeSnapshotMigration]) + + expect( + yield* db.get( + sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'released_knowledge_snapshot_head'`, + ), + ).toEqual({ name: "released_knowledge_snapshot_head" }) + expect(yield* db.get(sql`SELECT count(*) AS count FROM released_knowledge_snapshot_head`)).toEqual({ count: 0 }) + expect(yield* db.run(sql`PRAGMA foreign_key_check`)).toEqual([]) + }), + ) + }) + test("legacy activity migration backfills V2 source identity and rejects mismatched legacy admissions", async () => { await run( Effect.gen(function* () { @@ -768,18 +866,136 @@ describe("DatabaseMigration", () => { Effect.gen(function* () { const db = yield* makeDb yield* DatabaseMigration.apply(db) + yield* seedReceiptScope(db) + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ( + 'owner-1', + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + 10000 + ), ( + 'owner-released', + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + 10000 + ) + `) + yield* db.run(sql` + UPDATE session_provider_owner_lease + SET released_at = CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + WHERE owner_token = 'owner-released' + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, provider_id, model_id, + registry_tool_ids, permission_filtered_tool_ids, final_offered_tool_ids, call_ids, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, request_state, created_at + ) VALUES ( + 'receipt-provider-unowned', 99, 'session-provider-lifecycle', 'message-provider-lifecycle', + 'provider-test', 'model-test', '[]', '[]', '[]', '[]', + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'prepared', 1 + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, provider_id, model_id, + registry_tool_ids, permission_filtered_tool_ids, final_offered_tool_ids, call_ids, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, owner_token, request_state, created_at + ) VALUES ( + 'receipt-provider-expired-owner', 98, 'session-provider-lifecycle', 'message-provider-lifecycle', + 'provider-test', 'model-test', '[]', '[]', '[]', '[]', + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'owner-released', 'prepared', 100 + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) yield* db.run(sql` INSERT INTO session_tool_request_receipt ( receipt_id, request_ordinal, session_id, user_message_id, provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, final_offered_tool_ids, call_ids, request_state, provider_state, prompt_epoch, prompt_window_id, effective_history_hash, - request_input_hash, owner_token, created_at + request_input_hash, owner_token, context_selection_id, context_eligibility, + context_readiness, context_activation, context_activation_fingerprint, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, created_at ) VALUES ( 'receipt-provider-lifecycle', 1, 'session-provider-lifecycle', 'message-provider-lifecycle', 'provider-test', 'model-test', '[]', '[]', '[]', '[]', 'prepared', 'preparing', - 0, 'window-0', 'history-0', 'input-hash', 'owner-1', 1 + 0, 'window-0', 'history-0', 'input-hash', 'owner-1', NULL, + ${JSON.stringify(contextEligibility)}, ${JSON.stringify(contextReadiness)}, + ${contextActivation(1)}, + lower(hex(zeroblob(32))), ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, provider_id, model_id, + registry_tool_ids, permission_filtered_tool_ids, final_offered_tool_ids, call_ids, + request_state, provider_state, prompt_epoch, prompt_window_id, effective_history_hash, + request_input_hash, owner_token, context_selection_id, context_eligibility, + context_readiness, context_activation, context_activation_fingerprint, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, created_at + ) VALUES ( + 'receipt-provider-forged-activation', 2, 'session-provider-lifecycle', 'message-provider-lifecycle', + 'provider-test', 'model-test', '[]', '[]', '[]', '[]', 'prepared', 'preparing', + 0, 'window-forged', 'history-forged', 'input-forged', 'owner-1', NULL, + ${JSON.stringify(contextEligibility)}, ${JSON.stringify(contextReadiness)}, + ${contextActivation(2).replace('"readinessAgeMs":2', '"readinessAgeMs":99')}, + lower(hex(zeroblob(32))), ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 2 ) `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = '[]', + released_knowledge_selected_refs_fingerprint = ${emptyReleasedRefsFingerprint} + WHERE receipt_id = 'receipt-provider-forged-activation' + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', final_request_hash = ${wireRequestHash}, + adapter_prepared_at = 2, provider_request_hash = ${wireRequestHash}, + prepared_turn_hash = ${preparedTurnHash}, + system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, + wire_request_hash = ${wireRequestHash}, + tool_definition_hash = ${preparedTurnHash} + WHERE receipt_id = 'receipt-provider-forged-activation' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) expect( Exit.isFailure( @@ -794,10 +1010,62 @@ describe("DatabaseMigration", () => { .pipe(Effect.exit), ), ).toBe(true) + for (const update of [ + sql`UPDATE session_tool_request_receipt SET final_request_hash = ${wireRequestHash} WHERE receipt_id = 'receipt-provider-lifecycle'`, + sql`UPDATE session_tool_request_receipt SET provider_request_hash = ${wireRequestHash} WHERE receipt_id = 'receipt-provider-lifecycle'`, + sql`UPDATE session_tool_request_receipt SET adapter_prepared_at = 2 WHERE receipt_id = 'receipt-provider-lifecycle'`, + sql`UPDATE session_tool_request_receipt SET prompt_cache_key = 'cache-key' WHERE receipt_id = 'receipt-provider-lifecycle'`, + sql`UPDATE session_tool_request_receipt SET tool_definition_hash = ${preparedTurnHash} WHERE receipt_id = 'receipt-provider-lifecycle'`, + sql`UPDATE session_tool_request_receipt SET final_offered_tool_ids = '["read"]' WHERE receipt_id = 'receipt-provider-lifecycle'`, + ]) { + expect(Exit.isFailure(yield* db.run(update).pipe(Effect.exit))).toBe(true) + } + for (const invalidSeal of [ + sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', final_request_hash = ${wireRequestHash}, + provider_request_hash = ${wireRequestHash}, adapter_prepared_at = 2, + prepared_turn_hash = ${preparedTurnHash}, system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, wire_request_hash = ${wireRequestHash}, + tool_definition_hash = '' + WHERE receipt_id = 'receipt-provider-lifecycle' + `, + sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', final_request_hash = ${wireRequestHash}, + provider_request_hash = ${wireRequestHash}, adapter_prepared_at = 2, + prepared_turn_hash = ${preparedTurnHash}, system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, wire_request_hash = ${wireRequestHash}, + tool_definition_hash = ${preparedTurnHash}, final_offered_tool_ids = '[1,1]' + WHERE receipt_id = 'receipt-provider-lifecycle' + `, + sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', final_request_hash = ${wireRequestHash}, + provider_request_hash = ${wireRequestHash}, adapter_prepared_at = 2, + prepared_turn_hash = ${preparedTurnHash}, system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, wire_request_hash = ${wireRequestHash}, + tool_definition_hash = ${preparedTurnHash}, final_offered_tool_ids = '["read", "read"]' + WHERE receipt_id = 'receipt-provider-lifecycle' + `, + ]) { + expect(Exit.isFailure(yield* db.run(invalidSeal).pipe(Effect.exit))).toBe(true) + } yield* db.run(sql` UPDATE session_tool_request_receipt - SET provider_state = 'prepared', final_request_hash = 'final-hash', - adapter_prepared_at = 2, provider_request_hash = 'final-hash' + SET released_knowledge_selected_refs = '[]', + released_knowledge_selected_refs_fingerprint = ${emptyReleasedRefsFingerprint} + WHERE receipt_id = 'receipt-provider-lifecycle' + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', final_request_hash = ${wireRequestHash}, + adapter_prepared_at = 2, provider_request_hash = ${wireRequestHash}, + prepared_turn_hash = ${preparedTurnHash}, + system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, + wire_request_hash = ${wireRequestHash}, + tool_definition_hash = ${preparedTurnHash} WHERE receipt_id = 'receipt-provider-lifecycle' `) yield* db.run(sql` @@ -818,6 +1086,19 @@ describe("DatabaseMigration", () => { .pipe(Effect.exit), ), ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = '[{"sourceStore":"project"}]' + WHERE receipt_id = 'receipt-provider-lifecycle' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) yield* db.run(sql` UPDATE session_tool_request_receipt SET provider_state = 'streaming', streaming_at = 4 @@ -846,6 +1127,54 @@ describe("DatabaseMigration", () => { .pipe(Effect.exit), ), ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET context_activation = '{"outcome":"mutated"}' + WHERE receipt_id = 'receipt-provider-lifecycle' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET released_knowledge_exact_refs = '[{"id":"mutated"}]' + WHERE receipt_id = 'receipt-provider-lifecycle' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + + expect( + Exit.isFailure( + yield* db + .run( + sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, provider_id, model_id, + registry_tool_ids, permission_filtered_tool_ids, final_offered_tool_ids, call_ids, + request_state, provider_state, prompt_epoch, prompt_window_id, effective_history_hash, + request_input_hash, final_request_hash, adapter_prepared_at, owner_token, created_at + ) VALUES ( + 'receipt-provider-missing-context', 2, 'session-provider-lifecycle', + 'message-provider-lifecycle', 'provider-test', 'model-test', '[]', '[]', '[]', '[]', + 'prepared', 'prepared', 0, 'window-0', 'history-0', 'input-hash-2', 'final-hash-2', + 2, 'owner-2', 2 + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) yield* db.run(sql` INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) @@ -904,22 +1233,2400 @@ describe("DatabaseMigration", () => { .pipe(Effect.exit), ), ).toBe(true) - yield* db.run(sql` - INSERT INTO part (id, message_id, session_id, provenance, time_created, time_updated, data) - VALUES ( - 'part-valid-provenance', 'message-provenance', 'session-provenance', - '{"source":"compaction_continue","owner_session_id":"session-provenance","owner_prompt_epoch":1,"owner_run_id":"run-1","durable":true}', - 1, 1, '{"type":"text","text":"x"}' - ) - `) + yield* db.run(sql` + INSERT INTO part (id, message_id, session_id, provenance, time_created, time_updated, data) + VALUES ( + 'part-valid-provenance', 'message-provenance', 'session-provenance', + '{"source":"compaction_continue","owner_session_id":"session-provenance","owner_prompt_epoch":1,"owner_run_id":"run-1","durable":true}', + 1, 1, '{"type":"text","text":"x"}' + ) + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE part + SET message_id = 'message-provenance-other', session_id = 'session-provenance-other' + WHERE id = 'part-valid-provenance' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE part + SET provenance = '{"source":"compaction_continue","owner_session_id":"session-provenance","owner_prompt_epoch":2,"owner_run_id":"run-1","durable":true}' + WHERE id = 'part-valid-provenance' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + }), + ) + }) + + test("upgrades an existing learning-job database with a fenced admission outbox", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const outboxMigrationIndex = migrations.findIndex( + (migration) => migration.id === learningAdmissionOutboxMigration.id, + ) + expect(outboxMigrationIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, outboxMigrationIndex)) + expect( + yield* db.get( + sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'learning_admission_outbox'`, + ), + ).toBeUndefined() + + yield* DatabaseMigration.applyOnly(db, [learningAdmissionOutboxMigration]) + + expect( + yield* db.get( + sql`SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'learning_admission_outbox'`, + ), + ).toEqual({ name: "learning_admission_outbox" }) + expect( + yield* db.all(sql` + SELECT name FROM sqlite_master + WHERE type = 'trigger' AND name LIKE 'learning_admission_outbox_%' + ORDER BY name + `), + ).toEqual([ + { name: "learning_admission_outbox_identity_immutable" }, + { name: "learning_admission_outbox_job_binding" }, + { name: "learning_admission_outbox_terminal_immutable" }, + { name: "learning_admission_outbox_transition_guard" }, + ]) + yield* db.run(sql` + INSERT INTO learning_admission_outbox ( + intent_id, session_id, run_id, trigger, dedupe_key, payload_json, + payload_fingerprint, state, created_at, updated_at + ) VALUES ( + 'intent-upgrade', 'session-upgrade', 'run-upgrade', 'session_finalization', + 'session_finalization:session-upgrade:run-upgrade', '{}', ${"a".repeat(64)}, + 'pending', 1, 1 + ) + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE learning_admission_outbox + SET state = 'admitted', job_id = 'missing-job', candidate_input_ref = 'artifact-ref', + settled_at = 2, updated_at = 2 + WHERE intent_id = 'intent-upgrade' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + UPDATE learning_admission_outbox + SET state = 'rejected', rejection_code = 'legacy_terminal_missing_exact_intent', + rejection_detail = 'exact terminal admission payload was unavailable', settled_at = 2, updated_at = 2 + WHERE intent_id = 'intent-upgrade' + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE learning_admission_outbox + SET rejection_detail = 'rewritten', updated_at = 3 + WHERE intent_id = 'intent-upgrade' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + }), + ) + }) + + test("upgrades durable permission routing and keeps route and feedback immutable", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const migrationIndex = migrations.findIndex( + (migration) => migration.id === activityPermissionRouteFeedbackMigration.id, + ) + expect(migrationIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, migrationIndex)) + expect(yield* db.all(sql`PRAGMA table_info(session_activity_permission_request)`)).not.toContainEqual( + expect.objectContaining({ name: "workspace_id" }), + ) + yield* db.run(sql` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('permission-route-project', '/tmp/permission-route-project', '[]', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session ( + id, project_id, slug, directory, title, version, time_created, time_updated, workspace_id + ) VALUES ( + 'permission-route-session', 'permission-route-project', 'permission-route-session', + '/tmp/permission-route-project', 'Permission route migration', '1', 1, 1, 'wrk_original' + ) + `) + yield* db.run(sql` + INSERT INTO session_intent ( + intent_id, session_id, source, state, selected_variant, selected_payload_hash, + delivery, admitted_message_id, mutation_epoch, version, time_created, time_admitted, time_updated + ) VALUES ( + 'permission-route-intent', 'permission-route-session', 'composer', 'admitted', 'original', + 'permission-route-payload', 'turn', 'permission-route-message', 0, 1, 1, 1, 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_activity_admission ( + admission_id, session_id, source_kind, legacy_intent_id, admitted_message_id, delivery, + payload_fingerprint_kind, payload_fingerprint, created_at + ) VALUES ( + 'permission-route-admission', 'permission-route-session', 'legacy_intent', 'permission-route-intent', + 'permission-route-message', 'turn', 'payload_hash', 'permission-route-payload', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_legacy_activity ( + activity_id, session_id, ordinal, trigger_admission_id, owner_token, state, + terminal_reason, created_at, settled_at + ) VALUES ( + 'permission-route-activity', 'permission-route-session', 0, 'permission-route-admission', + 'permission-route-owner', 'active', NULL, 1, NULL + ) + `) + yield* db.run(sql` + INSERT INTO session_activity_permission_request ( + request_id, activity_kind, activity_id, session_id, project_id, request_kind, + idempotency_key, permission, patterns, always_patterns, metadata_hash, + tool_message_id, tool_call_id, state, authority_epoch, requested_scope, + owner_type, owner_id, created_at, expires_at, decided_at + ) VALUES ( + 'permission-route-request', 'legacy', 'permission-route-activity', + 'permission-route-session', 'permission-route-project', 'tool', + 'permission-route-request-key', 'bash', '["ls"]', '[]', 'metadata-hash', + 'permission-route-message', 'permission-route-call', 'pending', 0, 'once', + 'runtime', 'permission-route-owner', 1, NULL, NULL + ) + `) + + yield* DatabaseMigration.applyOnly(db, [activityPermissionRouteFeedbackMigration]) + + expect(yield* db.all(sql`PRAGMA table_info(session_activity_permission_request)`)).toContainEqual( + expect.objectContaining({ name: "workspace_id" }), + ) + expect(yield* db.all(sql`PRAGMA table_info(session_activity_permission_decision)`)).toContainEqual( + expect.objectContaining({ name: "feedback" }), + ) + expect( + yield* db.all(sql` + SELECT name FROM sqlite_master + WHERE type = 'trigger' AND name IN ( + 'session_activity_permission_route_immutable', + 'session_activity_permission_feedback_immutable' + ) + ORDER BY name + `), + ).toEqual([ + { name: "session_activity_permission_feedback_immutable" }, + { name: "session_activity_permission_route_immutable" }, + ]) + expect( + yield* db.get(sql` + SELECT workspace_id FROM session_activity_permission_request + WHERE request_id = 'permission-route-request' + `), + ).toEqual({ workspace_id: "wrk_original" }) + yield* db.run(sql` + INSERT INTO session_activity_permission_decision ( + decision_id, request_id, idempotency_key, decision, actor_type, actor_id, + scope, authority_epoch, decided_at, expires_at, feedback + ) VALUES ( + 'permission-route-decision', 'permission-route-request', 'permission-route-decision-key', + 'denied', 'user', 'permission-user', 'once', 0, 2, NULL, 'keep this feedback' + ) + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_activity_permission_request + SET workspace_id = 'wrk_rewritten' + WHERE request_id = 'permission-route-request' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_activity_permission_decision + SET feedback = 'rewritten' + WHERE request_id = 'permission-route-request' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + }), + ) + }) + + test("fails closed for pre-W3 prepared receipts while preserving terminal recovery", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const preparedTurnMigrationIndex = migrations.findIndex( + (migration) => migration.id === preparedProviderTurnReceiptMigration.id, + ) + expect(preparedTurnMigrationIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, preparedTurnMigrationIndex)) + yield* seedReceiptScope(db) + + for (const [index, state] of ["prepared", "dispatching", "streaming"].entries()) { + const receiptID = `receipt-pre-w3-${state}` + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, provider_id, model_id, + registry_tool_ids, permission_filtered_tool_ids, final_offered_tool_ids, call_ids, + request_state, provider_state, prompt_epoch, prompt_window_id, effective_history_hash, + request_input_hash, owner_token, context_selection_id, context_eligibility, + context_readiness, context_activation, context_activation_fingerprint, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, created_at + ) VALUES ( + ${receiptID}, ${index + 1}, 'session-pre-w3', ${`message-pre-w3-${state}`}, + 'provider-test', 'model-test', '[]', '[]', '[]', '[]', 'prepared', 'preparing', + 0, ${`window-pre-w3-${state}`}, ${`history-pre-w3-${state}`}, + ${`input-pre-w3-${state}`}, 'owner-pre-w3', NULL, '{}', '{}', '{}', + lower(hex(zeroblob(32))), ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, ${index + 1} + ) + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = '[]', + released_knowledge_selected_refs_fingerprint = ${emptyReleasedRefsFingerprint} + WHERE receipt_id = ${receiptID} + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', final_request_hash = ${wireRequestHash}, + provider_request_hash = ${wireRequestHash}, adapter_prepared_at = 2 + WHERE receipt_id = ${receiptID} + `) + if (state === "prepared") continue + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'dispatching', request_state = 'dispatched', dispatching_at = 3 + WHERE receipt_id = ${receiptID} + `) + if (state === "dispatching") continue + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'streaming', streaming_at = 4 + WHERE receipt_id = ${receiptID} + `) + } + + yield* DatabaseMigration.applyOnly(db, [preparedProviderTurnReceiptMigration]) + + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET provider_state = 'dispatching', request_state = 'dispatched', dispatching_at = 5 + WHERE receipt_id = 'receipt-pre-w3-prepared' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'failed', terminal_at = 5, request_error_code = 'pre_w3_prepared_quarantined' + WHERE receipt_id = 'receipt-pre-w3-prepared' + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'settled', terminal_at = 5 + WHERE receipt_id = 'receipt-pre-w3-dispatching' + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'indeterminate_after_crash', terminal_at = 5, + request_error_code = 'pre_w3_streaming_recovery' + WHERE receipt_id = 'receipt-pre-w3-streaming' + `) + + expect( + yield* db.all<{ + receipt_id: string + provider_state: string + prepared_turn_hash: string | null + tool_result_reference_ids: string + tool_result_reference_count: number + }>(sql` + SELECT receipt_id, provider_state, prepared_turn_hash, + tool_result_reference_ids, tool_result_reference_count + FROM session_tool_request_receipt + WHERE receipt_id LIKE 'receipt-pre-w3-%' + ORDER BY receipt_id + `), + ).toEqual([ + { + receipt_id: "receipt-pre-w3-dispatching", + provider_state: "settled", + prepared_turn_hash: null, + tool_result_reference_ids: "[]", + tool_result_reference_count: 0, + }, + { + receipt_id: "receipt-pre-w3-prepared", + provider_state: "failed", + prepared_turn_hash: null, + tool_result_reference_ids: "[]", + tool_result_reference_count: 0, + }, + { + receipt_id: "receipt-pre-w3-streaming", + provider_state: "indeterminate_after_crash", + prepared_turn_hash: null, + tool_result_reference_ids: "[]", + tool_result_reference_count: 0, + }, + ]) + }), + ) + }) + + test("upgrades provider attempt transition law without classifying live rows", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const migrationIndex = migrations.findIndex( + (migration) => migration.id === providerAttemptPreDispatchTerminalMigration.id, + ) + expect(migrationIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, migrationIndex)) + yield* seedReceiptScope(db) + yield* db.run(sql` + INSERT INTO context_location_identity ( + security_namespace_id, location_key, project_scope_key, + canonical_root, observed_project_id, created_at + ) VALUES ( + ${receiptNamespace}, 'attempt-upgrade-location', ${receiptProjectScope}, + '/tmp/attempt-upgrade', 'attempt-upgrade-project', 1 + ) + `) + yield* db.run(sql` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('attempt-upgrade-project', '/tmp/attempt-upgrade', '[]', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) + VALUES ('attempt-upgrade-session', 'attempt-upgrade-project', 'attempt-upgrade', + '/tmp/attempt-upgrade', 'Attempt upgrade', 'test', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session_input (id, session_id, prompt, delivery, admitted_seq, promoted_seq, time_created) + VALUES ('attempt-upgrade-input', 'attempt-upgrade-session', '{"text":"upgrade"}', 'steer', 0, 0, 1) + `) + yield* db.run(sql` + INSERT INTO session_activity ( + activity_id, session_id, ordinal, trigger_input_id, delivery, state, created_at + ) VALUES ( + 'attempt-upgrade-activity', 'attempt-upgrade-session', 0, + 'attempt-upgrade-input', 'steer', 'active', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_context_selection ( + selection_id, session_id, activity_id, revision, trigger_input_id, location_key, + security_namespace_id, project_scope_key, + query_fingerprint, authorization_fingerprint, authorization_epoch, + execution_fingerprint, selected_source_fingerprint, + observed_location_mutation_epoch, next_revalidation_at, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + graph_revisions, graph_statuses, selected_refs, projection, + projection_hash, token_count, artifact_write_status, inline_audit, created_at + ) VALUES ( + 'attempt-upgrade-selection', 'attempt-upgrade-session', 'attempt-upgrade-activity', 0, + 'attempt-upgrade-input', 'attempt-upgrade-location', ${receiptNamespace}, ${receiptProjectScope}, + 'query', 'authorization', 1, 'execution', 'sources', 0, 100, + 'unavailable', '[]', ${emptyReleasedRefsFingerprint}, '{}', '{}', '[]', 'projection', + 'projection-hash', 1, 'degraded_unavailable', '{}', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, state, created_at + ) VALUES + ('attempt-upgrade-prepared', 'attempt-upgrade-session', 'attempt-upgrade-activity', 0, + 'attempt-upgrade-selection', 'projection-hash', 'request-0', 'provider', 'prepared', 1), + ('attempt-upgrade-dispatching', 'attempt-upgrade-session', 'attempt-upgrade-activity', 1, + 'attempt-upgrade-selection', 'projection-hash', 'request-1', 'provider', 'dispatching', 2), + ('attempt-upgrade-orphan', 'attempt-upgrade-session', 'attempt-upgrade-activity', 2, + 'attempt-upgrade-selection', 'projection-hash', 'request-2', 'provider', 'prepared', 3), + ('attempt-upgrade-started-receipt', 'attempt-upgrade-session', 'attempt-upgrade-activity', 3, + 'attempt-upgrade-selection', 'projection-hash', 'request-3', 'provider', 'prepared', 4) + `) + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, + provider_attempt_id, context_selection_id, + context_eligibility, context_readiness, context_activation, context_activation_fingerprint, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, final_request_hash, adapter_prepared_at, + prompt_epoch, prompt_window_id, effective_history_hash, + request_state, provider_state, owner_token, created_at + ) VALUES ( + 'attempt-upgrade-receipt', 1, 'attempt-upgrade-session', 'attempt-upgrade-input', + 'attempt-upgrade-prepared', 'attempt-upgrade-selection', '{}', '{}', '{}', + lower(hex(zeroblob(32))), ${receiptNamespace}, ${receiptProjectScope}, + 'unavailable', '[]', ${emptyReleasedRefsFingerprint}, + 'provider', 'model', '[]', '[]', '[]', '[]', NULL, NULL, NULL, NULL, NULL, + 'prepared', 'preparing', 'stale-owner', 3 + ), ( + 'attempt-upgrade-started-receipt', 2, 'attempt-upgrade-session', 'attempt-upgrade-input', + 'attempt-upgrade-started-receipt', 'attempt-upgrade-selection', '{}', '{}', '{}', + lower(hex(zeroblob(32))), ${receiptNamespace}, ${receiptProjectScope}, + 'unavailable', '[]', ${emptyReleasedRefsFingerprint}, + 'provider', 'model', '[]', '[]', '[]', '[]', NULL, NULL, 0, + 'attempt-upgrade-window', 'attempt-upgrade-history', + 'prepared', 'preparing', 'stale-owner', 4 + ) + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = '[]', + released_knowledge_selected_refs_fingerprint = ${emptyReleasedRefsFingerprint} + WHERE receipt_id = 'attempt-upgrade-started-receipt' + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', final_request_hash = ${wireRequestHash}, + provider_request_hash = ${wireRequestHash}, + prepared_turn_hash = ${preparedTurnHash}, system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, wire_request_hash = ${wireRequestHash}, + tool_result_reference_ids = '[]', tool_result_reference_count = 0, + tool_definition_hash = ${wireRequestHash}, adapter_prepared_at = 4 + WHERE receipt_id = 'attempt-upgrade-started-receipt' + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'dispatching', request_state = 'dispatched', dispatching_at = 5 + WHERE receipt_id = 'attempt-upgrade-started-receipt' + `) + yield* db.run(sql` + INSERT INTO compaction_run ( + run_id, session_id, from_prompt_epoch, trigger, state, created_at, + source_window_id, source_effective_history_hash, source_message_count, source_projection_version, + continuation_wakeup_at, continuation_state, continuation_receipt_id, + continuation_admitted_at + ) VALUES ( + 'attempt-upgrade-compaction', 'attempt-upgrade-session', 0, 'turn_start', 'committed', 4, + 'attempt-upgrade-window', 'attempt-upgrade-history', 1, 1, + 4, 'admitted', 'attempt-upgrade-receipt', 4 + ) + `) + + yield* DatabaseMigration.applyOnly(db, [providerAttemptPreDispatchTerminalMigration]) + + expect( + yield* db.all<{ + attempt_id: string + state: string + settled_at: number | null + error_code: string | null + }>(sql` + SELECT attempt_id, state, settled_at, error_code + FROM session_provider_attempt + ORDER BY provider_turn_seq + `), + ).toEqual([ + { + attempt_id: "attempt-upgrade-prepared", + state: "prepared", + settled_at: null, + error_code: null, + }, + { + attempt_id: "attempt-upgrade-dispatching", + state: "dispatching", + settled_at: null, + error_code: null, + }, + { + attempt_id: "attempt-upgrade-orphan", + state: "prepared", + settled_at: null, + error_code: null, + }, + { + attempt_id: "attempt-upgrade-started-receipt", + state: "prepared", + settled_at: null, + error_code: null, + }, + ]) + expect( + yield* db.get(sql` + SELECT provider_state, request_state, terminal_at, request_error_code + FROM session_tool_request_receipt + WHERE receipt_id = 'attempt-upgrade-receipt' + `), + ).toEqual({ + provider_state: "preparing", + request_state: "prepared", + terminal_at: null, + request_error_code: null, + }) + expect( + yield* db.get(sql` + SELECT continuation_state, continuation_receipt_id, continuation_admitted_at, + continuation_wakeup_at, continuation_error_code + FROM compaction_run + WHERE run_id = 'attempt-upgrade-compaction' + `), + ).toEqual({ + continuation_state: "admitted", + continuation_receipt_id: "attempt-upgrade-receipt", + continuation_admitted_at: 4, + continuation_wakeup_at: 4, + continuation_error_code: null, + }) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_provider_attempt + SET state = 'failed', settled_at = 3 + WHERE attempt_id = 'attempt-upgrade-dispatching' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(false) + yield* db.run(sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, state, created_at + ) VALUES ( + 'attempt-upgrade-post-migration', 'attempt-upgrade-session', 'attempt-upgrade-activity', 4, + 'attempt-upgrade-selection', 'projection-hash', 'request-4', 'provider', 'prepared', 5 + ) + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_provider_attempt + SET state = 'failed', settled_at = 5 + WHERE attempt_id = 'attempt-upgrade-post-migration' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + UPDATE session_provider_attempt + SET state = 'failed', settled_at = 5, error_code = 'provider_not_dispatched' + WHERE attempt_id = 'attempt-upgrade-post-migration' + `) + yield* DatabaseMigration.applyOnly(db, [providerOwnerAuthorityMigration]) + expect( + yield* db.all(sql` + SELECT attempt_id, owner_token + FROM session_provider_attempt + ORDER BY provider_turn_seq + `), + ).toEqual([ + { attempt_id: "attempt-upgrade-prepared", owner_token: null }, + { attempt_id: "attempt-upgrade-dispatching", owner_token: null }, + { attempt_id: "attempt-upgrade-orphan", owner_token: null }, + { attempt_id: "attempt-upgrade-started-receipt", owner_token: null }, + { attempt_id: "attempt-upgrade-post-migration", owner_token: null }, + ]) + expect( + Exit.isFailure( + yield* db + .run( + sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, state, created_at + ) VALUES ( + 'attempt-upgrade-unowned-new', 'attempt-upgrade-session', 'attempt-upgrade-activity', 5, + 'attempt-upgrade-selection', 'projection-hash', 'request-5', 'provider', 'prepared', 6 + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ('attempt-upgrade-owner', 5, 5, 10) + `) + yield* db.run(sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, owner_token, state, created_at + ) VALUES ( + 'attempt-upgrade-owned-new', 'attempt-upgrade-session', 'attempt-upgrade-activity', 5, + 'attempt-upgrade-selection', 'projection-hash', 'request-5', 'provider', + 'attempt-upgrade-owner', 'prepared', 6 + ) + `) + }), + ) + }) + + test("enforces exact provider attempt receipt and wire identity", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* DatabaseMigration.apply(db) + yield* seedReceiptScope(db) + yield* db.run(sql` + INSERT INTO context_location_identity ( + security_namespace_id, location_key, project_scope_key, + canonical_root, observed_project_id, created_at + ) VALUES ( + ${receiptNamespace}, 'provider-identity-location', ${receiptProjectScope}, + '/tmp/provider-identity', 'provider-identity-project', 1 + ) + `) + yield* db.run(sql` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('provider-identity-project', '/tmp/provider-identity', '[]', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) + VALUES ('provider-identity-session', 'provider-identity-project', 'provider-identity', + '/tmp/provider-identity', 'Provider identity', 'test', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session_input (id, session_id, prompt, delivery, admitted_seq, promoted_seq, time_created) + VALUES ('provider-identity-input', 'provider-identity-session', '{"text":"identity"}', 'steer', 0, 0, 1) + `) + yield* db.run(sql` + INSERT INTO session_activity ( + activity_id, session_id, ordinal, trigger_input_id, delivery, state, created_at + ) VALUES ('provider-identity-activity', 'provider-identity-session', 0, + 'provider-identity-input', 'steer', 'active', 1) + `) + yield* db.run(sql` + UPDATE session_activity + SET state = 'settled', settled_at = 2 + WHERE activity_id = 'provider-identity-activity' + `) + yield* db.run(sql` + INSERT INTO session_activity ( + activity_id, session_id, ordinal, trigger_input_id, delivery, state, created_at + ) VALUES ('provider-identity-wrong-activity', 'provider-identity-session', 1, + 'provider-identity-input', 'steer', 'active', 2) + `) + yield* db.run(sql` + INSERT INTO session_context_selection ( + selection_id, session_id, activity_id, revision, trigger_input_id, location_key, + security_namespace_id, project_scope_key, + query_fingerprint, authorization_fingerprint, authorization_epoch, + execution_fingerprint, selected_source_fingerprint, + observed_location_mutation_epoch, next_revalidation_at, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + graph_revisions, graph_statuses, selected_refs, projection, + projection_hash, token_count, artifact_write_status, inline_audit, created_at + ) VALUES ( + 'provider-identity-selection', 'provider-identity-session', 'provider-identity-activity', 0, + 'provider-identity-input', 'provider-identity-location', ${receiptNamespace}, ${receiptProjectScope}, + 'query', 'authorization', 1, 'execution', 'sources', 0, 100, + 'unavailable', '[]', ${emptyReleasedRefsFingerprint}, '{}', '{}', '[]', 'projection', + 'provider-identity-projection', 1, 'degraded_unavailable', '{}', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ( + 'provider-identity-owner', + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + 10000 + ) + `) + yield* db.run(sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, owner_token, state, created_at + ) VALUES + ('provider-identity-valid', 'provider-identity-session', 'provider-identity-activity', 0, + 'provider-identity-selection', 'provider-identity-projection', ${"a".repeat(64)}, + 'provider', 'provider-identity-owner', 'prepared', 2), + ('provider-identity-wrong-activity', 'provider-identity-session', 'provider-identity-wrong-activity', 1, + 'provider-identity-selection', 'provider-identity-projection', ${"a".repeat(64)}, + 'provider', 'provider-identity-owner', 'prepared', 2), + ('provider-identity-wrong-projection', 'provider-identity-session', 'provider-identity-activity', 2, + 'provider-identity-selection', 'wrong-projection', ${"a".repeat(64)}, + 'provider', 'provider-identity-owner', 'prepared', 2), + ('provider-identity-wrong-provider', 'provider-identity-session', 'provider-identity-activity', 3, + 'provider-identity-selection', 'provider-identity-projection', ${"a".repeat(64)}, + 'wrong-provider', 'provider-identity-owner', 'prepared', 2), + ('provider-identity-wrong-request', 'provider-identity-session', 'provider-identity-activity', 4, + 'provider-identity-selection', 'provider-identity-projection', ${"f".repeat(64)}, + 'provider', 'provider-identity-owner', 'prepared', 2) + `) + + for (const [ordinal, attemptID] of [ + "provider-identity-wrong-activity", + "provider-identity-wrong-projection", + "provider-identity-wrong-provider", + "provider-identity-wrong-request", + ].entries()) { + expect( + Exit.isFailure( + yield* db + .run( + sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, + provider_attempt_id, context_selection_id, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, request_input_hash, + request_state, provider_state, owner_token, created_at + ) VALUES ( + ${`receipt-${attemptID}`}, ${ordinal + 1}, 'provider-identity-session', + 'provider-identity-input', ${attemptID}, 'provider-identity-selection', + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${"a".repeat(64)}, 'prepared', 'preparing', 'provider-identity-owner', 3 + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + } + + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, + provider_attempt_id, context_selection_id, + context_eligibility, context_readiness, context_activation, context_activation_fingerprint, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, request_input_hash, + prompt_epoch, prompt_window_id, effective_history_hash, + request_state, provider_state, owner_token, created_at + ) VALUES ( + 'provider-identity-receipt', 10, 'provider-identity-session', 'provider-identity-input', + 'provider-identity-valid', 'provider-identity-selection', + ${JSON.stringify(contextEligibility)}, ${JSON.stringify(contextReadiness)}, + ${contextActivation(3, "provider-identity-selection", "provider-identity-projection")}, + lower(hex(zeroblob(32))), ${receiptNamespace}, ${receiptProjectScope}, + 'unavailable', '[]', ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${"a".repeat(64)}, 0, 'provider-identity-window', 'provider-identity-history', + 'prepared', 'preparing', 'provider-identity-owner', 3 + ) + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, + provider_attempt_id, context_selection_id, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, request_input_hash, + request_state, provider_state, owner_token, created_at + ) VALUES ( + 'provider-identity-duplicate', 11, 'provider-identity-session', 'provider-identity-input', + 'provider-identity-valid', 'provider-identity-selection', + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${"a".repeat(64)}, 'prepared', 'preparing', 'provider-identity-owner', 3 + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = '[]', + released_knowledge_selected_refs_fingerprint = ${emptyReleasedRefsFingerprint} + WHERE receipt_id = 'provider-identity-receipt' + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_provider_attempt + SET prepared_turn_hash = ${preparedTurnHash} + WHERE attempt_id = 'provider-identity-valid' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_provider_attempt + SET prepared_turn_hash = ${"B".repeat(64)}, wire_request_hash = ${wireRequestHash} + WHERE attempt_id = 'provider-identity-valid' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + UPDATE session_provider_attempt + SET prepared_turn_hash = ${preparedTurnHash}, wire_request_hash = ${wireRequestHash} + WHERE attempt_id = 'provider-identity-valid' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET prepared_turn_hash = ${preparedTurnHash}, wire_request_hash = ${wireRequestHash} + WHERE attempt_id = 'provider-identity-valid' + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_provider_attempt + SET prepared_turn_hash = ${"f".repeat(64)}, wire_request_hash = ${wireRequestHash} + WHERE attempt_id = 'provider-identity-valid' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', final_request_hash = ${wireRequestHash}, + provider_request_hash = ${wireRequestHash}, + prepared_turn_hash = ${"f".repeat(64)}, system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, wire_request_hash = ${wireRequestHash}, + tool_result_reference_ids = '[]', tool_result_reference_count = 0, + tool_definition_hash = ${wireRequestHash}, adapter_prepared_at = 4 + WHERE receipt_id = 'provider-identity-receipt' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', final_request_hash = ${wireRequestHash}, + provider_request_hash = ${wireRequestHash}, + prepared_turn_hash = ${preparedTurnHash}, system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, wire_request_hash = ${wireRequestHash}, + tool_result_reference_ids = '[]', tool_result_reference_count = 0, + tool_definition_hash = ${wireRequestHash}, adapter_prepared_at = 4 + WHERE receipt_id = 'provider-identity-receipt' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET state = 'dispatching' + WHERE attempt_id = 'provider-identity-valid' + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'dispatching', request_state = 'dispatched', dispatching_at = 5 + WHERE receipt_id = 'provider-identity-receipt' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET prepared_turn_hash = ${preparedTurnHash}, wire_request_hash = ${wireRequestHash} + WHERE attempt_id = 'provider-identity-wrong-request' + `) + yield* db.run(sql` + UPDATE session_provider_owner_lease + SET released_at = CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + WHERE owner_token = 'provider-identity-owner' + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_provider_attempt + SET state = 'indeterminate_after_crash', error_code = 'caller_supplied_recovery' + WHERE attempt_id = 'provider-identity-wrong-request' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_provider_attempt + SET state = 'indeterminate_after_crash', error_code = 'process_recovery' + WHERE attempt_id = 'provider-identity-wrong-request' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ( + 'provider-identity-recovery-owner', + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + 10000 + ) + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET state = 'indeterminate_after_crash', error_code = 'process_recovery' + WHERE attempt_id = 'provider-identity-wrong-request' + `) + expect( + yield* db.get(sql` + SELECT state, error_code, settled_at, prepared_turn_hash, wire_request_hash + FROM session_provider_attempt + WHERE attempt_id = 'provider-identity-wrong-request' + `), + ).toEqual({ + state: "indeterminate_after_crash", + error_code: "process_recovery", + settled_at: null, + prepared_turn_hash: preparedTurnHash, + wire_request_hash: wireRequestHash, + }) + }), + ) + }) + + test("preserves exact historical receipts but rejects ambiguous and mismatched provider attempt upgrades", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const migrationIndex = migrations.findIndex( + (migration) => migration.id === providerTurnIdentityAuthorityMigration.id, + ) + expect(migrationIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, migrationIndex)) + yield* seedReceiptScope(db) + yield* db.run(sql` + INSERT INTO context_location_identity ( + security_namespace_id, location_key, project_scope_key, + canonical_root, observed_project_id, created_at + ) VALUES ( + ${receiptNamespace}, 'provider-upgrade-location', ${receiptProjectScope}, + '/tmp/provider-upgrade', 'provider-upgrade-project', 1 + ) + `) + yield* db.run(sql` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('provider-upgrade-project', '/tmp/provider-upgrade', '[]', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) + VALUES ('provider-upgrade-session', 'provider-upgrade-project', 'provider-upgrade', + '/tmp/provider-upgrade', 'Provider upgrade', 'test', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session_input (id, session_id, prompt, delivery, admitted_seq, promoted_seq, time_created) + VALUES ('provider-upgrade-input', 'provider-upgrade-session', '{"text":"upgrade"}', 'steer', 0, 0, 1) + `) + yield* db.run(sql` + INSERT INTO session_activity ( + activity_id, session_id, ordinal, trigger_input_id, delivery, state, created_at + ) VALUES ('provider-upgrade-activity', 'provider-upgrade-session', 0, + 'provider-upgrade-input', 'steer', 'active', 1) + `) + yield* db.run(sql` + INSERT INTO session_context_selection ( + selection_id, session_id, activity_id, revision, trigger_input_id, location_key, + security_namespace_id, project_scope_key, + query_fingerprint, authorization_fingerprint, authorization_epoch, + execution_fingerprint, selected_source_fingerprint, + observed_location_mutation_epoch, next_revalidation_at, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + graph_revisions, graph_statuses, selected_refs, projection, + projection_hash, token_count, artifact_write_status, inline_audit, created_at + ) VALUES ( + 'provider-upgrade-selection', 'provider-upgrade-session', 'provider-upgrade-activity', 0, + 'provider-upgrade-input', 'provider-upgrade-location', ${receiptNamespace}, ${receiptProjectScope}, + 'query', 'authorization', 1, 'execution', 'sources', 0, 100, + 'unavailable', '[]', ${emptyReleasedRefsFingerprint}, '{}', '{}', '[]', 'projection', + 'provider-upgrade-projection', 1, 'degraded_unavailable', '{}', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ('provider-upgrade-owner', 1, 1, 100) + `) + yield* db.run(sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, owner_token, state, created_at + ) VALUES ('provider-upgrade-attempt', 'provider-upgrade-session', 'provider-upgrade-activity', 0, + 'provider-upgrade-selection', 'provider-upgrade-projection', ${"a".repeat(64)}, + 'provider', 'provider-upgrade-owner', 'prepared', 2) + `) + for (const ordinal of [1, 2]) { + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, + provider_attempt_id, context_selection_id, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, request_input_hash, + request_state, provider_state, owner_token, created_at + ) VALUES ( + ${`provider-upgrade-receipt-${ordinal}`}, ${ordinal}, 'provider-upgrade-session', + 'provider-upgrade-input', 'provider-upgrade-attempt', 'provider-upgrade-selection', + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${"a".repeat(64)}, 'prepared', 'preparing', 'provider-upgrade-owner', ${ordinal + 2} + ) + `) + } + expect( + Exit.isFailure( + yield* DatabaseMigration.applyOnly(db, [providerTurnIdentityAuthorityMigration]).pipe(Effect.exit), + ), + ).toBe(true) + expect( + yield* db.all(sql` + SELECT name FROM pragma_table_info('session_provider_attempt') + WHERE name IN ('prepared_turn_hash', 'wire_request_hash') + `), + ).toEqual([]) + yield* db.run(sql` + DELETE FROM session_tool_request_receipt + WHERE receipt_id IN ('provider-upgrade-receipt-1', 'provider-upgrade-receipt-2') + `) + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, + provider_attempt_id, context_selection_id, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, request_input_hash, + request_state, provider_state, owner_token, created_at + ) VALUES ( + 'provider-upgrade-receipt-1', 1, 'provider-upgrade-session', + 'provider-upgrade-input', 'provider-upgrade-attempt', 'provider-upgrade-selection', + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${"f".repeat(64)}, 'prepared', 'preparing', 'provider-upgrade-owner', 3 + ) + `) + expect( + Exit.isFailure( + yield* DatabaseMigration.applyOnly(db, [providerTurnIdentityAuthorityMigration]).pipe(Effect.exit), + ), + ).toBe(true) + expect( + yield* db.all(sql` + SELECT name FROM pragma_table_info('session_provider_attempt') + WHERE name IN ('prepared_turn_hash', 'wire_request_hash') + `), + ).toEqual([]) + yield* db.run(sql` + DELETE FROM session_tool_request_receipt + WHERE receipt_id = 'provider-upgrade-receipt-1' + `) + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, + provider_attempt_id, context_selection_id, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, request_input_hash, + request_state, provider_state, owner_token, created_at + ) VALUES ( + 'provider-upgrade-receipt-1', 1, 'provider-upgrade-session', + 'provider-upgrade-input', 'provider-upgrade-attempt', 'provider-upgrade-selection', + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${"a".repeat(64)}, 'prepared', 'preparing', 'provider-upgrade-owner', 3 + ) + `) + yield* DatabaseMigration.applyOnly(db, [providerTurnIdentityAuthorityMigration]) + expect( + yield* db.get(sql` + SELECT released_at IS NOT NULL AS released + FROM session_provider_owner_lease + WHERE owner_token = 'provider-upgrade-owner' + `), + ).toEqual({ released: 1 }) + expect( + yield* db.get(sql` + SELECT prepared_turn_hash, wire_request_hash + FROM session_provider_attempt + WHERE attempt_id = 'provider-upgrade-attempt' + `), + ).toEqual({ prepared_turn_hash: null, wire_request_hash: null }) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_provider_attempt + SET state = 'dispatching' + WHERE attempt_id = 'provider-upgrade-attempt' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, owner_token, state, created_at + ) VALUES ( + 'provider-upgrade-backdated-attempt', 'provider-upgrade-session', + 'provider-upgrade-activity', 1, 'provider-upgrade-selection', + 'provider-upgrade-projection', ${"a".repeat(64)}, 'provider', + 'provider-upgrade-owner', 'prepared', 1 + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_provider_attempt + SET prepared_turn_hash = ${preparedTurnHash}, wire_request_hash = ${wireRequestHash} + WHERE attempt_id = 'provider-upgrade-attempt' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + UPDATE session_provider_attempt + SET state = 'failed', settled_at = 5, error_code = 'legacy_wire_identity_missing' + WHERE attempt_id = 'provider-upgrade-attempt' + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'failed', terminal_at = 5, + request_error_code = 'legacy_wire_identity_missing' + WHERE receipt_id = 'provider-upgrade-receipt-1' + `) + expect( + yield* db.get(sql` + SELECT attempt.state AS attempt_state, receipt.provider_state AS receipt_state + FROM session_provider_attempt attempt + JOIN session_tool_request_receipt receipt + ON receipt.provider_attempt_id = attempt.attempt_id + WHERE attempt.attempt_id = 'provider-upgrade-attempt' + `), + ).toEqual({ attempt_state: "failed", receipt_state: "failed" }) + }), + ) + }) + + test("rejects future and overlong pre-authority provider owner leases", async () => { + for (const lease of [ + { + ownerToken: "provider-upgrade-future-owner", + registeredAt: Date.now() + 60_000, + heartbeatAt: Date.now() + 60_000, + leaseExpiresAt: Date.now() + 120_000, + }, + { + ownerToken: "provider-upgrade-overlong-owner", + registeredAt: 1, + heartbeatAt: 1, + leaseExpiresAt: 31_536_000_002, + }, + ]) { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const migrationIndex = migrations.findIndex( + (migration) => migration.id === providerTurnIdentityAuthorityMigration.id, + ) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, migrationIndex)) + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ( + ${lease.ownerToken}, ${lease.registeredAt}, ${lease.heartbeatAt}, ${lease.leaseExpiresAt} + ) + `) + expect( + Exit.isFailure( + yield* DatabaseMigration.applyOnly(db, [providerTurnIdentityAuthorityMigration]).pipe(Effect.exit), + ), + ).toBe(true) + expect( + yield* db.all(sql` + SELECT name FROM pragma_table_info('session_provider_attempt') + WHERE name IN ('prepared_turn_hash', 'wire_request_hash') + `), + ).toEqual([]) + }), + ) + } + }) + + test("permits only exact physical-start evidence to quarantine an undispatched provider receipt", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* DatabaseMigration.apply(db) + yield* DatabaseMigration.applyOnly(db, [providerCrossStateRecoveryMigration]) + yield* seedReceiptScope(db) + yield* db.run(sql` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('provider-cross-state-project', '/tmp/provider-cross-state', '[]', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) + VALUES ('provider-cross-state-session', 'provider-cross-state-project', 'provider-cross-state', + '/tmp/provider-cross-state', 'Provider cross state', 'test', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session_input (id, session_id, prompt, delivery, admitted_seq, promoted_seq, time_created) + VALUES ('provider-cross-state-input', 'provider-cross-state-session', '{"text":"cross"}', 'steer', 0, 0, 1) + `) + yield* db.run(sql` + INSERT INTO session_activity ( + activity_id, session_id, ordinal, trigger_input_id, delivery, state, created_at + ) VALUES ('provider-cross-state-activity', 'provider-cross-state-session', 0, + 'provider-cross-state-input', 'steer', 'active', 1) + `) + yield* db.run(sql` + INSERT INTO context_location_identity ( + security_namespace_id, location_key, project_scope_key, + canonical_root, observed_project_id, created_at + ) VALUES ( + ${receiptNamespace}, 'provider-cross-state-location', ${receiptProjectScope}, + '/tmp/provider-cross-state', 'provider-cross-state-project', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_context_selection ( + selection_id, session_id, activity_id, revision, trigger_input_id, location_key, + security_namespace_id, project_scope_key, + query_fingerprint, authorization_fingerprint, authorization_epoch, + execution_fingerprint, selected_source_fingerprint, + observed_location_mutation_epoch, next_revalidation_at, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + graph_revisions, graph_statuses, selected_refs, projection, + projection_hash, token_count, artifact_write_status, inline_audit, created_at + ) VALUES ( + 'provider-cross-state-selection', 'provider-cross-state-session', 'provider-cross-state-activity', 0, + 'provider-cross-state-input', 'provider-cross-state-location', ${receiptNamespace}, ${receiptProjectScope}, + 'query', 'authorization', 1, 'execution', 'sources', 0, 100, + 'unavailable', '[]', ${emptyReleasedRefsFingerprint}, '{}', '{}', '[]', 'projection', + 'provider-cross-state-projection', 1, 'degraded_unavailable', '{}', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ( + 'provider-cross-state-owner', + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + 10000 + ) + `) + yield* db.run(sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, owner_token, state, created_at + ) VALUES ( + 'provider-cross-state-attempt', 'provider-cross-state-session', 'provider-cross-state-activity', 0, + 'provider-cross-state-selection', 'provider-cross-state-projection', ${wireRequestHash}, + 'provider', 'provider-cross-state-owner', 'prepared', 2 + ) + `) + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, + provider_attempt_id, context_selection_id, + context_eligibility, context_readiness, context_activation, context_activation_fingerprint, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, request_input_hash, + prompt_epoch, prompt_window_id, effective_history_hash, + request_state, provider_state, owner_token, created_at + ) VALUES ( + 'provider-cross-state-receipt', 1, 'provider-cross-state-session', + 'provider-cross-state-input', 'provider-cross-state-attempt', 'provider-cross-state-selection', + ${JSON.stringify(contextEligibility)}, ${JSON.stringify(contextReadiness)}, + ${contextActivation(3, "provider-cross-state-selection", "provider-cross-state-projection")}, + lower(hex(zeroblob(32))), + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${wireRequestHash}, 0, 'provider-cross-state-window', 'provider-cross-state-history', + 'prepared', 'preparing', 'provider-cross-state-owner', 3 + ) + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = '[]', + released_knowledge_selected_refs_fingerprint = ${emptyReleasedRefsFingerprint} + WHERE receipt_id = 'provider-cross-state-receipt' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET prepared_turn_hash = ${preparedTurnHash}, wire_request_hash = ${wireRequestHash} + WHERE attempt_id = 'provider-cross-state-attempt' + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', adapter_prepared_at = 4, + final_request_hash = ${wireRequestHash}, provider_request_hash = ${wireRequestHash}, + prepared_turn_hash = ${preparedTurnHash}, + system_stable_hash = ${systemStableHash}, system_volatile_hash = ${systemVolatileHash}, + wire_request_hash = ${wireRequestHash}, tool_definition_hash = ${wireRequestHash}, + tool_result_reference_ids = '[]', tool_result_reference_count = 0 + WHERE receipt_id = 'provider-cross-state-receipt' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET state = 'dispatching' + WHERE attempt_id = 'provider-cross-state-attempt' + `) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET provider_state = 'indeterminate_after_crash', terminal_at = 5, + request_error_code = 'wrong_recovery_reason' + WHERE receipt_id = 'provider-cross-state-receipt' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'indeterminate_after_crash', terminal_at = 5, + request_error_code = 'provider_started_outcome_unknown_after_process_restart' + WHERE receipt_id = 'provider-cross-state-receipt' + `) + expect( + yield* db.get(sql` + SELECT provider_state, request_error_code + FROM session_tool_request_receipt + WHERE receipt_id = 'provider-cross-state-receipt' + `), + ).toEqual({ + provider_state: "indeterminate_after_crash", + request_error_code: "provider_started_outcome_unknown_after_process_restart", + }) + }), + ) + }) + + test("quarantines exact provider cross-state upgrades but rejects identity mismatches atomically", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const migrationIndex = migrations.findIndex( + (migration) => migration.id === providerCrossStateRecoveryMigration.id, + ) + expect(migrationIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, migrationIndex)) + yield* seedReceiptScope(db) + yield* db.run(sql` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('provider-cross-upgrade-project', '/tmp/provider-cross-upgrade', '[]', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) + VALUES ('provider-cross-upgrade-session', 'provider-cross-upgrade-project', 'provider-cross-upgrade', + '/tmp/provider-cross-upgrade', 'Provider cross upgrade', 'test', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session_input (id, session_id, prompt, delivery, admitted_seq, promoted_seq, time_created) + VALUES ('provider-cross-upgrade-input', 'provider-cross-upgrade-session', + '{"text":"cross"}', 'steer', 0, 0, 1) + `) + yield* db.run(sql` + INSERT INTO session_prompt_epoch ( + session_id, epoch, state, reason, created_at, authority_state + ) VALUES ( + 'provider-cross-upgrade-session', 0, 'active', 'bootstrap', 1, 'legacy_pending' + ) + `) + yield* db.run(sql` + INSERT INTO session_activity ( + activity_id, session_id, ordinal, trigger_input_id, delivery, state, created_at + ) VALUES ('provider-cross-upgrade-activity', 'provider-cross-upgrade-session', 0, + 'provider-cross-upgrade-input', 'steer', 'active', 1) + `) + yield* db.run(sql` + INSERT INTO context_location_identity ( + security_namespace_id, location_key, project_scope_key, + canonical_root, observed_project_id, created_at + ) VALUES ( + ${receiptNamespace}, 'provider-cross-upgrade-location', ${receiptProjectScope}, + '/tmp/provider-cross-upgrade', 'provider-cross-upgrade-project', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_context_selection ( + selection_id, session_id, activity_id, revision, trigger_input_id, location_key, + security_namespace_id, project_scope_key, + query_fingerprint, authorization_fingerprint, authorization_epoch, + execution_fingerprint, selected_source_fingerprint, + observed_location_mutation_epoch, next_revalidation_at, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + graph_revisions, graph_statuses, selected_refs, projection, + projection_hash, token_count, artifact_write_status, inline_audit, created_at + ) VALUES ( + 'provider-cross-upgrade-selection', 'provider-cross-upgrade-session', + 'provider-cross-upgrade-activity', 0, 'provider-cross-upgrade-input', + 'provider-cross-upgrade-location', ${receiptNamespace}, ${receiptProjectScope}, + 'query', 'authorization', 1, 'execution', 'sources', 0, 100, + 'unavailable', '[]', ${emptyReleasedRefsFingerprint}, '{}', '{}', '[]', 'projection', + 'provider-cross-upgrade-projection', 1, 'degraded_unavailable', '{}', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ( + 'provider-cross-upgrade-owner', + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + 10000 + ) + `) + yield* db.run(sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, owner_token, state, created_at + ) VALUES + ('provider-cross-upgrade-attempt-dispatching', 'provider-cross-upgrade-session', + 'provider-cross-upgrade-activity', 0, 'provider-cross-upgrade-selection', + 'provider-cross-upgrade-projection', ${wireRequestHash}, 'provider', + 'provider-cross-upgrade-owner', 'prepared', 2), + ('provider-cross-upgrade-attempt-streaming', 'provider-cross-upgrade-session', + 'provider-cross-upgrade-activity', 1, 'provider-cross-upgrade-selection', + 'provider-cross-upgrade-projection', ${wireRequestHash}, 'provider', + 'provider-cross-upgrade-owner', 'prepared', 2), + ('provider-cross-upgrade-attempt-settled', 'provider-cross-upgrade-session', + 'provider-cross-upgrade-activity', 2, 'provider-cross-upgrade-selection', + 'provider-cross-upgrade-projection', ${wireRequestHash}, 'provider', + 'provider-cross-upgrade-owner', 'prepared', 2), + ('provider-cross-upgrade-attempt-failed', 'provider-cross-upgrade-session', + 'provider-cross-upgrade-activity', 3, 'provider-cross-upgrade-selection', + 'provider-cross-upgrade-projection', ${wireRequestHash}, 'provider', + 'provider-cross-upgrade-owner', 'prepared', 2) + `) + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, + provider_attempt_id, context_selection_id, + context_eligibility, context_readiness, context_activation, context_activation_fingerprint, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + provider_id, model_id, registry_tool_ids, permission_filtered_tool_ids, + final_offered_tool_ids, call_ids, request_input_hash, + prompt_epoch, prompt_window_id, effective_history_hash, + request_state, provider_state, owner_token, created_at + ) VALUES + ('provider-cross-upgrade-receipt-dispatching', 1, 'provider-cross-upgrade-session', + 'provider-cross-upgrade-input', 'provider-cross-upgrade-attempt-dispatching', + 'provider-cross-upgrade-selection', '{}', '{}', '{}', lower(hex(zeroblob(32))), + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${wireRequestHash}, 0, 'provider-cross-upgrade-window', 'provider-cross-upgrade-history', + 'prepared', 'preparing', 'provider-cross-upgrade-owner', 3), + ('provider-cross-upgrade-receipt-streaming', 2, 'provider-cross-upgrade-session', + 'provider-cross-upgrade-input', 'provider-cross-upgrade-attempt-streaming', + 'provider-cross-upgrade-selection', '{}', '{}', '{}', lower(hex(zeroblob(32))), + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${wireRequestHash}, 0, 'provider-cross-upgrade-window', 'provider-cross-upgrade-history', + 'prepared', 'preparing', 'provider-cross-upgrade-owner', 3), + ('provider-cross-upgrade-receipt-settled', 3, 'provider-cross-upgrade-session', + 'provider-cross-upgrade-input', 'provider-cross-upgrade-attempt-settled', + 'provider-cross-upgrade-selection', '{}', '{}', '{}', lower(hex(zeroblob(32))), + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${wireRequestHash}, 0, 'provider-cross-upgrade-window', 'provider-cross-upgrade-history', + 'prepared', 'preparing', 'provider-cross-upgrade-owner', 3), + ('provider-cross-upgrade-receipt-failed', 4, 'provider-cross-upgrade-session', + 'provider-cross-upgrade-input', 'provider-cross-upgrade-attempt-failed', + 'provider-cross-upgrade-selection', '{}', '{}', '{}', lower(hex(zeroblob(32))), + ${receiptNamespace}, ${receiptProjectScope}, 'unavailable', '[]', + ${emptyReleasedRefsFingerprint}, 'provider', 'model', '[]', '[]', '[]', '[]', + ${wireRequestHash}, 0, 'provider-cross-upgrade-window', 'provider-cross-upgrade-history', + 'prepared', 'preparing', 'provider-cross-upgrade-owner', 3) + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = '[]', + released_knowledge_selected_refs_fingerprint = ${emptyReleasedRefsFingerprint} + WHERE receipt_id LIKE 'provider-cross-upgrade-receipt-%' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET prepared_turn_hash = ${preparedTurnHash}, wire_request_hash = ${wireRequestHash} + WHERE attempt_id LIKE 'provider-cross-upgrade-attempt-%' + `) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', adapter_prepared_at = 4, + final_request_hash = ${wireRequestHash}, provider_request_hash = ${wireRequestHash}, + prepared_turn_hash = ${preparedTurnHash}, system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, wire_request_hash = ${wireRequestHash}, + tool_definition_hash = ${wireRequestHash}, tool_result_reference_ids = '[]', + tool_result_reference_count = 0 + WHERE receipt_id LIKE 'provider-cross-upgrade-receipt-%' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET state = 'dispatching' + WHERE attempt_id LIKE 'provider-cross-upgrade-attempt-%' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET state = 'streaming', first_event_at = 5 + WHERE attempt_id = 'provider-cross-upgrade-attempt-streaming' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET state = 'settled', settled_at = 6 + WHERE attempt_id = 'provider-cross-upgrade-attempt-settled' + `) + yield* db.run(sql` + UPDATE session_provider_attempt + SET state = 'failed', settled_at = 6, error_code = 'provider_failed' + WHERE attempt_id = 'provider-cross-upgrade-attempt-failed' + `) + yield* db.run(sql` + INSERT INTO compaction_run ( + run_id, session_id, from_prompt_epoch, trigger, state, created_at, + source_window_id, source_effective_history_hash, source_message_count, + source_projection_version, continuation_wakeup_at, + continuation_state, continuation_receipt_id, continuation_admitted_at + ) VALUES ( + 'provider-cross-upgrade-continuation', 'provider-cross-upgrade-session', 0, + 'turn_start', 'committed', 5, 'provider-cross-upgrade-window', + 'provider-cross-upgrade-history', 1, 1, 5, + 'admitted', 'provider-cross-upgrade-receipt-dispatching', 5 + ) + `) + + const bindingTrigger = yield* db.get<{ sql: string }>(sql` + SELECT sql + FROM sqlite_master + WHERE type = 'trigger' AND name = 'session_tool_request_receipt_binding_immutable' + `) + expect(bindingTrigger).toBeDefined() + yield* db.run(sql`DROP TRIGGER session_tool_request_receipt_binding_immutable`) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_id = 'mismatched-provider' + WHERE receipt_id = 'provider-cross-upgrade-receipt-dispatching' + `) + + expect( + Exit.isFailure( + yield* DatabaseMigration.applyOnly(db, [providerCrossStateRecoveryMigration]).pipe(Effect.exit), + ), + ).toBe(true) + expect( + yield* db.get(sql` + SELECT provider_state, request_error_code, terminal_at + FROM session_tool_request_receipt + WHERE receipt_id = 'provider-cross-upgrade-receipt-dispatching' + `), + ).toEqual({ provider_state: "prepared", request_error_code: null, terminal_at: null }) + expect( + yield* db.get(sql` + SELECT continuation_state, continuation_error_code + FROM compaction_run + WHERE run_id = 'provider-cross-upgrade-continuation' + `), + ).toEqual({ continuation_state: "admitted", continuation_error_code: null }) + expect( + yield* db.get(sql` + SELECT authority_state, recovery_reason + FROM session_prompt_epoch + WHERE session_id = 'provider-cross-upgrade-session' AND state = 'active' + `), + ).toEqual({ authority_state: "legacy_pending", recovery_reason: null }) + expect( + yield* db.get(sql` + SELECT name FROM sqlite_master + WHERE type = 'trigger' AND name = 'session_tool_request_receipt_cross_state_recovery_guard' + `), + ).toBeUndefined() + + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_id = 'provider' + WHERE receipt_id = 'provider-cross-upgrade-receipt-dispatching' + `) + yield* db.run(sql.raw(bindingTrigger!.sql)) + yield* DatabaseMigration.applyOnly(db, [providerCrossStateRecoveryMigration]) + expect( + yield* db.all(sql` + SELECT provider_state, request_error_code, terminal_at + FROM session_tool_request_receipt + WHERE receipt_id LIKE 'provider-cross-upgrade-receipt-%' + ORDER BY request_ordinal + `), + ).toEqual( + Array.from({ length: 4 }, () => ({ + provider_state: "indeterminate_after_crash", + request_error_code: "provider_started_outcome_unknown_after_process_restart", + terminal_at: expect.any(Number), + })), + ) + expect( + yield* db.all(sql` + SELECT state, error_code, settled_at, first_event_at + FROM session_provider_attempt + WHERE attempt_id LIKE 'provider-cross-upgrade-attempt-%' + ORDER BY provider_turn_seq + `), + ).toEqual([ + { state: "dispatching", error_code: null, settled_at: null, first_event_at: null }, + { state: "streaming", error_code: null, settled_at: null, first_event_at: 5 }, + { state: "settled", error_code: null, settled_at: 6, first_event_at: null }, + { state: "failed", error_code: "provider_failed", settled_at: 6, first_event_at: null }, + ]) + expect( + yield* db.get(sql` + SELECT continuation_state, continuation_receipt_id, continuation_error_code, + continuation_dispatching_at, continuation_terminal_at + FROM compaction_run + WHERE run_id = 'provider-cross-upgrade-continuation' + `), + ).toEqual({ + continuation_state: "indeterminate", + continuation_receipt_id: "provider-cross-upgrade-receipt-dispatching", + continuation_error_code: "provider_started_outcome_unknown_after_process_restart", + continuation_dispatching_at: expect.any(Number), + continuation_terminal_at: expect.any(Number), + }) + expect( + yield* db.get(sql` + SELECT authority_state, recovery_reason + FROM session_prompt_epoch + WHERE session_id = 'provider-cross-upgrade-session' AND state = 'active' + `), + ).toEqual({ + authority_state: "recovery_required", + recovery_reason: "provider outcome is unknown after process restart", + }) + expect( + yield* db.get(sql` + SELECT state, reason + FROM session_history_state + WHERE session_id = 'provider-cross-upgrade-session' + `), + ).toEqual({ + state: "recovery_required", + reason: "provider outcome is unknown after process restart", + }) + }), + ) + }) + + test("enforces durable prompt history recovery state invariants", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* DatabaseMigration.apply(db) + yield* db.run(sql` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('project-history-authority', '/repo', '[]', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session ( + id, project_id, slug, directory, title, version, time_created, time_updated + ) VALUES ( + 'session-history-authority', 'project-history-authority', 'history-authority', '/repo', + 'History authority', '1', 1, 1 + ) + `) + yield* db.run(sql` + INSERT INTO session_prompt_epoch ( + session_id, epoch, state, reason, created_at, authority_state, + projection_version, canonicalization_version, base_message_count, + effective_history_hash, first_window_id, window_id + ) VALUES ( + 'session-history-authority', 0, 'active', 'bootstrap', 1, 'ready', + 1, 1, 0, 'history-hash', 'window-0', 'window-0' + ) + `) + + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_prompt_epoch + SET authority_state = 'recovery_required' + WHERE session_id = 'session-history-authority' AND epoch = 0 + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + INSERT INTO session_history_state ( + session_id, state, reason, time_created, time_updated + ) VALUES ( + 'session-history-authority', 'recovery_required', 'corrupt history', 1, 1 + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + + yield* db.run(sql` + UPDATE session_prompt_epoch + SET authority_state = 'recovery_required', recovery_reason = 'corrupt history' + WHERE session_id = 'session-history-authority' AND epoch = 0 + `) + yield* db.run(sql` + INSERT INTO session_history_state ( + session_id, state, reason, time_created, time_updated + ) VALUES ( + 'session-history-authority', 'recovery_required', 'corrupt history', 1, 1 + ) + `) + + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_prompt_epoch + SET authority_state = 'ready', recovery_reason = NULL + WHERE session_id = 'session-history-authority' AND epoch = 0 + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_history_state + SET reason = NULL + WHERE session_id = 'session-history-authority' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + yield* db.get( + sql` + SELECT authority_state, recovery_reason + FROM session_prompt_epoch + WHERE session_id = 'session-history-authority' AND epoch = 0 + `, + ), + ).toEqual({ authority_state: "recovery_required", recovery_reason: "corrupt history" }) + expect( + yield* db.get( + sql` + SELECT state, reason + FROM session_history_state + WHERE session_id = 'session-history-authority' + `, + ), + ).toEqual({ state: "recovery_required", reason: "corrupt history" }) + + yield* db.run(sql` + INSERT INTO session ( + id, project_id, slug, directory, title, version, time_created, time_updated + ) VALUES ( + 'session-missing-authority', 'project-history-authority', 'missing-authority', '/repo', + 'Missing authority', '1', 2, 2 + ) + `) + yield* db.run(sql` + INSERT INTO message (id, session_id, time_created, time_updated, data) + VALUES ('message-missing-authority', 'session-missing-authority', 2, 2, '{"role":"user"}') + `) + yield* db.run(sql` + INSERT INTO session_prompt_epoch ( + session_id, epoch, state, reason, created_at, authority_state, + projection_version, canonicalization_version, base_message_count, + effective_history_hash, first_window_id, window_id, source_end_message_id + ) VALUES ( + 'session-missing-authority', 0, 'active', 'bootstrap', 2, 'ready', + 1, 1, 0, 'missing-history-hash', 'missing-window-0', 'missing-window-0', + 'message-missing-authority' + ) + `) + yield* db.run(sql`DELETE FROM message WHERE id = 'message-missing-authority'`) + + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE session_prompt_epoch + SET recovery_reason = NULL + WHERE session_id = 'session-missing-authority' AND epoch = 0 + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + yield* db.run(sql` + UPDATE session_prompt_epoch + SET authority_state = 'recovery_required', recovery_reason = 'referenced message missing' + WHERE session_id = 'session-missing-authority' AND epoch = 0 + `) + expect( + yield* db.get(sql` + SELECT authority_state, recovery_reason + FROM session_prompt_epoch + WHERE session_id = 'session-missing-authority' AND epoch = 0 + `), + ).toEqual({ + authority_state: "recovery_required", + recovery_reason: "referenced message missing", + }) + }), + ) + }) + + test("adds nullable Session suspension without inferring historical recovery", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`CREATE TABLE session (id text PRIMARY KEY)`) + yield* db.run(sql`INSERT INTO session (id) VALUES ('historical')`) + + yield* DatabaseMigration.applyOnly(db, [timeSuspendedMigration]) + + expect(yield* db.get(sql`SELECT time_suspended FROM session WHERE id = 'historical'`)).toEqual({ + time_suspended: null, + }) + expect( + yield* db.get( + sql`SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'session_time_suspended_idx'`, + ), + ).toEqual({ name: "session_time_suspended_idx" }) + }), + ) + }) + + test("preserves historical task admission and outbox rows across the L1 rebuild", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`PRAGMA foreign_keys = ON`) + yield* db.run(sql`CREATE TABLE session (id TEXT PRIMARY KEY)`) + yield* db.run(sql`INSERT INTO session (id) VALUES ('ses_parent')`) + yield* DatabaseMigration.applyOnly(db, [taskRunDeliveryMigration]) + yield* db.run(sql` + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + attempts, time_created, time_updated + ) VALUES ( + 'run_historical', 'run_historical', 'request', 'ses_parent', 'msg_parent', + 'call_historical', 'ses_child', 1, 'background', 'research', 'researching', + 2, 100, 200 + ) + `) + yield* db.run(sql` + INSERT INTO task_admission ( + admission_key, request_hash, run_id, parent_session_id, parent_message_id, + tool_call_id, delivery_mode, time_created + ) VALUES ( + 'admission_historical', 'request', 'run_historical', 'ses_parent', 'msg_parent', + 'call_historical', 'background', 100 + ) + `) + yield* db.run(sql` + INSERT INTO task_notification_outbox ( + id, run_id, message_id, parent_session_id, directory, payload, status, + attempts, available_at, time_created, time_updated + ) VALUES ( + 'outbox_historical', 'run_historical', 'msg_outbox', 'ses_parent', '/repo', '{}', + 'delivering', 1, 150, 100, 200 + ) + `) + + yield* DatabaseMigration.applyOnly(db, [subagentControlPlaneMigration]) + + expect( + yield* db.get( + sql`SELECT state, phase, control_state, input_state, workspace_preflight_state, start_attempts FROM task_run WHERE run_id = 'run_historical'`, + ), + ).toEqual({ + state: "running", + phase: "research", + control_state: "open", + input_state: "legacy", + workspace_preflight_state: "legacy", + start_attempts: 2, + }) + expect( + yield* db.get( + sql`SELECT admission_key, origin_kind, origin_key FROM task_admission WHERE run_id = 'run_historical'`, + ), + ).toEqual({ + admission_key: "admission_historical", + origin_kind: "task_tool", + origin_key: "admission_historical", + }) + expect( + yield* db.get( + sql`SELECT status, event_kind, time_admitted FROM task_notification_outbox WHERE run_id = 'run_historical'`, + ), + ).toEqual({ status: "processing", event_kind: "terminal", time_admitted: null }) + const activeIndex = yield* db.get<{ sql: string }>( + sql`SELECT sql FROM sqlite_master WHERE type = 'index' AND name = 'task_run_child_active_idx'`, + ) + expect(activeIndex?.sql).toContain( + "WHERE state IN ('admitted', 'provisioning', 'running', 'researching', 'finalizing')", + ) + expect(activeIndex?.sql).not.toContain("'queued'") + }), + ) + }) + + test("preserves unknown historical structured transport and backfills only exact receipts", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`CREATE TABLE session (id TEXT PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE message (id TEXT PRIMARY KEY, session_id TEXT NOT NULL, data TEXT NOT NULL)`) + yield* db.run(sql`INSERT INTO session (id) VALUES ('ses_structured_parent')`) + yield* DatabaseMigration.applyOnly(db, [taskRunDeliveryMigration, subagentControlPlaneMigration]) + yield* db.run(sql` + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + reason, attempts, raw_result_message_id, structured_result_message_id, output, + time_created, time_updated, time_settled + ) VALUES + ( + 'run_structured', 'run_structured', 'request-structured', 'ses_structured_parent', + 'msg-structured-parent', 'call-structured', 'ses-structured-child', 1, 'foreground', + 'settled', 'completed', 'structured_output_valid', 1, 'msg-structured-research', + 'msg-structured-result', + '{"result":"ok"}', 100, 110, 110 + ), + ( + 'run_text_fallback', 'run_text_fallback', 'request-text', 'ses_structured_parent', + 'msg-text-parent', 'call-text', 'ses-text-child', 1, 'foreground', + 'settled', 'completed', 'structured_output_text_fallback', 2, 'msg-text-research', 'msg-text-result', + '{"result":"ok"}', 100, 120, 120 + ), + ( + 'run_degraded', 'run_degraded', 'request-degraded', 'ses_structured_parent', + 'msg-degraded-parent', 'call-degraded', 'ses-degraded-child', 1, 'foreground', + 'settled', 'completed', 'structured_output_degraded_text', 2, 'msg-degraded-research', NULL, + '{"_degraded":true,"_reason":"structured_output_invalid","_attempts":2,"_raw":"research"}', + 100, 130, 130 + ) + `) + yield* db.run(sql` + INSERT INTO message (id, session_id, data) VALUES + ('msg-text-research', 'ses-text-child', '{"role":"assistant"}'), + ('msg-text-result', 'ses-text-child', '{"role":"assistant"}'), + ('msg-degraded-research', 'ses-degraded-child', '{"role":"assistant"}') + `) + + yield* DatabaseMigration.applyOnly(db, [ + taskStructuredOutputReceiptSchemaMigration, + taskStructuredOutputReceiptMigration, + ]) + + expect( + yield* db.all(sql` + SELECT run_id, structured_output_receipt + FROM task_run + ORDER BY run_id + `), + ).toEqual([ + { + run_id: "run_degraded", + structured_output_receipt: '{"attempt":2,"transport":"degraded_text","reason":"structured_output_invalid"}', + }, + { run_id: "run_structured", structured_output_receipt: null }, + { run_id: "run_text_fallback", structured_output_receipt: '{"attempt":2,"transport":"text_fallback"}' }, + ]) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run + SET structured_output_receipt = '{"attempt":1,"transport":"structured"}' + WHERE run_id = 'run_degraded' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run + SET output = '{"result":"changed"}' + WHERE run_id = 'run_structured' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run + SET time_updated = time_updated + 1 + WHERE run_id = 'run_structured' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(false) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run + SET reason = 'structured_output_text_fallback' + WHERE run_id = 'run_structured' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + reason, attempts, structured_result_message_id, structured_output_receipt, + time_created, time_updated, time_settled + ) VALUES ( + 'run_invalid_receipt', 'run_invalid_receipt', 'request-invalid', + 'ses_structured_parent', 'msg-invalid-parent', 'call-invalid', + 'ses-invalid-child', 1, 'foreground', 'settled', 'completed', + 'structured_output_text_fallback', 2, 'msg-invalid-result', + '{"attempt":1,"transport":"text_fallback"}', 100, 140, 140 + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + }), + ) + }) + + test("rejects ambiguous historical task structured output receipts", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`CREATE TABLE session (id TEXT PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE message (id TEXT PRIMARY KEY, session_id TEXT NOT NULL, data TEXT NOT NULL)`) + yield* db.run(sql`INSERT INTO session (id) VALUES ('ses_ambiguous_structured_parent')`) + yield* DatabaseMigration.applyOnly(db, [taskRunDeliveryMigration, subagentControlPlaneMigration]) + yield* db.run(sql` + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + reason, attempts, structured_result_message_id, output, + time_created, time_updated, time_settled + ) VALUES ( + 'run_ambiguous_degraded', 'run_ambiguous_degraded', 'request-ambiguous', + 'ses_ambiguous_structured_parent', 'msg-ambiguous-parent', 'call-ambiguous', + 'ses-ambiguous-child', 1, 'foreground', 'settled', 'completed', + 'structured_output_degraded_text', 2, NULL, + '{"_degraded":true,"_attempts":2,"_raw":"research"}', 100, 110, 110 + ) + `) + + yield* DatabaseMigration.applyOnly(db, [taskStructuredOutputReceiptSchemaMigration]) + expect( + Exit.isFailure( + yield* DatabaseMigration.applyOnly(db, [taskStructuredOutputReceiptMigration]).pipe(Effect.exit), + ), + ).toBe(true) + expect( + yield* db.all(sql` + SELECT name FROM pragma_table_info('task_run') + WHERE name = 'structured_output_receipt' + `), + ).toEqual([{ name: "structured_output_receipt" }]) + expect( + yield* db.get(sql` + SELECT structured_output_receipt + FROM task_run + WHERE run_id = 'run_ambiguous_degraded' + `), + ).toEqual({ structured_output_receipt: null }) + }), + ) + }) + + test("freezes durable task structured contracts and binds new terminal receipts", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + yield* db.run(sql`CREATE TABLE session (id TEXT PRIMARY KEY)`) + yield* db.run(sql`CREATE TABLE message (id TEXT PRIMARY KEY, session_id TEXT NOT NULL, data TEXT NOT NULL)`) + yield* db.run(sql`INSERT INTO session (id) VALUES ('ses_execution_spec_parent')`) + yield* DatabaseMigration.applyOnly(db, [taskRunDeliveryMigration, subagentControlPlaneMigration]) + yield* db.run(sql` + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + execution_spec, time_created, time_updated + ) VALUES ( + 'run_invalid_execution_spec', 'run_invalid_execution_spec', 'request-invalid-spec', + 'ses_execution_spec_parent', 'msg-invalid-spec-parent', 'call-invalid-spec', + 'ses-invalid-spec-child', 1, 'foreground', 'admission', 'admitted', + '{"structuredOutput":{"schema":{},"allowTextFallback":true,"receiptVersion":1,"maxAttempts":3}}', + 100, 100 + ) + `) + yield* DatabaseMigration.applyOnly(db, [taskStructuredOutputReceiptSchemaMigration]) + yield* DatabaseMigration.applyOnly(db, [taskStructuredOutputReceiptMigration]) + + expect( + Exit.isFailure( + yield* DatabaseMigration.applyOnly(db, [taskExecutionSpecAuthorityMigration]).pipe(Effect.exit), + ), + ).toBe(true) + expect( + yield* db.get(sql` + SELECT json_extract(execution_spec, '$.structuredOutput.maxAttempts') AS max_attempts + FROM task_run + WHERE run_id = 'run_invalid_execution_spec' + `), + ).toEqual({ max_attempts: 3 }) + expect( + yield* db.get(sql` + SELECT name + FROM sqlite_master + WHERE type = 'trigger' AND name = 'task_run_execution_spec_immutable' + `), + ).toBeUndefined() + + yield* db.run(sql` + UPDATE task_run + SET execution_spec = '{"agent":"researcher","model":{"providerID":"test","modelID":"test"},"structuredOutput":{"schema":{"type":"object"},"allowTextFallback":true,"receiptVersion":1,"maxAttempts":2}}' + WHERE run_id = 'run_invalid_execution_spec' + `) + yield* db.run(sql` + INSERT INTO message (id, session_id, data) VALUES + ('msg-valid-research', 'ses-invalid-spec-child', '{"role":"assistant"}'), + ('msg-valid-result', 'ses-invalid-spec-child', '{"role":"assistant"}'), + ('msg-fallback-research', 'ses-no-fallback-child', '{"role":"assistant"}'), + ('msg-fallback-result', 'ses-no-fallback-child', '{"role":"assistant"}') + `) + yield* db.run(sql` + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + execution_spec, time_created, time_updated + ) VALUES + ( + 'run_contract_requires_receipt', 'run_contract_requires_receipt', 'request-contract-receipt', + 'ses_execution_spec_parent', 'msg-contract-parent', 'call-contract-receipt', + 'ses-contract-child', 1, 'foreground', 'admission', 'admitted', + '{"agent":"researcher","model":{"providerID":"test","modelID":"test"},"structuredOutput":{"schema":{"type":"object"},"allowTextFallback":true,"receiptVersion":1,"maxAttempts":2}}', + 100, 100 + ), + ( + 'run_contract_disallows_fallback', 'run_contract_disallows_fallback', 'request-no-fallback', + 'ses_execution_spec_parent', 'msg-no-fallback-parent', 'call-no-fallback', + 'ses-no-fallback-child', 1, 'foreground', 'admission', 'admitted', + '{"agent":"researcher","model":{"providerID":"test","modelID":"test"},"structuredOutput":{"schema":{"type":"object"},"allowTextFallback":false,"receiptVersion":1,"maxAttempts":2}}', + 100, 100 + ), + ( + 'run_historical_contract', 'run_historical_contract', 'request-historical-contract', + 'ses_execution_spec_parent', 'msg-historical-parent', 'call-historical-contract', + 'ses-historical-child', 1, 'foreground', 'settled', 'completed', + '{"agent":"researcher","model":{"providerID":"test","modelID":"test"},"structuredOutput":{"schema":{"type":"object"},"allowTextFallback":true,"receiptVersion":1,"maxAttempts":2}}', + 100, 200 + ) + `) + yield* db.run(sql` + UPDATE task_run + SET reason = 'text_output_valid', attempts = 1, time_settled = 200 + WHERE run_id = 'run_historical_contract' + `) + yield* DatabaseMigration.applyOnly(db, [taskExecutionSpecAuthorityMigration]) + + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run + SET execution_spec = '{"agent":"researcher","model":{"providerID":"test","modelID":"test"},"structuredOutput":{"schema":{"type":"array"},"allowTextFallback":true,"receiptVersion":1,"maxAttempts":2}}' + WHERE run_id = 'run_invalid_execution_spec' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run + SET phase = 'settled', state = 'completed', reason = 'text_output_valid', + attempts = 1, output = 'plain text', time_settled = 200, time_updated = 200 + WHERE run_id = 'run_contract_requires_receipt' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run + SET phase = 'settled', state = 'completed', reason = 'structured_output_text_fallback', + attempts = 2, raw_result_message_id = 'msg-fallback-research', + structured_result_message_id = 'msg-fallback-result', + structured_output_receipt = '{"attempt":2,"transport":"text_fallback"}', + time_settled = 200, time_updated = 200 + WHERE run_id = 'run_contract_disallows_fallback' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run SET time_updated = 201 + WHERE run_id = 'run_historical_contract' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(false) + expect( + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run + SET output = 'historical terminal payload cannot be rewritten' + WHERE run_id = 'run_historical_contract' + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) expect( Exit.isFailure( yield* db .run( sql` - UPDATE part - SET message_id = 'message-provenance-other', session_id = 'session-provenance-other' - WHERE id = 'part-valid-provenance' + UPDATE task_run + SET reason = 'structured_output_valid', structured_result_message_id = 'msg-inferred', + structured_output_receipt = '{"attempt":1,"transport":"structured"}' + WHERE run_id = 'run_historical_contract' `, ) .pipe(Effect.exit), @@ -930,54 +3637,26 @@ describe("DatabaseMigration", () => { yield* db .run( sql` - UPDATE part - SET provenance = '{"source":"compaction_continue","owner_session_id":"session-provenance","owner_prompt_epoch":2,"owner_run_id":"run-1","durable":true}' - WHERE id = 'part-valid-provenance' + UPDATE task_run + SET phase = 'settled', state = 'completed', reason = 'structured_output_valid', + attempts = 1, raw_result_message_id = 'msg-valid-research', + structured_result_message_id = 'msg-valid-result', + structured_output_receipt = '{"attempt":1,"transport":"structured"}', + time_settled = 200, time_updated = 200 + WHERE run_id = 'run_invalid_execution_spec' `, ) .pipe(Effect.exit), ), - ).toBe(true) - }), - ) - }) - - test("enforces durable prompt history recovery state invariants", async () => { - await run( - Effect.gen(function* () { - const db = yield* makeDb - yield* DatabaseMigration.apply(db) - yield* db.run(sql` - INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) - VALUES ('project-history-authority', '/repo', '[]', 1, 1) - `) - yield* db.run(sql` - INSERT INTO session ( - id, project_id, slug, directory, title, version, time_created, time_updated - ) VALUES ( - 'session-history-authority', 'project-history-authority', 'history-authority', '/repo', - 'History authority', '1', 1, 1 - ) - `) - yield* db.run(sql` - INSERT INTO session_prompt_epoch ( - session_id, epoch, state, reason, created_at, authority_state, - projection_version, canonicalization_version, base_message_count, - effective_history_hash, first_window_id, window_id - ) VALUES ( - 'session-history-authority', 0, 'active', 'bootstrap', 1, 'ready', - 1, 1, 0, 'history-hash', 'window-0', 'window-0' - ) - `) - + ).toBe(false) expect( Exit.isFailure( yield* db .run( sql` - UPDATE session_prompt_epoch - SET authority_state = 'recovery_required' - WHERE session_id = 'session-history-authority' AND epoch = 0 + UPDATE task_run + SET output = '{"result":"rewritten"}' + WHERE run_id = 'run_invalid_execution_spec' `, ) .pipe(Effect.exit), @@ -988,38 +3667,39 @@ describe("DatabaseMigration", () => { yield* db .run( sql` - INSERT INTO session_history_state ( - session_id, state, reason, time_created, time_updated + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + reason, attempts, structured_result_message_id, structured_output_receipt, + time_created, time_updated, time_settled ) VALUES ( - 'session-history-authority', 'recovery_required', 'corrupt history', 1, 1 + 'run_unbound_receipt', 'run_unbound_receipt', 'request-unbound-receipt', + 'ses_execution_spec_parent', 'msg-unbound-parent', 'call-unbound-receipt', + 'ses-unbound-child', 1, 'foreground', 'settled', 'completed', + 'structured_output_valid', 1, 'msg-unbound-result', + '{"attempt":1,"transport":"structured"}', 100, 200, 200 ) `, ) .pipe(Effect.exit), ), ).toBe(true) - - yield* db.run(sql` - UPDATE session_prompt_epoch - SET authority_state = 'recovery_required', recovery_reason = 'corrupt history' - WHERE session_id = 'session-history-authority' AND epoch = 0 - `) - yield* db.run(sql` - INSERT INTO session_history_state ( - session_id, state, reason, time_created, time_updated - ) VALUES ( - 'session-history-authority', 'recovery_required', 'corrupt history', 1, 1 - ) - `) - expect( Exit.isFailure( yield* db .run( sql` - UPDATE session_prompt_epoch - SET authority_state = 'ready', recovery_reason = NULL - WHERE session_id = 'session-history-authority' AND epoch = 0 + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + execution_spec, time_created, time_updated + ) VALUES ( + 'run_invalid_new_spec', 'run_invalid_new_spec', 'request-invalid-new-spec', + 'ses_execution_spec_parent', 'msg-invalid-new-parent', 'call-invalid-new-spec', + 'ses-invalid-new-child', 1, 'foreground', 'admission', 'admitted', + '{"structuredOutput":{"schema":{},"allowTextFallback":1,"receiptVersion":1,"maxAttempts":2}}', + 300, 300 + ) `, ) .pipe(Effect.exit), @@ -1030,184 +3710,153 @@ describe("DatabaseMigration", () => { yield* db .run( sql` - UPDATE session_history_state - SET reason = NULL - WHERE session_id = 'session-history-authority' + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + execution_spec, time_created, time_updated + ) VALUES ( + 'run_missing_structured_identity', 'run_missing_structured_identity', + 'request-missing-structured-identity', 'ses_execution_spec_parent', + 'msg-missing-structured-identity-parent', 'call-missing-structured-identity', + 'ses-missing-structured-identity-child', 1, 'foreground', 'admission', 'admitted', + '{"structuredOutput":{"schema":{},"allowTextFallback":true,"receiptVersion":1,"maxAttempts":2}}', + 300, 300 + ) `, ) .pipe(Effect.exit), ), ).toBe(true) - expect( - yield* db.get( - sql` - SELECT authority_state, recovery_reason - FROM session_prompt_epoch - WHERE session_id = 'session-history-authority' AND epoch = 0 - `, - ), - ).toEqual({ authority_state: "recovery_required", recovery_reason: "corrupt history" }) - expect( - yield* db.get( - sql` - SELECT state, reason - FROM session_history_state - WHERE session_id = 'session-history-authority' - `, - ), - ).toEqual({ state: "recovery_required", reason: "corrupt history" }) + }), + ) + }) + test("rejects pre-authority structured evidence and binds exact material snapshots", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const authorityIndex = migrations.findIndex( + (migration) => migration.id === taskStructuredOutputEvidenceAuthorityMigration.id, + ) + expect(authorityIndex).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, authorityIndex)) + const now = Date.now() + yield* db.run(sql` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('project-evidence', '/project-evidence', '[]', ${now}, ${now}) + `) yield* db.run(sql` INSERT INTO session ( id, project_id, slug, directory, title, version, time_created, time_updated - ) VALUES ( - 'session-missing-authority', 'project-history-authority', 'missing-authority', '/repo', - 'Missing authority', '1', 2, 2 - ) + ) VALUES + ('session-evidence-parent', 'project-evidence', 'parent', '/project-evidence', 'parent', 'test', ${now}, ${now}), + ('session-evidence-child', 'project-evidence', 'child', '/project-evidence', 'child', 'test', ${now}, ${now}) `) yield* db.run(sql` - INSERT INTO message (id, session_id, time_created, time_updated, data) - VALUES ('message-missing-authority', 'session-missing-authority', 2, 2, '{"role":"user"}') + INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES + ('message-evidence-raw', 'session-evidence-child', ${now}, ${now}, '{"role":"assistant"}'), + ('message-evidence-result', 'session-evidence-child', ${now}, ${now}, '{"role":"assistant"}') `) yield* db.run(sql` - INSERT INTO session_prompt_epoch ( - session_id, epoch, state, reason, created_at, authority_state, - projection_version, canonicalization_version, base_message_count, - effective_history_hash, first_window_id, window_id, source_end_message_id + INSERT INTO part (id, message_id, session_id, time_created, time_updated, data) VALUES + ('part-evidence-raw', 'message-evidence-raw', 'session-evidence-child', ${now}, ${now}, + '{"type":"text","text":"research"}'), + ('part-evidence-result', 'message-evidence-result', 'session-evidence-child', ${now}, ${now}, + '{"type":"text","text":"{\\"result\\":\\"ok\\"}"}') + `) + const contract = '{"schema":{"type":"object"},"allowTextFallback":true,"receiptVersion":1,"maxAttempts":2}' + yield* db.run(sql` + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, + tool_call_id, child_session_id, generation, delivery_mode, phase, state, + attempts, execution_owner, lease_expires_at, execution_spec, version, + time_created, time_updated ) VALUES ( - 'session-missing-authority', 0, 'active', 'bootstrap', 2, 'ready', - 1, 1, 0, 'missing-history-hash', 'missing-window-0', 'missing-window-0', - 'message-missing-authority' + 'run-evidence', 'run-evidence', 'request-evidence', 'session-evidence-parent', + 'message-evidence-parent', 'call-evidence', 'session-evidence-child', 1, + 'foreground', 'finalize', 'finalizing', 1, 'evidence-owner', ${now + 60_000}, + ${`{"prompt":{"text":"inspect"},"agent":"researcher","model":{"providerID":"test","modelID":"test"},"structuredOutput":${contract}}`}, + 3, ${now}, ${now} ) `) - yield* db.run(sql`DELETE FROM message WHERE id = 'message-missing-authority'`) + const preAuthorityInsert = yield* db + .run( + sql` + INSERT INTO task_structured_output_evidence ( + run_id, child_session_id, owner_token, claim_generation, expected_version, + terminal_state, attempts, contract_json, raw_result_message_id, raw_message_json, + raw_parts_json, result_message_id, result_message_json, result_parts_json, + output, structured_output_receipt, failure_code, created_at + ) VALUES ( + 'run-evidence', 'session-evidence-child', 'evidence-owner', 0, 3, + 'completed', 1, ${contract}, 'message-evidence-raw', '{"role":"assistant"}', + '[]', 'message-evidence-result', '{"role":"assistant"}', '[]', + '{"result":"ok"}', '{"attempt":1,"transport":"structured"}', NULL, ${now} + ) + `, + ) + .pipe(Effect.exit) + expect(Exit.isSuccess(preAuthorityInsert)).toBe(true) expect( Exit.isFailure( - yield* db - .run( - sql` - UPDATE session_prompt_epoch - SET recovery_reason = NULL - WHERE session_id = 'session-missing-authority' AND epoch = 0 - `, - ) - .pipe(Effect.exit), + yield* DatabaseMigration.applyOnly(db, [taskStructuredOutputEvidenceAuthorityMigration]).pipe(Effect.exit), ), ).toBe(true) - yield* db.run(sql` - UPDATE session_prompt_epoch - SET authority_state = 'recovery_required', recovery_reason = 'referenced message missing' - WHERE session_id = 'session-missing-authority' AND epoch = 0 - `) expect( yield* db.get(sql` - SELECT authority_state, recovery_reason - FROM session_prompt_epoch - WHERE session_id = 'session-missing-authority' AND epoch = 0 + SELECT name FROM sqlite_master + WHERE type = 'trigger' AND name = 'task_structured_output_evidence_insert_guard' `), - ).toEqual({ - authority_state: "recovery_required", - recovery_reason: "referenced message missing", - }) - }), - ) - }) - - test("adds nullable Session suspension without inferring historical recovery", async () => { - await run( - Effect.gen(function* () { - const db = yield* makeDb - yield* db.run(sql`CREATE TABLE session (id text PRIMARY KEY)`) - yield* db.run(sql`INSERT INTO session (id) VALUES ('historical')`) - - yield* DatabaseMigration.applyOnly(db, [timeSuspendedMigration]) - - expect(yield* db.get(sql`SELECT time_suspended FROM session WHERE id = 'historical'`)).toEqual({ - time_suspended: null, - }) - expect( - yield* db.get( - sql`SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'session_time_suspended_idx'`, - ), - ).toEqual({ name: "session_time_suspended_idx" }) - }), - ) - }) + ).toBeUndefined() - test("preserves historical task admission and outbox rows across the L1 rebuild", async () => { - await run( - Effect.gen(function* () { - const db = yield* makeDb - yield* db.run(sql`PRAGMA foreign_keys = ON`) - yield* db.run(sql`CREATE TABLE session (id TEXT PRIMARY KEY)`) - yield* db.run(sql`INSERT INTO session (id) VALUES ('ses_parent')`) - yield* DatabaseMigration.applyOnly(db, [taskRunDeliveryMigration]) - yield* db.run(sql` - INSERT INTO task_run ( - run_id, root_run_id, request_hash, parent_session_id, parent_message_id, - tool_call_id, child_session_id, generation, delivery_mode, phase, state, - attempts, time_created, time_updated - ) VALUES ( - 'run_historical', 'run_historical', 'request', 'ses_parent', 'msg_parent', - 'call_historical', 'ses_child', 1, 'background', 'research', 'researching', - 2, 100, 200 - ) - `) + yield* db.run(sql`DELETE FROM task_structured_output_evidence WHERE run_id = 'run-evidence'`) + yield* DatabaseMigration.applyOnly(db, [taskStructuredOutputEvidenceAuthorityMigration]) yield* db.run(sql` - INSERT INTO task_admission ( - admission_key, request_hash, run_id, parent_session_id, parent_message_id, - tool_call_id, delivery_mode, time_created + INSERT INTO task_structured_output_evidence ( + run_id, child_session_id, owner_token, claim_generation, expected_version, + terminal_state, attempts, contract_json, raw_result_message_id, raw_message_json, + raw_parts_json, result_message_id, result_message_json, result_parts_json, + output, structured_output_receipt, failure_code, created_at ) VALUES ( - 'admission_historical', 'request', 'run_historical', 'ses_parent', 'msg_parent', - 'call_historical', 'background', 100 + 'run-evidence', 'session-evidence-child', 'evidence-owner', 0, 3, + 'completed', 1, ${contract}, 'message-evidence-raw', '{"role":"assistant"}', + '[]', 'message-evidence-result', '{"role":"assistant"}', '[]', + '{"result":"ok"}', '{"attempt":1,"transport":"structured"}', NULL, ${now} ) `) yield* db.run(sql` - INSERT INTO task_notification_outbox ( - id, run_id, message_id, parent_session_id, directory, payload, status, - attempts, available_at, time_created, time_updated - ) VALUES ( - 'outbox_historical', 'run_historical', 'msg_outbox', 'ses_parent', '/repo', '{}', - 'delivering', 1, 150, 100, 200 - ) + INSERT INTO task_structured_output_evidence_part ( + run_id, role, ordinal, part_id, message_id, session_id, part_json + ) VALUES + ('run-evidence', 'raw', 0, 'part-evidence-raw', 'message-evidence-raw', + 'session-evidence-child', '{"type":"text","text":"research"}'), + ('run-evidence', 'result', 0, 'part-evidence-result', 'message-evidence-result', + 'session-evidence-child', '{"type":"text","text":"{\\"result\\":\\"ok\\"}"}') `) - - yield* DatabaseMigration.applyOnly(db, [subagentControlPlaneMigration]) - - expect( - yield* db.get( - sql`SELECT state, phase, control_state, input_state, workspace_preflight_state, start_attempts FROM task_run WHERE run_id = 'run_historical'`, - ), - ).toEqual({ - state: "running", - phase: "research", - control_state: "open", - input_state: "legacy", - workspace_preflight_state: "legacy", - start_attempts: 2, - }) expect( - yield* db.get( - sql`SELECT admission_key, origin_kind, origin_key FROM task_admission WHERE run_id = 'run_historical'`, + Exit.isFailure( + yield* db + .run( + sql` + UPDATE task_run SET + phase = 'settled', state = 'completed', reason = 'structured_output_valid', attempts = 1, + raw_result_message_id = 'message-evidence-raw', + structured_result_message_id = 'message-evidence-result', + structured_output_receipt = '{"attempt":1,"transport":"structured"}', + output = '{"result":"ok"}', execution_owner = NULL, lease_expires_at = NULL, + version = 4, time_updated = ${now + 1}, time_settled = ${now + 1} + WHERE run_id = 'run-evidence' + `, + ) + .pipe(Effect.exit), ), - ).toEqual({ - admission_key: "admission_historical", - origin_kind: "task_tool", - origin_key: "admission_historical", + ).toBe(true) + expect(yield* db.get(sql`SELECT state, phase, version FROM task_run WHERE run_id = 'run-evidence'`)).toEqual({ + state: "finalizing", + phase: "finalize", + version: 3, }) - expect( - yield* db.get( - sql`SELECT status, event_kind, time_admitted FROM task_notification_outbox WHERE run_id = 'run_historical'`, - ), - ).toEqual({ status: "processing", event_kind: "terminal", time_admitted: null }) - const activeIndex = yield* db.get<{ sql: string }>( - sql`SELECT sql FROM sqlite_master WHERE type = 'index' AND name = 'task_run_child_active_idx'`, - ) - expect(activeIndex?.sql).toContain( - "WHERE state IN ('admitted', 'provisioning', 'running', 'researching', 'finalizing')", - ) - expect(activeIndex?.sql).not.toContain("'queued'") }), ) }) @@ -1995,3 +4644,33 @@ describe("DatabaseMigration", () => { ) }) }) + +function seedReceiptScope(db: Database.Interface["db"]) { + return Effect.gen(function* () { + yield* db.run(sql` + INSERT INTO context_security_namespace (id, kind, binding_hash, created_at) + VALUES (${receiptNamespace}, 'implicit_local', 'receipt-namespace-binding', 1) + `) + yield* db.run(sql` + INSERT INTO context_project_scope_identity ( + security_namespace_id, project_scope_key, project_kind, project_identity_hash, created_at + ) VALUES ( + ${receiptNamespace}, ${receiptProjectScope}, 'registered_root', 'receipt-project-identity', 1 + ) + `) + }) +} + +function contextActivation(recordedAt: number, selectionId?: string, projectionHash?: string) { + return JSON.stringify({ + schemaVersion: 1, + recordedAt, + readinessAgeMs: recordedAt, + readinessExpiresInMs: contextReadiness.expiresAt - recordedAt, + outcome: "not_requested", + enabledCapabilities: [], + fallbackReasons: [], + decision: contextEligibility, + ...(selectionId && projectionHash ? { selection: { selectionId, projectionHash } } : {}), + }) +} diff --git a/packages/core/test/released-knowledge-turn-binding-migration.test.ts b/packages/core/test/released-knowledge-turn-binding-migration.test.ts new file mode 100644 index 000000000..894288dc5 --- /dev/null +++ b/packages/core/test/released-knowledge-turn-binding-migration.test.ts @@ -0,0 +1,873 @@ +import { describe, expect, test } from "bun:test" +import { SqliteClient } from "@effect/sql-sqlite-bun" +import { EffectDrizzleSqlite } from "@deepagent-code/effect-drizzle-sqlite" +import { sql } from "drizzle-orm" +import { Effect, Exit } from "effect" +import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" +import { Database } from "../src/database/database" +import { DatabaseMigration } from "../src/database/migration" +import releasedKnowledgeTurnBindingMigration from "../src/database/migration/20260811193000_released_knowledge_turn_binding" +import { DeepAgentReleasedSnapshot, type Binding } from "../src/deepagent/released-snapshot" + +const emptyExactRefsFingerprint = "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" +const activationFingerprint = "a".repeat(64) +const preparedTurnHash = "b".repeat(64) +const systemStableHash = "c".repeat(64) +const systemVolatileHash = "d".repeat(64) +const wireRequestHash = "e".repeat(64) +const namespace = "binding-namespace" +const primaryScope = "binding-primary-scope" +const otherScope = "binding-other-scope" +const sessionId = "binding-session" +const selectionId = "binding-selection" +const ownerToken = "binding-provider-owner" +const contextEligibility = { + requested: false, + project: false, + enabled: { + contextProjectionV2: false, + contextQueryToolsV2: false, + coreV2ExecutionOwner: false, + }, +} +const contextReadiness = { + observedAt: 0, + expiresAt: 10_000, +} + +describe("released knowledge turn binding migration", () => { + test("rejects direct advanced-state inserts and permits a canonical unavailable receipt to dispatch", async () => { + await runCurrent( + Effect.gen(function* () { + const db = (yield* Database.Service).db + yield* seedAuthority(db) + + const directInsertResults = yield* Effect.all( + (["dispatching", "streaming"] as const).map((providerState, index) => + insertReceipt(db, { + id: `direct-${providerState}`, + ordinal: index + 1, + providerState, + securityNamespaceId: namespace, + projectScopeKey: primaryScope, + }).pipe(Effect.exit), + ), + ) + expect(directInsertResults.every(Exit.isFailure)).toBe(true) + + yield* insertReceipt(db, { + id: "canonical-unavailable", + ordinal: 3, + providerState: "preparing", + securityNamespaceId: namespace, + projectScopeKey: primaryScope, + binding: DeepAgentReleasedSnapshot.binding(undefined), + activation: true, + }) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = '[]', + released_knowledge_selected_refs_fingerprint = ${emptyExactRefsFingerprint} + WHERE receipt_id = 'canonical-unavailable' + `) + const unsealed = yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared' + WHERE receipt_id = 'canonical-unavailable' + `, + ) + .pipe(Effect.exit) + expect(Exit.isFailure(unsealed)).toBe(true) + const duplicateToolResults = yield* sealPreparedTurn(db, "canonical-unavailable", ["call-1", "call-1"]).pipe( + Effect.exit, + ) + expect(Exit.isFailure(duplicateToolResults)).toBe(true) + yield* sealPreparedTurn(db, "canonical-unavailable", []) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'dispatching', request_state = 'dispatched', dispatching_at = 3 + WHERE receipt_id = 'canonical-unavailable' + `) + + expect( + yield* db.get(sql` + SELECT provider_state, released_knowledge_binding_state + FROM session_tool_request_receipt + WHERE receipt_id = 'canonical-unavailable' + `), + ).toEqual({ provider_state: "dispatching", released_knowledge_binding_state: "unavailable" }) + }), + ) + }) + + test("seals one canonical selected-ref subset and rejects non-members, non-canonical order, and rewrites", async () => { + await runCurrent( + Effect.gen(function* () { + const db = (yield* Database.Service).db + const binding = yield* seedAuthority(db) + const selected = binding.exactRefs.slice(0, 1) + yield* insertReceipt(db, { + id: "bound-selected-subset", + ordinal: 1, + providerState: "preparing", + securityNamespaceId: namespace, + projectScopeKey: primaryScope, + binding, + activation: true, + }) + + const nonMember = yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = ${JSON.stringify([ + { ...binding.exactRefs[0], id: "not-in-snapshot" }, + ])}, + released_knowledge_selected_refs_fingerprint = ${"b".repeat(64)} + WHERE receipt_id = 'bound-selected-subset' + `, + ) + .pipe(Effect.exit) + expect(Exit.isFailure(nonMember)).toBe(true) + + const nonCanonical = yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = ${JSON.stringify([...binding.exactRefs].reverse())}, + released_knowledge_selected_refs_fingerprint = ${binding.exactRefsFingerprint} + WHERE receipt_id = 'bound-selected-subset' + `, + ) + .pipe(Effect.exit) + expect(Exit.isFailure(nonCanonical)).toBe(true) + + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = ${JSON.stringify(selected)}, + released_knowledge_selected_refs_fingerprint = ${DeepAgentReleasedSnapshot.exactRefsFingerprint(selected)} + WHERE receipt_id = 'bound-selected-subset' + `) + const rewrite = yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET released_knowledge_selected_refs = '[]', + released_knowledge_selected_refs_fingerprint = ${emptyExactRefsFingerprint} + WHERE receipt_id = 'bound-selected-subset' + `, + ) + .pipe(Effect.exit) + expect(Exit.isFailure(rewrite)).toBe(true) + + yield* sealPreparedTurn(db, "bound-selected-subset", ["call-1"]) + const rewritePreparedTurn = yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET system_stable_hash = ${"f".repeat(64)} + WHERE receipt_id = 'bound-selected-subset' + `, + ) + .pipe(Effect.exit) + expect(Exit.isFailure(rewritePreparedTurn)).toBe(true) + yield* db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'dispatching' + WHERE receipt_id = 'bound-selected-subset' + `) + expect( + yield* db.get(sql` + SELECT provider_state, released_knowledge_selected_refs + FROM session_tool_request_receipt + WHERE receipt_id = 'bound-selected-subset' + `), + ).toEqual({ provider_state: "dispatching", released_knowledge_selected_refs: JSON.stringify(selected) }) + }), + ) + }) + + test("rejects malformed bound receipts, wrong scope, and selection binding mismatch", async () => { + await runCurrent( + Effect.gen(function* () { + const db = (yield* Database.Service).db + const binding = yield* seedAuthority(db) + + const invalidReceipts = [ + insertReceipt(db, { + id: "wrong-fingerprint", + ordinal: 1, + providerState: "preparing", + securityNamespaceId: namespace, + projectScopeKey: primaryScope, + binding, + exactRefsFingerprint: "b".repeat(64), + }), + insertReceipt(db, { + id: "extra-json-key", + ordinal: 2, + providerState: "preparing", + securityNamespaceId: namespace, + projectScopeKey: primaryScope, + binding, + exactRefsJson: JSON.stringify(binding.exactRefs.map((ref) => ({ ...ref, extra: true }))), + }), + insertReceipt(db, { + id: "wrong-scope", + ordinal: 3, + providerState: "preparing", + securityNamespaceId: namespace, + projectScopeKey: otherScope, + binding, + }), + ] + expect( + (yield* Effect.all(invalidReceipts.map((receipt) => receipt.pipe(Effect.exit)))).every(Exit.isFailure), + ).toBe(true) + + yield* seedSelection(db, DeepAgentReleasedSnapshot.binding(undefined)) + const mismatchedSelection = yield* insertReceipt(db, { + id: "selection-binding-mismatch", + ordinal: 4, + providerState: "preparing", + securityNamespaceId: namespace, + projectScopeKey: primaryScope, + contextSelectionId: selectionId, + binding, + }).pipe(Effect.exit) + expect(Exit.isFailure(mismatchedSelection)).toBe(true) + }), + ) + }) + + test("rejects provider attempts from another selection or session", async () => { + await runCurrent( + Effect.gen(function* () { + const db = (yield* Database.Service).db + const binding = yield* seedAuthority(db) + yield* seedSelection(db, binding) + yield* db.run(sql` + INSERT INTO session_context_selection ( + selection_id, session_id, activity_id, revision, trigger_input_id, location_key, + security_namespace_id, project_scope_key, + query_fingerprint, authorization_fingerprint, authorization_epoch, + execution_fingerprint, selected_source_fingerprint, + observed_location_mutation_epoch, next_revalidation_at, + released_knowledge_binding_state, released_knowledge_snapshot_id, + released_knowledge_generation, released_knowledge_membership_hash, + released_knowledge_manifest_hash, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + graph_revisions, graph_statuses, selected_refs, projection, + projection_hash, token_count, artifact_write_status, artifact_ref, created_at + ) + SELECT + 'binding-selection-other', session_id, activity_id, revision + 1, trigger_input_id, location_key, + security_namespace_id, project_scope_key, + query_fingerprint, authorization_fingerprint, authorization_epoch, + execution_fingerprint, selected_source_fingerprint, + observed_location_mutation_epoch, next_revalidation_at, + released_knowledge_binding_state, released_knowledge_snapshot_id, + released_knowledge_generation, released_knowledge_membership_hash, + released_knowledge_manifest_hash, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + graph_revisions, graph_statuses, selected_refs, projection, + projection_hash, token_count, artifact_write_status, artifact_ref, created_at + FROM session_context_selection + WHERE selection_id = ${selectionId} + `) + yield* db.run(sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, owner_token, state, created_at + ) VALUES ( + 'attempt-other-selection', ${sessionId}, 'binding-activity', 0, 'binding-selection-other', + 'projection-hash', 'request-hash', 'provider', ${ownerToken}, 'prepared', 1 + ) + `) + yield* db.run(sql` + INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) + VALUES ('binding-session-other', 'binding-project', 'binding-session-other', '/tmp/binding-primary', 'Other', 'test', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session_provider_attempt ( + attempt_id, session_id, activity_id, provider_turn_seq, selection_id, + projection_hash, request_hash, provider_id, owner_token, state, created_at + ) VALUES ( + 'attempt-other-session', 'binding-session-other', 'binding-activity', 0, ${selectionId}, + 'projection-hash', 'request-hash', 'provider', ${ownerToken}, 'prepared', 1 + ) + `) + + const results = yield* Effect.all([ + insertReceipt(db, { + id: "receipt-attempt-other-selection", + ordinal: 1, + providerState: "preparing", + securityNamespaceId: namespace, + projectScopeKey: primaryScope, + contextSelectionId: selectionId, + providerAttemptId: "attempt-other-selection", + requestInputHash: "request-hash", + binding, + }).pipe(Effect.exit), + insertReceipt(db, { + id: "receipt-attempt-other-session", + ordinal: 2, + providerState: "preparing", + securityNamespaceId: namespace, + projectScopeKey: primaryScope, + contextSelectionId: selectionId, + providerAttemptId: "attempt-other-session", + requestInputHash: "request-hash", + binding, + }).pipe(Effect.exit), + ]) + expect(results.every(Exit.isFailure)).toBe(true) + }), + ) + }) + + test("backfills historical selections and receipts as legacy_unbound and prevents dispatch", async () => { + await runRaw( + Effect.gen(function* () { + const db = yield* EffectDrizzleSqlite.makeWithDefaults() + yield* createHistoricalSchema(db) + yield* db.run(sql` + INSERT INTO session_activity (activity_id, state, created_at) + VALUES ('historical-activity', 'active', 1) + `) + yield* db.run(sql` + INSERT INTO session_context_selection (selection_id, session_id, activity_id, location_key) + VALUES ('historical-selection', 'historical-session', 'historical-activity', 'historical-location') + `) + yield* db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, session_id, context_selection_id, provider_state, + final_request_hash, adapter_prepared_at, prompt_epoch, prompt_window_id, + effective_history_hash, context_eligibility, context_readiness, + context_activation, context_activation_fingerprint + ) VALUES ( + 'historical-receipt', 'historical-session', 'historical-selection', 'prepared', + 'historical-final-hash', 1, 1, 'historical-window', + 'historical-history-hash', '{}', '{}', '{}', ${activationFingerprint} + ) + `) + + yield* DatabaseMigration.applyOnly(db, [releasedKnowledgeTurnBindingMigration]) + + expect( + yield* db.get(sql` + SELECT released_knowledge_binding_state + FROM session_context_selection + WHERE selection_id = 'historical-selection' + `), + ).toEqual({ released_knowledge_binding_state: "legacy_unbound" }) + expect( + yield* db.get(sql` + SELECT state, settled_at + FROM session_activity + WHERE activity_id = 'historical-activity' + `), + ).toEqual({ state: "interrupted", settled_at: 1 }) + expect( + yield* db.get(sql` + SELECT released_knowledge_binding_state + FROM session_tool_request_receipt + WHERE receipt_id = 'historical-receipt' + `), + ).toEqual({ released_knowledge_binding_state: "legacy_unbound" }) + + const dispatch = yield* db + .run( + sql` + UPDATE session_tool_request_receipt + SET provider_state = 'dispatching' + WHERE receipt_id = 'historical-receipt' + `, + ) + .pipe(Effect.exit) + expect(Exit.isFailure(dispatch)).toBe(true) + }), + ) + }) + + test("exposes the required columns, partial indexes, triggers, and valid foreign keys", async () => { + await runCurrent( + Effect.gen(function* () { + const db = (yield* Database.Service).db + const expectedBindingColumns = [ + "released_knowledge_binding_state", + "released_knowledge_snapshot_id", + "released_knowledge_generation", + "released_knowledge_membership_hash", + "released_knowledge_manifest_hash", + "released_knowledge_exact_refs", + "released_knowledge_exact_refs_fingerprint", + "released_knowledge_selected_refs", + "released_knowledge_selected_refs_fingerprint", + ] + const receiptColumns = (yield* db.all<{ name: string }>(sql` + SELECT name FROM pragma_table_info('session_tool_request_receipt') + `)).map((column) => column.name) + const selectionColumns = (yield* db.all<{ name: string }>(sql` + SELECT name FROM pragma_table_info('session_context_selection') + `)).map((column) => column.name) + expect(receiptColumns).toEqual( + expect.arrayContaining([ + ...expectedBindingColumns, + "released_knowledge_security_namespace_id", + "released_knowledge_project_scope_key", + "prepared_turn_hash", + "system_stable_hash", + "system_volatile_hash", + "wire_request_hash", + "tool_result_reference_ids", + "tool_result_reference_count", + ]), + ) + expect(selectionColumns).toEqual( + expect.arrayContaining([ + ...expectedBindingColumns.filter((column) => !column.includes("selected_refs")), + "security_namespace_id", + "project_scope_key", + ]), + ) + + expect( + yield* db.all<{ name: string; partial: number }>(sql` + SELECT name, partial + FROM pragma_index_list('session_tool_request_receipt') + WHERE name = 'session_tool_request_receipt_released_snapshot_idx' + `), + ).toEqual([{ name: "session_tool_request_receipt_released_snapshot_idx", partial: 1 }]) + expect( + yield* db.all<{ name: string; partial: number }>(sql` + SELECT name, partial + FROM pragma_index_list('session_context_selection') + WHERE name = 'session_context_selection_released_snapshot_idx' + `), + ).toEqual([{ name: "session_context_selection_released_snapshot_idx", partial: 1 }]) + + expect( + (yield* db.all<{ name: string }>(sql` + SELECT name + FROM sqlite_master + WHERE type = 'trigger' + AND name IN ( + 'session_tool_request_receipt_released_knowledge_immutable', + 'session_tool_request_receipt_released_knowledge_selected_refs_seal', + 'session_tool_request_receipt_authority_insert_guard', + 'session_tool_request_receipt_released_knowledge_dispatch_guard', + 'session_tool_request_receipt_prepared_turn_insert_guard', + 'session_tool_request_receipt_prepared_turn_seal', + 'session_context_selection_immutable', + 'session_context_selection_released_knowledge_insert_guard' + ) + ORDER BY name + `)).map((trigger) => trigger.name), + ).toEqual([ + "session_context_selection_immutable", + "session_context_selection_released_knowledge_insert_guard", + "session_tool_request_receipt_authority_insert_guard", + "session_tool_request_receipt_prepared_turn_insert_guard", + "session_tool_request_receipt_prepared_turn_seal", + "session_tool_request_receipt_released_knowledge_dispatch_guard", + "session_tool_request_receipt_released_knowledge_immutable", + "session_tool_request_receipt_released_knowledge_selected_refs_seal", + ]) + + expect( + yield* db.all<{ table: string; from: string; to: string }>(sql` + SELECT "table", "from", "to" + FROM pragma_foreign_key_list('session_tool_request_receipt') + WHERE "from" = 'released_knowledge_snapshot_id' + `), + ).toEqual([ + { + table: "released_knowledge_snapshot", + from: "released_knowledge_snapshot_id", + to: "snapshot_id", + }, + ]) + expect( + yield* db.all<{ table: string; from: string; to: string }>(sql` + SELECT "table", "from", "to" + FROM pragma_foreign_key_list('session_context_selection') + WHERE "from" = 'released_knowledge_snapshot_id' + `), + ).toEqual([ + { + table: "released_knowledge_snapshot", + from: "released_knowledge_snapshot_id", + to: "snapshot_id", + }, + ]) + expect(yield* db.all(sql`PRAGMA foreign_key_check`)).toEqual([]) + }), + ) + }) +}) + +function runCurrent(effect: Effect.Effect) { + return Effect.runPromise(effect.pipe(Effect.provide(Database.layerFromPath(":memory:")), Effect.scoped)) +} + +function runRaw(effect: Effect.Effect) { + return Effect.runPromise( + effect.pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), Effect.scoped), + ) +} + +type DatabaseClient = Database.Interface["db"] + +function seedAuthority(db: DatabaseClient) { + return Effect.gen(function* () { + yield* db.run(sql` + INSERT INTO session_provider_owner_lease ( + owner_token, registered_at, heartbeat_at, lease_expires_at + ) VALUES ( + ${ownerToken}, + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER), + CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + 30000 + ) + `) + yield* db.run(sql` + INSERT INTO context_security_namespace (id, kind, binding_hash, created_at) + VALUES (${namespace}, 'implicit_local', 'binding-authority', 1) + `) + yield* db.run(sql` + INSERT INTO context_project_scope_identity ( + security_namespace_id, project_scope_key, project_kind, + project_identity_hash, observed_project_id, created_at + ) VALUES + (${namespace}, ${primaryScope}, 'registered_root', 'primary-identity', 'project-primary', 1), + (${namespace}, ${otherScope}, 'registered_root', 'other-identity', 'project-other', 1) + `) + yield* db.run(sql` + INSERT INTO context_location_identity ( + security_namespace_id, location_key, project_scope_key, + canonical_root, observed_project_id, created_at + ) VALUES ( + ${namespace}, 'binding-location', ${primaryScope}, + '/tmp/binding-primary', 'project-primary', 1 + ) + `) + const binding = DeepAgentReleasedSnapshot.binding( + yield* DeepAgentReleasedSnapshot.publish( + db, + { + snapshotId: "binding-snapshot", + evaluationId: "binding-evaluation", + scope: { + securityNamespaceId: namespace, + projectScopeKey: primaryScope, + legacyProjectId: "project-primary", + }, + expectedParentSnapshotId: null, + expectedGeneration: 0, + releaseKind: "legacy_baseline", + verdict: "passed", + documents: [ + { + sourceStore: "project", + id: "binding-document", + version: 1, + hash: `sha256:${"c".repeat(64)}`, + type: "knowledge", + scope: "durable:project:project-primary", + }, + { + sourceStore: "project", + id: "binding-skill", + version: 1, + hash: `sha256:${"d".repeat(64)}`, + type: "skill", + scope: "durable:project:project-primary", + }, + ], + evaluationMatrix: { score: 1 }, + baselineRef: "binding-migration-test", + repetitions: 1, + actor: { type: "system", id: "test" }, + now: 1, + }, + { + userGlobal: { get: () => null }, + project: { + get: (id, version) => { + const ref = [ + { + sourceStore: "project" as const, + id: "binding-document", + version: 1, + hash: `sha256:${"c".repeat(64)}`, + type: "knowledge" as const, + scope: "durable:project:project-primary", + }, + { + sourceStore: "project" as const, + id: "binding-skill", + version: 1, + hash: `sha256:${"d".repeat(64)}`, + type: "skill" as const, + scope: "durable:project:project-primary", + }, + ].find((document) => document.id === id && document.version === version) + return ref + ? { + ...ref, + status: "active" as const, + superseded_by: null, + created_round: null, + domain: null, + tags: [], + description: ref.id, + provenance: { source: "human" as const }, + links: [], + body: ref.id, + } + : null + }, + }, + }, + ), + ) + if (binding.state !== "bound") return yield* Effect.die("expected a released snapshot binding") + return binding + }) +} + +function seedSelection(db: DatabaseClient, binding: Binding) { + return Effect.gen(function* () { + yield* db.run(sql` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('binding-project', '/tmp/binding-primary', '[]', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) + VALUES (${sessionId}, 'binding-project', 'binding-session', '/tmp/binding-primary', 'Binding session', 'test', 1, 1) + `) + yield* db.run(sql` + INSERT INTO session_input (id, session_id, prompt, delivery, admitted_seq, promoted_seq, time_created) + VALUES ('binding-input', ${sessionId}, '{"text":"binding"}', 'steer', 0, 0, 1) + `) + yield* db.run(sql` + INSERT INTO session_activity ( + activity_id, session_id, ordinal, trigger_input_id, delivery, state, created_at + ) VALUES ('binding-activity', ${sessionId}, 0, 'binding-input', 'steer', 'active', 1) + `) + yield* db.run(sql` + INSERT INTO session_context_selection ( + selection_id, session_id, activity_id, revision, trigger_input_id, location_key, + security_namespace_id, project_scope_key, + query_fingerprint, authorization_fingerprint, authorization_epoch, + execution_fingerprint, selected_source_fingerprint, + observed_location_mutation_epoch, next_revalidation_at, + released_knowledge_binding_state, released_knowledge_snapshot_id, + released_knowledge_generation, released_knowledge_membership_hash, + released_knowledge_manifest_hash, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint, + graph_revisions, graph_statuses, selected_refs, projection, + projection_hash, token_count, artifact_write_status, artifact_ref, created_at + ) VALUES ( + ${selectionId}, ${sessionId}, 'binding-activity', 0, 'binding-input', 'binding-location', + ${namespace}, ${primaryScope}, + 'query', 'authorization', 1, 'execution', 'source', 0, 100, + ${binding.state}, ${binding.state === "bound" ? binding.snapshotId : null}, + ${binding.state === "bound" ? binding.generation : null}, + ${binding.state === "bound" ? binding.membershipHash : null}, + ${binding.state === "bound" ? binding.manifestHash : null}, + ${JSON.stringify(binding.exactRefs)}, ${binding.exactRefsFingerprint}, + '{}', '{}', '[]', 'projection', 'projection-hash', 1, 'available', 'artifact-ref', 1 + ) + `) + }) +} + +function insertReceipt( + db: DatabaseClient, + input: { + readonly id: string + readonly ordinal: number + readonly providerState: "preparing" | "dispatching" | "streaming" + readonly securityNamespaceId: string + readonly projectScopeKey: string + readonly contextSelectionId?: string + readonly providerAttemptId?: string + readonly binding?: Binding + readonly exactRefsJson?: string + readonly exactRefsFingerprint?: string + readonly activation?: boolean + readonly requestInputHash?: string + }, +) { + return db.run(sql` + INSERT INTO session_tool_request_receipt ( + receipt_id, request_ordinal, session_id, user_message_id, provider_id, model_id, + registry_tool_ids, permission_filtered_tool_ids, final_offered_tool_ids, call_ids, + provider_state, request_state, created_at, context_selection_id, provider_attempt_id, + owner_token, request_input_hash, + final_request_hash, adapter_prepared_at, prompt_epoch, prompt_window_id, + effective_history_hash, context_eligibility, context_readiness, + context_activation, context_activation_fingerprint, + released_knowledge_security_namespace_id, released_knowledge_project_scope_key, + released_knowledge_binding_state, released_knowledge_snapshot_id, + released_knowledge_generation, released_knowledge_membership_hash, + released_knowledge_manifest_hash, released_knowledge_exact_refs, + released_knowledge_exact_refs_fingerprint + ) VALUES ( + ${input.id}, ${input.ordinal}, ${sessionId}, 'binding-user-message', 'provider', 'model', + '[]', '[]', '[]', '[]', + ${input.providerState}, 'prepared', 1, ${input.contextSelectionId ?? null}, ${input.providerAttemptId ?? null}, + ${ownerToken}, ${input.requestInputHash ?? null}, + NULL, NULL, + ${input.activation ? 1 : null}, ${input.activation ? "prompt-window" : null}, + ${input.activation ? "history-hash" : null}, ${input.activation ? JSON.stringify(contextEligibility) : null}, + ${input.activation ? JSON.stringify(contextReadiness) : null}, + ${input.activation ? contextActivation(input.contextSelectionId) : null}, + ${input.activation ? activationFingerprint : null}, + ${input.securityNamespaceId}, ${input.projectScopeKey}, + ${input.binding?.state ?? null}, + ${input.binding?.state === "bound" ? input.binding.snapshotId : null}, + ${input.binding?.state === "bound" ? input.binding.generation : null}, + ${input.binding?.state === "bound" ? input.binding.membershipHash : null}, + ${input.binding?.state === "bound" ? input.binding.manifestHash : null}, + ${input.exactRefsJson ?? (input.binding ? JSON.stringify(input.binding.exactRefs) : null)}, + ${input.exactRefsFingerprint ?? input.binding?.exactRefsFingerprint ?? null} + ) + `) +} + +function contextActivation(contextSelectionId?: string) { + return JSON.stringify({ + schemaVersion: 1, + recordedAt: 1, + readinessAgeMs: 1, + readinessExpiresInMs: contextReadiness.expiresAt - 1, + outcome: "not_requested", + enabledCapabilities: [], + fallbackReasons: [], + decision: contextEligibility, + ...(contextSelectionId + ? { selection: { selectionId: contextSelectionId, projectionHash: "projection-hash" } } + : {}), + }) +} + +function sealPreparedTurn(db: DatabaseClient, receiptID: string, toolResultReferenceIDs: readonly string[]) { + return db.run(sql` + UPDATE session_tool_request_receipt + SET provider_state = 'prepared', + final_request_hash = ${wireRequestHash}, + provider_request_hash = ${wireRequestHash}, + adapter_prepared_at = 2, + tool_definition_hash = ${preparedTurnHash}, + prepared_turn_hash = ${preparedTurnHash}, + system_stable_hash = ${systemStableHash}, + system_volatile_hash = ${systemVolatileHash}, + wire_request_hash = ${wireRequestHash}, + tool_result_reference_ids = ${JSON.stringify(toolResultReferenceIDs)}, + tool_result_reference_count = ${toolResultReferenceIDs.length} + WHERE receipt_id = ${receiptID} + `) +} + +function createHistoricalSchema(db: DatabaseClient) { + return Effect.gen(function* () { + yield* db.run(sql` + CREATE TABLE released_knowledge_snapshot ( + snapshot_id TEXT PRIMARY KEY, + security_namespace_id TEXT, + project_scope_key TEXT, + published_generation INTEGER, + membership_hash TEXT, + manifest_hash TEXT, + verdict TEXT, + finalized_at INTEGER, + document_count INTEGER + ) + `) + yield* db.run(sql` + CREATE TABLE released_knowledge_snapshot_document ( + snapshot_id TEXT, + ordinal INTEGER, + source_store TEXT, + doc_id TEXT, + doc_version INTEGER, + doc_hash TEXT, + doc_type TEXT, + doc_scope TEXT + ) + `) + yield* db.run(sql` + CREATE TABLE context_project_scope_identity ( + security_namespace_id TEXT, + project_scope_key TEXT, + retired_at INTEGER + ) + `) + yield* db.run(sql` + CREATE TABLE context_location_identity ( + security_namespace_id TEXT, + project_scope_key TEXT, + location_key TEXT, + retired_at INTEGER + ) + `) + yield* db.run(sql` + CREATE TABLE session_activity ( + activity_id TEXT PRIMARY KEY, + session_id TEXT, + ordinal INTEGER, + trigger_input_id TEXT, + delivery TEXT, + state TEXT, + created_at INTEGER, + settled_at INTEGER + ) + `) + yield* db.run(sql` + CREATE TABLE session_context_selection ( + selection_id TEXT PRIMARY KEY, + session_id TEXT, + activity_id TEXT, + location_key TEXT + ) + `) + yield* db.run(sql` + CREATE TRIGGER session_context_selection_immutable + BEFORE UPDATE ON session_context_selection + BEGIN + SELECT RAISE(ABORT, 'session_context_selection is immutable'); + END + `) + yield* db.run(sql` + CREATE TABLE session_provider_attempt ( + attempt_id TEXT PRIMARY KEY, + session_id TEXT, + selection_id TEXT + ) + `) + yield* db.run(sql` + CREATE TABLE session_tool_request_receipt ( + receipt_id TEXT PRIMARY KEY, + session_id TEXT, + context_selection_id TEXT, + provider_attempt_id TEXT, + provider_state TEXT, + final_request_hash TEXT, + adapter_prepared_at INTEGER, + prompt_epoch INTEGER, + prompt_window_id TEXT, + effective_history_hash TEXT, + context_eligibility TEXT, + context_readiness TEXT, + context_activation TEXT, + context_activation_fingerprint TEXT + ) + `) + }) +} diff --git a/packages/core/test/task-structured-finalizer-response-migration.test.ts b/packages/core/test/task-structured-finalizer-response-migration.test.ts new file mode 100644 index 000000000..3db347c63 --- /dev/null +++ b/packages/core/test/task-structured-finalizer-response-migration.test.ts @@ -0,0 +1,160 @@ +import { describe, expect, test } from "bun:test" +import { SqliteClient } from "@effect/sql-sqlite-bun" +import { EffectDrizzleSqlite } from "@deepagent-code/effect-drizzle-sqlite" +import { Cause, Effect, Exit } from "effect" +import { DatabaseMigration } from "@deepagent-code/core/database/migration" +import { migrations } from "@deepagent-code/core/database/migration.gen" +import responseMigration from "@deepagent-code/core/database/migration/20260812223000_task_structured_finalizer_response" +import type { SqlClient as SqlClientService } from "effect/unstable/sql/SqlClient" + +const makeDb = EffectDrizzleSqlite.makeWithDefaults() +const run = (effect: Effect.Effect) => + Effect.runPromise( + effect.pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), Effect.scoped), + ) + +describe("task structured finalizer response migration", () => { + test("rejects mismatched identities and freezes an exact response chain", async () => { + await run( + Effect.gen(function* () { + const db = yield* makeDb + const index = migrations.findIndex((migration) => migration.id === responseMigration.id) + expect(index).toBeGreaterThan(0) + yield* DatabaseMigration.applyOnly(db, migrations.slice(0, index)) + const now = Date.now() + for (const statement of [ + ` + INSERT INTO project (id, worktree, sandboxes, time_created, time_updated) + VALUES ('project-response', '/project-response', '[]', ${now}, ${now})`, + ` + INSERT INTO session (id, project_id, slug, directory, title, version, time_created, time_updated) + VALUES + ('session-response-parent', 'project-response', 'parent', '/project-response', 'parent', 'test', ${now}, ${now}), + ('session-response-child', 'project-response', 'child', '/project-response', 'child', 'test', ${now}, ${now})`, + ` + INSERT INTO message (id, session_id, time_created, time_updated, data) VALUES + ('message-response-raw', 'session-response-child', ${now}, ${now}, '{"role":"assistant"}'), + ('message-response-request', 'session-response-child', ${now}, ${now}, + '{"role":"user","metadata":{"deepagent":{"structured_finalizer":{"run_id":"run-response","attempt":1,"source_message_id":"message-response-raw","allow_text":false}}}}'), + ('message-response-bad-metadata-request', 'session-response-child', ${now}, ${now}, + '{"role":"user","metadata":{"deepagent":{"structured_finalizer":{"run_id":"another-run","attempt":1,"source_message_id":"message-response-raw","allow_text":false}}}}'), + ('message-response-bad-metadata-result', 'session-response-child', ${now}, ${now}, + '{"role":"assistant","parentID":"message-response-bad-metadata-request","structured":{"result":"bad"}}'), + ('message-response-result', 'session-response-child', ${now}, ${now}, + '{"role":"assistant","parentID":"message-response-request","structured":{"result":"ok"}}')`, + ` + INSERT INTO part (id, message_id, session_id, time_created, time_updated, data) + VALUES ('part-response-result', 'message-response-result', 'session-response-child', ${now}, ${now}, + '{"type":"text","text":"ok"}')`, + ` + INSERT INTO task_run ( + run_id, root_run_id, request_hash, parent_session_id, parent_message_id, tool_call_id, + child_session_id, generation, delivery_mode, phase, state, attempts, raw_result_message_id, + execution_owner, lease_expires_at, claim_generation, execution_spec, version, control_state, + time_created, time_updated + ) VALUES ( + 'run-response', 'run-response', 'request-response', 'session-response-parent', + 'message-response-parent', 'call-response', 'session-response-child', 1, 'foreground', + 'finalize', 'finalizing', 1, 'message-response-raw', 'response-owner', ${now + 60_000}, 2, + '{"prompt":{"text":"inspect"},"agent":"researcher","model":{"providerID":"test","modelID":"test"},"structuredOutput":{"schema":{"type":"object"},"allowTextFallback":true,"receiptVersion":1,"maxAttempts":2}}', + 3, 'open', ${now}, ${now} + ) + `, + ]) { + yield* db.run(statement) + } + yield* DatabaseMigration.applyOnly(db, [responseMigration]) + expect( + yield* db.get( + "SELECT name FROM sqlite_master WHERE type = 'trigger' AND name = 'task_structured_finalizer_response_insert_guard'", + ), + ).toEqual({ name: "task_structured_finalizer_response_insert_guard" }) + expect( + yield* db.get(` + SELECT + run.state AS state, + run.control_state AS control_state, + run.interrupt_requested_at AS interrupt_requested_at, + run.child_session_id AS child_session_id, + run.execution_owner AS execution_owner, + run.claim_generation AS claim_generation, + run.version AS version, + run.attempts AS attempts, + run.raw_result_message_id AS source_message_id, + response.data AS response_message_json, + json_extract(response.data, '$.parentID') AS parent_id, + json_extract(request.data, '$.metadata.deepagent.structured_finalizer.run_id') AS metadata_run_id + FROM task_run run, message request, message response + WHERE run.run_id = 'run-response' + AND request.id = 'message-response-request' + AND response.id = 'message-response-result' + `), + ).toMatchObject({ + state: "finalizing", + control_state: "open", + interrupt_requested_at: null, + child_session_id: "session-response-child", + execution_owner: "response-owner", + claim_generation: 2, + version: 3, + attempts: 1, + source_message_id: "message-response-raw", + response_message_json: + '{"role":"assistant","parentID":"message-response-request","structured":{"result":"ok"}}', + parent_id: "message-response-request", + metadata_run_id: "run-response", + }) + + const insert = (requestMessageID: string, sourceMessageID = "message-response-raw") => + db.run(` + INSERT INTO task_structured_finalizer_response ( + run_id, attempt, child_session_id, owner_token, claim_generation, expected_version, + source_message_id, request_message_id, response_message_id, response_message_json, created_at + ) VALUES ( + 'run-response', 1, 'session-response-child', 'response-owner', 2, 3, + '${sourceMessageID}', '${requestMessageID}', 'message-response-result', + '{"role":"assistant","parentID":"message-response-request","structured":{"result":"ok"}}', ${now} + ) + `) + expect(Exit.isFailure(yield* insert("message-response-raw").pipe(Effect.exit))).toBe(true) + expect(Exit.isFailure(yield* insert("message-response-request", "mismatched-source").pipe(Effect.exit))).toBe( + true, + ) + expect( + Exit.isFailure( + yield* db + .run( + ` + INSERT INTO task_structured_finalizer_response ( + run_id, attempt, child_session_id, owner_token, claim_generation, expected_version, + source_message_id, request_message_id, response_message_id, response_message_json, created_at + ) VALUES ( + 'run-response', 1, 'session-response-child', 'response-owner', 2, 3, + 'message-response-raw', 'message-response-bad-metadata-request', + 'message-response-bad-metadata-result', + '{"role":"assistant","parentID":"message-response-bad-metadata-request","structured":{"result":"bad"}}', + ${now} + ) + `, + ) + .pipe(Effect.exit), + ), + ).toBe(true) + const legitimate = yield* insert("message-response-request").pipe(Effect.exit) + expect(Exit.isSuccess(legitimate), Exit.isFailure(legitimate) ? Cause.pretty(legitimate.cause) : "").toBe(true) + + for (const mutation of [ + "UPDATE message SET data = '{\"role\":\"assistant\"}' WHERE id = 'message-response-result'", + "DELETE FROM message WHERE id = 'message-response-request'", + `INSERT INTO part (id, message_id, session_id, time_created, time_updated, data) + VALUES ('part-response-extra', 'message-response-result', 'session-response-child', ${now}, ${now}, + '{"type":"text","text":"extra"}')`, + 'UPDATE part SET data = \'{"type":"text","text":"changed"}\' WHERE id = \'part-response-result\'', + "DELETE FROM part WHERE id = 'part-response-result'", + ]) { + expect(Exit.isFailure(yield* db.run(mutation).pipe(Effect.exit))).toBe(true) + } + }), + ) + }) +}) diff --git a/packages/deepagent-code/src/session/tool-request-receipt.sql.ts b/packages/deepagent-code/src/session/tool-request-receipt.sql.ts index a789c70cb..fa5815863 100644 --- a/packages/deepagent-code/src/session/tool-request-receipt.sql.ts +++ b/packages/deepagent-code/src/session/tool-request-receipt.sql.ts @@ -1,6 +1,10 @@ // BUG-009: drizzle-orm type bindings for session_tool_request_receipt. // One row per physical Provider dispatch — maps the registry→filter→wire pipeline. import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core" +import { ContextFederationRollout } from "@deepagent-code/core/context-federation/rollout" +import type { ContextActivationReceipt } from "@/context-federation/activation-receipt" +import type { DocumentRef, StoredBindingState } from "@deepagent-code/core/deepagent/released-snapshot" +import type { ProjectScopeKey, SecurityNamespaceID } from "@deepagent-code/core/context-federation/reference" export type RequestReceiptState = "prepared" | "dispatched" | "rejected" export type ProviderReceiptState = @@ -21,6 +25,22 @@ export const SessionToolRequestReceiptTable = sqliteTable("session_tool_request_ user_message_id: text().notNull(), assistant_message_id: text(), provider_attempt_id: text(), + context_selection_id: text(), + context_eligibility: text({ mode: "json" }).$type(), + context_readiness: text({ mode: "json" }).$type(), + context_activation: text({ mode: "json" }).$type(), + context_activation_fingerprint: text(), + released_knowledge_security_namespace_id: text().$type(), + released_knowledge_project_scope_key: text().$type(), + released_knowledge_binding_state: text().$type(), + released_knowledge_snapshot_id: text(), + released_knowledge_generation: integer(), + released_knowledge_membership_hash: text(), + released_knowledge_manifest_hash: text(), + released_knowledge_exact_refs: text({ mode: "json" }).$type(), + released_knowledge_exact_refs_fingerprint: text(), + released_knowledge_selected_refs: text({ mode: "json" }).$type(), + released_knowledge_selected_refs_fingerprint: text(), provider_id: text().notNull(), model_id: text().notNull(), protocol: text(), @@ -47,6 +67,12 @@ export const SessionToolRequestReceiptTable = sqliteTable("session_tool_request_ response_chain_refusal_reason: text(), request_input_hash: text(), final_request_hash: text(), + prepared_turn_hash: text(), + system_stable_hash: text(), + system_volatile_hash: text(), + wire_request_hash: text(), + tool_result_reference_ids: text({ mode: "json" }).$type().notNull().default([]), + tool_result_reference_count: integer().notNull().default(0), provider_state: text().$type().notNull().default("preparing"), adapter_prepared_at: integer(), dispatching_at: integer(), From 8ce6495324673ba921538b711ed8b6b5378dd9d5 Mon Sep 17 00:00:00 2001 From: deepagent-ai Date: Fri, 14 Aug 2026 00:26:57 +0800 Subject: [PATCH 32/44] feat(core): make released knowledge authoritative --- .../review/dialog-review-contract.test.ts | 66 +- .../components/review/dialog-review.api.ts | 31 +- .../src/components/review/dialog-review.tsx | 25 +- packages/core/src/deepagent/document-store.ts | 189 +++- .../src/deepagent/durable-knowledge-store.ts | Bin 18387 -> 29411 bytes .../core/src/deepagent/knowledge-retriever.ts | 269 ++++-- .../core/src/deepagent/knowledge-source.ts | 294 +++++- packages/core/src/deepagent/orchestrator.ts | 9 +- packages/core/src/deepagent/promotion.ts | 89 +- packages/core/src/deepagent/prompt-policy.ts | 26 +- .../core/src/deepagent/released-snapshot.ts | 890 ++++++++++++++++++ packages/core/src/deepagent/run-graph.ts | 4 +- packages/core/src/deepagent/session-state.ts | 8 +- .../test/deepagent/approval-status.test.ts | 44 +- .../deepagent/document-governance.test.ts | 181 ++++ .../test/deepagent/document-store-writer.ts | 18 + .../test/deepagent/document-store.test.ts | 43 +- .../domain-pack-detector-competition.test.ts | 12 +- .../domain-pack-retrieval-isolation.test.ts | 88 +- .../core/test/deepagent/domain-pack.test.ts | 10 +- .../deepagent/durable-knowledge-store.test.ts | 138 ++- .../deepagent/knowledge-contract-v3-2.test.ts | 126 ++- .../deepagent/knowledge-retriever-v3.test.ts | 83 +- .../core/test/deepagent/promotion.test.ts | 94 +- .../deepagent/released-selection-fixture.ts | 28 + .../test/deepagent/released-snapshot.test.ts | 830 ++++++++++++++++ .../deepagent/ship-gate-enforcement.test.ts | 40 +- .../workspace-isolation-read-path.test.ts | 88 +- .../instance/httpapi/groups/deepagent.ts | 145 ++- .../instance/httpapi/handlers/deepagent.ts | 432 +++++++-- .../test/deepagent/knowledge-mode.test.ts | 31 + .../server/httpapi-deepagent-release.test.ts | 695 ++++++++++++++ .../httpapi-exercise/scenarios/deepagent.ts | 32 +- 33 files changed, 4634 insertions(+), 424 deletions(-) create mode 100644 packages/core/src/deepagent/released-snapshot.ts create mode 100644 packages/core/test/deepagent/document-governance.test.ts create mode 100644 packages/core/test/deepagent/document-store-writer.ts create mode 100644 packages/core/test/deepagent/released-selection-fixture.ts create mode 100644 packages/core/test/deepagent/released-snapshot.test.ts create mode 100644 packages/deepagent-code/test/server/httpapi-deepagent-release.test.ts diff --git a/packages/app/src/components/review/dialog-review-contract.test.ts b/packages/app/src/components/review/dialog-review-contract.test.ts index 0a3b509ba..049fb47fe 100644 --- a/packages/app/src/components/review/dialog-review-contract.test.ts +++ b/packages/app/src/components/review/dialog-review-contract.test.ts @@ -4,8 +4,8 @@ import { listPending, reviewSummary, setStatus, listEnvFacts, decideEnvFact, mod // P1-C route contract: the V3.1 self-learning Review dialog talks to the raw-request escape-hatch // routes (NOT the generated SDK). These assertions lock the exact method/url/body so a backend // rename of /deepagent/knowledge/{pending,approve,reject-ids} or a payload shape change breaks CI -// here instead of silently shipping a dead Review UI. Mirrors the backend group schema -// (DeepAgentKnowledgeStatusInput = { ids: string[] }). +// here instead of silently shipping a dead Review UI. Review decisions must round-trip the exact +// immutable authority returned by the list endpoint; a bare id can alias project/global revisions. type Recorded = { method: string; url: string; body?: unknown; headers?: Record } function client(calls: Recorded[], data: unknown) { @@ -24,7 +24,13 @@ describe("DeepAgent review dialog route contract", () => { const calls: Recorded[] = [] const items = [ { + sourceStore: "project" as const, id: "memory:1", + version: 3, + hash: "hash-3", + candidateId: "candidate-3", + fingerprint: "fingerprint-3", + governanceRevision: "governance-3", type: "memory" as const, summary: "s", evidence_strength: "strong" as const, @@ -49,29 +55,73 @@ describe("DeepAgent review dialog route contract", () => { expect(calls).toEqual([{ method: "GET", url: "/deepagent/knowledge/review-summary" }]) }) - test("approve POSTs /deepagent/knowledge/approve with { ids }", async () => { + test("approve POSTs the exact listed authority to /deepagent/knowledge/approve", async () => { const calls: Recorded[] = [] - await setStatus(client(calls, { updated: ["a"] }), "approve", ["a", "b"]) + const item = { + sourceStore: "project" as const, + id: "a", + version: 2, + hash: "hash-a-2", + candidateId: "candidate-a", + fingerprint: "fingerprint-a", + governanceRevision: "governance-a-2", + type: "knowledge" as const, + summary: "A", + evidence_strength: "strong" as const, + evidence_refs: [], + approval_status: "pending" as const, + } + await setStatus(client(calls, { updated: item }), "approve", item) expect(calls).toEqual([ { method: "POST", url: "/deepagent/knowledge/approve", - body: { ids: ["a", "b"] }, + body: { + sourceStore: "project", + id: "a", + version: 2, + hash: "hash-a-2", + candidateId: "candidate-a", + fingerprint: "fingerprint-a", + expectedGovernanceRevision: "governance-a-2", + }, headers: { "Content-Type": "application/json" }, }, ]) }) - test("reject POSTs /deepagent/knowledge/reject-ids with { ids }", async () => { + test("reject POSTs the exact global authority to /deepagent/knowledge/reject-ids", async () => { const calls: Recorded[] = [] - await setStatus(client(calls, { updated: ["a"] }), "reject-ids", ["a"]) + const item = { + sourceStore: "user_global" as const, + id: "a", + version: 4, + hash: "hash-a-4", + candidateId: "candidate-a", + fingerprint: "fingerprint-a", + governanceRevision: "governance-a-4", + type: "memory" as const, + summary: "A", + evidence_strength: "medium" as const, + evidence_refs: [], + approval_status: "pending" as const, + } + await setStatus(client(calls, { updated: item }), "reject-ids", item) expect(calls).toEqual([ { method: "POST", url: "/deepagent/knowledge/reject-ids", - body: { ids: ["a"] }, + body: { + sourceStore: "user_global", + id: "a", + version: 4, + hash: "hash-a-4", + candidateId: "candidate-a", + fingerprint: "fingerprint-a", + expectedGovernanceRevision: "governance-a-4", + }, headers: { "Content-Type": "application/json" }, }, ]) diff --git a/packages/app/src/components/review/dialog-review.api.ts b/packages/app/src/components/review/dialog-review.api.ts index 169ef00f1..1a99c2e28 100644 --- a/packages/app/src/components/review/dialog-review.api.ts +++ b/packages/app/src/components/review/dialog-review.api.ts @@ -4,7 +4,13 @@ // for back-compat. Keep this file free of any solid-js/UI imports. export type KnowledgeItem = { + sourceStore: "user_global" | "project" id: string + version: number + hash: string + candidateId: string + fingerprint: string + governanceRevision: string type: "knowledge" | "strategy" | "methodology" | "memory" | "skill" | "failure_dossier" summary: string evidence_strength: "strong" | "medium" | "weak" | "none" @@ -47,16 +53,35 @@ export const reviewSummary = async (client: ReviewClient): Promise<{ pendingCoun export const setStatus = async ( client: ReviewClient, action: "approve" | "reject-ids", - ids: string[], + item: KnowledgeItem, ): Promise => { - await client.client.request<{ updated: string[] }>({ + await client.client.request<{ updated: KnowledgeItem }>({ method: "POST", url: `/deepagent/knowledge/${action}`, - body: { ids }, + body: { + sourceStore: item.sourceStore, + id: item.id, + version: item.version, + hash: item.hash, + candidateId: item.candidateId, + fingerprint: item.fingerprint, + expectedGovernanceRevision: item.governanceRevision, + }, headers: { "Content-Type": "application/json" }, }) } +export const reviewAuthorityKey = (item: KnowledgeItem) => + JSON.stringify([ + item.sourceStore, + item.id, + item.version, + item.hash, + item.candidateId, + item.fingerprint, + item.governanceRevision, + ]) + // V3.8.1 §G environment-fact use-gate. Provisional user-global environment facts surface here so the // user decides, per project, whether to adopt them (§G.5). Credentials never appear — only secret_ref // pointers. `degraded` marks a fact whose last connection attempt failed (§G.6). diff --git a/packages/app/src/components/review/dialog-review.tsx b/packages/app/src/components/review/dialog-review.tsx index c3c8a1656..b34f3a41e 100644 --- a/packages/app/src/components/review/dialog-review.tsx +++ b/packages/app/src/components/review/dialog-review.tsx @@ -11,6 +11,7 @@ import { listEnvFacts, decideEnvFact, modifyEnvFact, + reviewAuthorityKey, type KnowledgeItem, type ReviewClient, type EnvFactItem, @@ -25,6 +26,7 @@ export { listEnvFacts, decideEnvFact, modifyEnvFact, + reviewAuthorityKey, type KnowledgeItem, type ReviewClient, type EnvFactBody, @@ -187,33 +189,36 @@ export const DialogReview: Component<{ client: ReviewClient }> = (props) => { setCollapsed(next) } - const toggle = (id: string) => { + const toggle = (item: KnowledgeItem) => { + const id = reviewAuthorityKey(item) const next = new Set(selected()) if (next.has(id)) next.delete(id) else next.add(id) setSelected(next) } - const selectAll = () => setSelected(new Set(pending().map((i) => i.id))) + const selectAll = () => setSelected(new Set(pending().map(reviewAuthorityKey))) const invert = () => { const cur = selected() setSelected( new Set( pending() - .map((i) => i.id) + .map(reviewAuthorityKey) .filter((id) => !cur.has(id)), ), ) } const apply = async (action: "approve" | "reject-ids") => { - const ids = [...selected()] - if (ids.length === 0 || busy()) return + const decisions = (items() ?? []).filter((item) => selected().has(reviewAuthorityKey(item))) + if (decisions.length === 0 || busy()) return setBusy(true) try { - await setStatus(props.client, action, ids) + await Promise.all(decisions.map((item) => setStatus(props.client, action, item))) setSelected(new Set()) await refetch() } catch (error) { + setSelected(new Set()) + await refetch() showToast({ variant: "error", title: language.t("review.title"), @@ -225,14 +230,14 @@ export const DialogReview: Component<{ client: ReviewClient }> = (props) => { } const Row = (item: KnowledgeItem) => { - const checked = createMemo(() => selected().has(item.id)) + const checked = createMemo(() => selected().has(reviewAuthorityKey(item))) return (