diff --git a/docs/reference/specs/write-receipts.md b/docs/reference/specs/write-receipts.md index cc8dddc59..e14848288 100644 --- a/docs/reference/specs/write-receipts.md +++ b/docs/reference/specs/write-receipts.md @@ -12,6 +12,7 @@ An operator can read the original storage transaction result after its reply is | Live-state effects and their exact original receipt commit together. A lost reply can be inspected without repeating the write. Private state is absent from the reply. | `[unit]` `deploy/cloudflare-memory/writeReceipts.test.ts::original write receipts::reads a committed live-state write after its original reply is discarded` | | The ingress digest binds actual frozen HTTP bytes. A diagnostic header never supplies body proof. | `[unit]` `deploy/cloudflare-memory/writeReceipts.test.ts::original write receipt failures::derives request identity from bytes instead of diagnostic headers` | | Session receipts distinguish rows, attachments, source metadata and source-owner metadata. A refused result can still have committed metadata effects. | `[unit]` `deploy/cloudflare-memory/writeReceipts.test.ts::original write receipts::records row and source components together without exposing their payloads`; `::original write receipt components and limits::records the original * components`; `::original write receipt failures::records a refused result separately from committed source metadata` | +| A lost session batch reply continues to the run step only after the exact current-original receipt records success and the requested row and attachment counts. The client never resends that batch; missing, changed, partial, refused or unreadable proof keeps the original write uncertain. | `[unit]` `src/core/runLedgerWorker.test.ts::a session step after a lost write reply::*` | | Receipt insert or prune failure rolls back all effects in the owning transaction. No receipt can attest a rolled-back write. | `[unit]` `deploy/cloudflare-memory/writeReceipts.test.ts::original write receipt failures::rolls back source and row effects if receipt * fails` | | A retained duplicate request ID is fenced before mutation. A conflicting digest cannot replace its original receipt. | `[unit]` `deploy/cloudflare-memory/writeReceipts.test.ts::original write receipt failures::fences duplicate and conflicting IDs without replacing the original receipt` | | A frozen effect input is followed by owner and revocation checks inside the transaction. A local owner change during hashing saves no success receipt. RunHistory and SessionLog remain separate atomic boundaries. | `[unit]` `deploy/cloudflare-memory/writeReceipts.test.ts::original write receipt failures::refuses a stale * owner changed during hashing`; `::original write receipt components and limits::refuses a revocation that occurs during session effect hashing`; `::keeps a RunHistory-only owner change separate from SessionLog commit` | diff --git a/src/core/runLedgerWorker.test.ts b/src/core/runLedgerWorker.test.ts index 6ca198ca8..b147d4c94 100644 --- a/src/core/runLedgerWorker.test.ts +++ b/src/core/runLedgerWorker.test.ts @@ -298,6 +298,109 @@ describe("original write receipt client", () => { }); }); +describe("a session step after a lost write reply", () => { + const runId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const gen = "g1"; + const key = "slack:C1:1.0:review"; + const record: StepRecord = { + step: 1, + seq: 1, + turnIndex: 1, + inFlight: [], + inboxConsumedSeq: 0, + remainingMs: 60_000, + turn: 1, + iteration: 0, + }; + it.each(["recorded", "missing", "partial", "refused", "unavailable", "changed", "read-error"] as const)( + "uses only exact %s receipt evidence before the step, without resending session rows", + async (proof) => { + const paths: string[] = []; + let sent: { requestId: string; digest: string } | undefined; + const ledger = new WorkerRunLedger({ + baseUrl: "https://memory.test", + token: "fixture-token", + storeKey: "runs:default", + fetch: async (input, init) => { + const path = new URL(String(input)).pathname; + const payload = String(init?.body); + const body = JSON.parse(payload) as Record; + paths.push(path); + if (path === "/runs/session/write") { + const requestId = new Headers(init?.headers).get("x-switchboard-storage-request-id"); + if (!requestId) throw new Error("The original request ID is missing"); + sent = { requestId, digest: (await storeRequestWitness(path, payload, body, requestId)).digest }; + throw new TypeError("The committed write reply was lost"); + } + if (path === "/runs/write-receipt") { + const selector = body.selector as Record; + expect(selector).toEqual({ runId, gen, operation: "/runs/session/write", ...sent }); + if (proof === "read-error") throw new TypeError("The receipt read failed"); + if (proof === "unavailable") return Response.json({ kind: "unavailable" }); + return Response.json({ + kind: "source", + resource: { + type: "run", + id: runId, + userId: "access:alice", + channelId: "mcp:fixture", + channelVisibility: "machine", + }, + threadKey: "mcp:fixture:receipt", + receipt: + proof === "missing" + ? { state: "unknown", reason: "missing" } + : { + state: "recorded", + record: { + ...selector, + ...(proof === "changed" ? { digest: "c".repeat(64) } : {}), + version: 1, + effectDigest: "b".repeat(64), + result: proof === "refused" ? { ok: false, reason: "fenced" } : { ok: true, bytes: 100 }, + components: { + stateWritten: false, + eventsWritten: 0, + contextPinningApplied: false, + rowsWritten: proof === "partial" ? 0 : 1, + attachmentsWritten: 0, + sourcesWritten: false, + sourceOwnerWritten: false, + sourceTainted: false, + bytePolicyApplied: true, + }, + }, + }, + }); + } + if (path === "/runs/step") return Response.json({ ok: true }); + throw new Error(`Unexpected route: ${path}`); + }, + }); + const write = ledger.step( + runId, + gen, + record, + [{ idx: 0, message: { role: "assistant", content: [{ type: "text", text: "answer" }] } }], + key, + ); + if (proof === "recorded") expect(await write).toEqual({ ok: true }); + else await expect(write).rejects.toMatchObject({ name: "UncertainStoreError" }); + if (proof === "read-error") { + expect(paths[0]).toBe("/runs/session/write"); + expect(paths.filter((path) => path === "/runs/session/write")).toHaveLength(1); + expect(paths).toContain("/runs/write-receipt"); + expect(paths).not.toContain("/runs/step"); + } else + expect(paths).toEqual( + proof === "recorded" + ? ["/runs/session/write", "/runs/write-receipt", "/runs/step"] + : ["/runs/session/write", "/runs/write-receipt"], + ); + }, + ); +}); + describe("bounded original evidence client", () => { const id = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee"; const reply = () => ({ diff --git a/src/core/runLedgerWorker.ts b/src/core/runLedgerWorker.ts index a0b2417b1..e614e14fd 100644 --- a/src/core/runLedgerWorker.ts +++ b/src/core/runLedgerWorker.ts @@ -1024,11 +1024,45 @@ export class WorkerRunLedger implements RunLedger { if (!fence.ok) return fence; if (readiness.data.transcriptCodec !== 1) throw new RequiredTranscriptError("Memory codec unavailable"); } - const send = async (batch: TranscriptRow[], blobs: TranscriptAttachment[]) => { + const send = async (batch: TranscriptRow[], blobs: TranscriptAttachment[]): Promise => { const payload = { ...sendingBody, rows: batch, attachments: blobs }; if (new TextEncoder().encode(JSON.stringify(payload)).byteLength > TRANSCRIPT_REQUEST_BYTES) throw new RequiredTranscriptError("write exceeds existing request budget"); - const result = await this.post(path, payload); + let result: Awaited>; + try { + result = await this.post(path, payload); + } catch (err) { + // A lost reply can follow a committed batch. Read that exact request; + // no missing or partial proof permits sending the rows again. + if ( + path !== "/runs/session/write" || + !(err instanceof UncertainStoreError) || + err.request.operation !== path || + !err.request.requestId + ) + throw err; + let proof: WriteReceiptRead; + try { + proof = await this.readWriteReceipt({ + runId, + gen, + operation: path, + requestId: err.request.requestId, + digest: err.request.digest, + }); + } catch { + throw err; + } + if ( + proof.kind !== "source" || + proof.receipt.state !== "recorded" || + proof.receipt.record.result.ok !== true || + proof.receipt.record.components.rowsWritten !== batch.length || + proof.receipt.record.components.attachmentsWritten !== blobs.length + ) + throw err; + return { ok: true }; + } if (internal && result.data.ok === true && result.data.transcriptCodec !== 1) throw this.invalidResult( new UncertainStoreError("run ledger: transcript codec acknowledgement missing", result.request),