From 7835687a83bc9fb68f460d40cf49d0472b84456e Mon Sep 17 00:00:00 2001 From: drewstone Date: Wed, 16 Sep 2026 13:31:31 -0700 Subject: [PATCH] fix(completion): score current artifact and proposal revisions Reduce emission-ordered produced events by exact output identity so overwritten content and superseded approvals cannot remain eligible for completion. Preserve the raw event history at the caller and document the observation boundary. --- docs/produced-state.md | 13 +++++ src/produced-state-revisions.test.ts | 83 ++++++++++++++++++++++++++++ src/produced-state.ts | 30 +++++----- 3 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 docs/produced-state.md create mode 100644 src/produced-state-revisions.test.ts diff --git a/docs/produced-state.md b/docs/produced-state.md new file mode 100644 index 000000000..e144524c4 --- /dev/null +++ b/docs/produced-state.md @@ -0,0 +1,13 @@ +# Produced state and artifact revisions + +`extractProducedState(events)` reduces one emission-ordered event stream to the latest observed artifact and proposal state for completion checking. Keep the original stream for history. + +Artifacts use the exact `name`, then `uri`, then `artifactId` as their identity. A later event for that identity replaces its content and type. Distinct output paths remain distinct even when an emitter reuses an artifact ID. No path normalization, rename, or deletion is inferred. + +Proposals use `proposalId`. A later rejected or pending observation must not leave an older approval eligible for scoring. Missing content on the latest observation is unknown/empty evidence; it is not filled from an older revision. + +This behavior correction prevents an overwritten deliverable from satisfying completion through stale text. Consumers that need every historical version should retain the input stream instead of reading the reduced arrays. First-seen identity order is preserved for deterministic output. + +The helper does not verify a provider transaction. Tool-call names are invocations, not successful execution receipts. An agent-written file claiming payment or delivery is not independently observed merchant or fulfillment evidence. Validate those effects through the owning integration and use the existing completion and trace-verification APIs for the required checks. + +Input ordering is the caller's responsibility. Do not combine unordered collectors and claim the result is a current snapshot. Unobserved external changes and deletions cannot be reconstructed from this event subset. diff --git a/src/produced-state-revisions.test.ts b/src/produced-state-revisions.test.ts new file mode 100644 index 000000000..03a90af9a --- /dev/null +++ b/src/produced-state-revisions.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest' +import { extractProducedState, type RuntimeEventLike } from './produced-state' + +const artifact = (name: string, content?: string): RuntimeEventLike => ({ + type: 'artifact', artifactId: `vault:${name}`, name, mimeType: 'text/markdown', content, +}) + +describe('produced state represents the latest observed revision', () => { + it('replaces an obsolete deliverable instead of letting its old content satisfy the task', () => { + const state = extractProducedState([ + artifact('campaigns/offer.md', 'Unsupported discount: 50%'), + artifact('campaigns/offer.md', 'Approved offer: no discount'), + ]) + expect(state.artifacts).toEqual([{ kind: 'text', path: 'campaigns/offer.md', content: 'Approved offer: no discount' }]) + }) + + it('keeps only the newest observation when different artifact ids name the same output path', () => { + const state = extractProducedState([ + { type: 'artifact', artifactId: 'write-1', name: 'result.json', mimeType: 'text/plain', content: 'old' }, + { type: 'artifact', artifactId: 'write-2', name: 'result.json', mimeType: 'application/json', content: '{"paid":false}' }, + ]) + expect(state.artifacts).toEqual([{ kind: 'json', path: 'result.json', content: '{"paid":false}' }]) + }) + + it.each([undefined, ''])('does not backfill missing final content from a stale version (%s)', content => { + const state = extractProducedState([artifact('proof.md', 'Previously complete'), artifact('proof.md', content)]) + expect(state.artifacts).toEqual([{ kind: 'text', path: 'proof.md', content: '' }]) + }) + + it('retains distinct outputs and their first-seen ordering', () => { + const state = extractProducedState([ + artifact('a.md', 'a1'), artifact('b.md', 'b1'), artifact('a.md', 'a2'), + ]) + expect(state.artifacts.map(item => [item.path, item.content])).toEqual([['a.md', 'a2'], ['b.md', 'b1']]) + }) + + it('does not guess that different paths with the same id are aliases', () => { + const state = extractProducedState([ + { type: 'artifact', artifactId: 'reused-id', name: 'a.md', content: 'a' }, + { type: 'artifact', artifactId: 'reused-id', name: 'b.md', content: 'b' }, + ]) + expect(state.artifacts.map(item => item.path)).toEqual(['a.md', 'b.md']) + }) + + it('uses the same latest-observation rule for URI and id-only artifacts', () => { + const state = extractProducedState([ + { type: 'artifact', artifactId: 'first', uri: 'vault://proof', content: 'old uri' }, + { type: 'artifact', artifactId: 'second', uri: 'vault://proof', content: 'new uri' }, + { type: 'artifact', artifactId: 'id-only', content: 'old id' }, + { type: 'artifact', artifactId: 'id-only', content: 'new id' }, + ]) + expect(state.artifacts.map(item => [item.path, item.content])).toEqual([ + ['vault://proof', 'new uri'], ['id-only', 'new id'], + ]) + }) + + it('does not retain an old approval after a rejection of that proposal', () => { + const state = extractProducedState([ + { type: 'proposal_created', proposalId: 'p', title: 'Offer', status: 'approved', content: 'old' }, + { type: 'proposal_created', proposalId: 'p', title: 'Corrected offer', status: 'rejected', content: 'new' }, + ]) + expect(state.proposals).toEqual([{ id: 'p', title: 'Corrected offer', status: 'rejected', content: 'new' }]) + }) + + it('does not resurrect a stale body or approval when the latest proposal omits them', () => { + const state = extractProducedState([ + { type: 'proposal_created', proposalId: 'p', title: 'Offer', status: 'approved', content: 'old' }, + { type: 'proposal_created', proposalId: 'p', title: 'Revised offer' }, + ]) + expect(state.proposals).toEqual([{ id: 'p', title: 'Revised offer', status: 'pending' }]) + }) + + it('does not mutate the supplied stream or merge tool result text into artifacts', () => { + const events = Object.freeze([ + Object.freeze(artifact('proof.md', 'retained')), + Object.freeze({ type: 'tool_result', toolName: 'search', result: 'paid delivered activated' }), + Object.freeze({ type: 'text_delta', text: 'a claim is not a receipt' }), + ]) + expect(extractProducedState(events)).toEqual({ + artifacts: [{ kind: 'text', path: 'proof.md', content: 'retained' }], proposals: [], toolCalls: [], + }) + }) +}) diff --git a/src/produced-state.ts b/src/produced-state.ts index c8422a829..6ab248247 100644 --- a/src/produced-state.ts +++ b/src/produced-state.ts @@ -69,18 +69,21 @@ function artifactKind(mimeType: string | undefined): string { } /** - * Normalize a run's runtime event stream into `ProducedState`. + * Normalize an emission-ordered stream into its latest observed produced state. + * Artifacts are keyed by their exact output path (name, then URI, then id); + * proposals by id. Later observations replace earlier ones, including missing + * content or a rejected status. An obsolete version cannot prove completion. + * Distinct paths remain distinct; no path normalization or aliasing is inferred. + * Results retain first-seen identity order. Tool names describe invocation, + * not success, and are deduplicated in first-seen order. * - * Pure and total — unrecognized event types are skipped. `toolCalls` is - * deduplicated by name in first-seen order (completion cares about a tool's - * presence, not its call count). An artifact with neither a name nor a uri - * still yields an entry keyed by its `artifactId` so it is never silently - * dropped; an artifact with no `content` yields empty content, which the - * completion oracle's structural check then rejects on its own. + * An artifact without observed content yields empty content, which the + * completion oracle rejects. This projection does not verify external effects, + * recover dropped events, or establish freshness beyond the supplied stream. */ export function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState { - const artifacts: Artifact[] = [] - const proposals: ProducedProposal[] = [] + const artifacts = new Map() + const proposals = new Map() const toolCalls: string[] = [] const seenTools = new Set() @@ -93,14 +96,15 @@ export function extractProducedState(events: readonly RuntimeEventLike[]): Produ } } else if (ev.type === 'artifact') { const a = ev as ArtifactEventLike - artifacts.push({ + const path = a.name ?? a.uri ?? a.artifactId + artifacts.set(path, { kind: artifactKind(a.mimeType), - path: a.name ?? a.uri ?? a.artifactId, + path, content: a.content ?? '', }) } else if (ev.type === 'proposal_created') { const p = ev as ProposalEventLike - proposals.push({ + proposals.set(p.proposalId, { id: p.proposalId, title: p.title, status: p.status ?? 'pending', @@ -109,5 +113,5 @@ export function extractProducedState(events: readonly RuntimeEventLike[]): Produ } } - return { artifacts, proposals, toolCalls } + return { artifacts: [...artifacts.values()], proposals: [...proposals.values()], toolCalls } }