Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions docs/produced-state.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Produced state and artifact revisions

`extractProducedState(events)` reduces one emission-ordered event stream to the latest observed artifact and proposal state for completion checking. Keep the original stream for history.

Artifacts use the exact `name`, then `uri`, then `artifactId` as their identity. A later event for that identity replaces its content and type. Distinct output paths remain distinct even when an emitter reuses an artifact ID. No path normalization, rename, or deletion is inferred.

Proposals use `proposalId`. A later rejected or pending observation must not leave an older approval eligible for scoring. Missing content on the latest observation is unknown/empty evidence; it is not filled from an older revision.

This behavior correction prevents an overwritten deliverable from satisfying completion through stale text. Consumers that need every historical version should retain the input stream instead of reading the reduced arrays. First-seen identity order is preserved for deterministic output.

The helper does not verify a provider transaction. Tool-call names are invocations, not successful execution receipts. An agent-written file claiming payment or delivery is not independently observed merchant or fulfillment evidence. Validate those effects through the owning integration and use the existing completion and trace-verification APIs for the required checks.

Input ordering is the caller's responsibility. Do not combine unordered collectors and claim the result is a current snapshot. Unobserved external changes and deletions cannot be reconstructed from this event subset.
83 changes: 83 additions & 0 deletions src/produced-state-revisions.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
import { describe, expect, it } from 'vitest'
import { extractProducedState, type RuntimeEventLike } from './produced-state'

const artifact = (name: string, content?: string): RuntimeEventLike => ({
type: 'artifact', artifactId: `vault:${name}`, name, mimeType: 'text/markdown', content,
})

describe('produced state represents the latest observed revision', () => {
it('replaces an obsolete deliverable instead of letting its old content satisfy the task', () => {
const state = extractProducedState([
artifact('campaigns/offer.md', 'Unsupported discount: 50%'),
artifact('campaigns/offer.md', 'Approved offer: no discount'),
])
expect(state.artifacts).toEqual([{ kind: 'text', path: 'campaigns/offer.md', content: 'Approved offer: no discount' }])
})

it('keeps only the newest observation when different artifact ids name the same output path', () => {
const state = extractProducedState([
{ type: 'artifact', artifactId: 'write-1', name: 'result.json', mimeType: 'text/plain', content: 'old' },
{ type: 'artifact', artifactId: 'write-2', name: 'result.json', mimeType: 'application/json', content: '{"paid":false}' },
])
expect(state.artifacts).toEqual([{ kind: 'json', path: 'result.json', content: '{"paid":false}' }])
})

it.each([undefined, ''])('does not backfill missing final content from a stale version (%s)', content => {
const state = extractProducedState([artifact('proof.md', 'Previously complete'), artifact('proof.md', content)])
expect(state.artifacts).toEqual([{ kind: 'text', path: 'proof.md', content: '' }])
})

it('retains distinct outputs and their first-seen ordering', () => {
const state = extractProducedState([
artifact('a.md', 'a1'), artifact('b.md', 'b1'), artifact('a.md', 'a2'),
])
expect(state.artifacts.map(item => [item.path, item.content])).toEqual([['a.md', 'a2'], ['b.md', 'b1']])
})

it('does not guess that different paths with the same id are aliases', () => {
const state = extractProducedState([
{ type: 'artifact', artifactId: 'reused-id', name: 'a.md', content: 'a' },
{ type: 'artifact', artifactId: 'reused-id', name: 'b.md', content: 'b' },
])
expect(state.artifacts.map(item => item.path)).toEqual(['a.md', 'b.md'])
})

it('uses the same latest-observation rule for URI and id-only artifacts', () => {
const state = extractProducedState([
{ type: 'artifact', artifactId: 'first', uri: 'vault://proof', content: 'old uri' },
{ type: 'artifact', artifactId: 'second', uri: 'vault://proof', content: 'new uri' },
{ type: 'artifact', artifactId: 'id-only', content: 'old id' },
{ type: 'artifact', artifactId: 'id-only', content: 'new id' },
])
expect(state.artifacts.map(item => [item.path, item.content])).toEqual([
['vault://proof', 'new uri'], ['id-only', 'new id'],
])
})

it('does not retain an old approval after a rejection of that proposal', () => {
const state = extractProducedState([
{ type: 'proposal_created', proposalId: 'p', title: 'Offer', status: 'approved', content: 'old' },
{ type: 'proposal_created', proposalId: 'p', title: 'Corrected offer', status: 'rejected', content: 'new' },
])
expect(state.proposals).toEqual([{ id: 'p', title: 'Corrected offer', status: 'rejected', content: 'new' }])
})

it('does not resurrect a stale body or approval when the latest proposal omits them', () => {
const state = extractProducedState([
{ type: 'proposal_created', proposalId: 'p', title: 'Offer', status: 'approved', content: 'old' },
{ type: 'proposal_created', proposalId: 'p', title: 'Revised offer' },
])
expect(state.proposals).toEqual([{ id: 'p', title: 'Revised offer', status: 'pending' }])
})

it('does not mutate the supplied stream or merge tool result text into artifacts', () => {
const events = Object.freeze([
Object.freeze(artifact('proof.md', 'retained')),
Object.freeze({ type: 'tool_result', toolName: 'search', result: 'paid delivered activated' }),
Object.freeze({ type: 'text_delta', text: 'a claim is not a receipt' }),
])
expect(extractProducedState(events)).toEqual({
artifacts: [{ kind: 'text', path: 'proof.md', content: 'retained' }], proposals: [], toolCalls: [],
})
})
})
30 changes: 17 additions & 13 deletions src/produced-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,18 +69,21 @@ function artifactKind(mimeType: string | undefined): string {
}

/**
* Normalize a run's runtime event stream into `ProducedState`.
* Normalize an emission-ordered stream into its latest observed produced state.
* Artifacts are keyed by their exact output path (name, then URI, then id);
* proposals by id. Later observations replace earlier ones, including missing
* content or a rejected status. An obsolete version cannot prove completion.
* Distinct paths remain distinct; no path normalization or aliasing is inferred.
* Results retain first-seen identity order. Tool names describe invocation,
* not success, and are deduplicated in first-seen order.
*
* Pure and total — unrecognized event types are skipped. `toolCalls` is
* deduplicated by name in first-seen order (completion cares about a tool's
* presence, not its call count). An artifact with neither a name nor a uri
* still yields an entry keyed by its `artifactId` so it is never silently
* dropped; an artifact with no `content` yields empty content, which the
* completion oracle's structural check then rejects on its own.
* An artifact without observed content yields empty content, which the
* completion oracle rejects. This projection does not verify external effects,
* recover dropped events, or establish freshness beyond the supplied stream.
*/
export function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState {
const artifacts: Artifact[] = []
const proposals: ProducedProposal[] = []
const artifacts = new Map<string, Artifact>()
const proposals = new Map<string, ProducedProposal>()
const toolCalls: string[] = []
const seenTools = new Set<string>()

Expand All @@ -93,14 +96,15 @@ export function extractProducedState(events: readonly RuntimeEventLike[]): Produ
}
} else if (ev.type === 'artifact') {
const a = ev as ArtifactEventLike
artifacts.push({
const path = a.name ?? a.uri ?? a.artifactId
artifacts.set(path, {
kind: artifactKind(a.mimeType),
path: a.name ?? a.uri ?? a.artifactId,
path,
content: a.content ?? '',
})
} else if (ev.type === 'proposal_created') {
const p = ev as ProposalEventLike
proposals.push({
proposals.set(p.proposalId, {
id: p.proposalId,
title: p.title,
status: p.status ?? 'pending',
Expand All @@ -109,5 +113,5 @@ export function extractProducedState(events: readonly RuntimeEventLike[]): Produ
}
}

return { artifacts, proposals, toolCalls }
return { artifacts: [...artifacts.values()], proposals: [...proposals.values()], toolCalls }
}
Loading