From 759a2af8552edbb659647d53dee406385373946f Mon Sep 17 00:00:00 2001 From: Benjamin Borbe Date: Sun, 15 Nov 2026 10:53:40 +0100 Subject: [PATCH 1/2] feat(delivery): carry per-run token counts onto the task file Add the four token counts ClaudeResult already parses (input, output, cache-read, cache-creation) to AgentResultInfo and Result, populate them in claude/agent-step.go, copy them in agent_runner.go, and publish them as the metrics_*_tokens frontmatter keys plus a per-run metrics_runs history list (the flat keys are last-write-wins). A run with no usage summary publishes no token key, matching the existing turn-count absence rule. --- CHANGELOG.md | 4 ++ agent_runner.go | 18 ++++---- agent_status.go | 15 +++++++ agent_step.go | 18 ++++++++ claude/agent-step.go | 61 ++++++++++++++++++++------- delivery/content-generator.go | 6 ++- delivery/result-deliverer.go | 69 +++++++++++++++++++++++++++++++ delivery/result-deliverer_test.go | 67 ++++++++++++++++++++++++++++++ 8 files changed, 234 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3b734d1..e0cb0c5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Please choose versions by [Semantic Versioning](http://semver.org/). * MINOR version when you add functionality in a backwards-compatible manner, and * PATCH version when you make backwards-compatible bug fixes. +## Unreleased + +- feat(delivery): **carry the run's four token counts onto the task file.** `ClaudeResult` already parses `input_tokens`, `output_tokens`, `cache_read_input_tokens` and `cache_creation_tokens` and meters them to Prometheus, but `AgentResultInfo` and the deliverer carried only turns and interaction count, so the counts never reached the vault. They now flow `ClaudeResult` → `Result` → `AgentResultInfo` → the published frontmatter as `metrics_input_tokens`, `metrics_output_tokens`, `metrics_cache_read_tokens` and `metrics_cache_creation_tokens`, and each run is also appended to a `metrics_runs` history list — the flat keys are last-write-wins, so a redriven task would otherwise lose earlier runs and a per-window budget would undercount. A run with no usage summary publishes no token key rather than a false zero, matching the existing turn-count absence rule. Change set: `agent_status.go`, `agent_step.go`, `agent_runner.go`, `claude/agent-step.go`, `delivery/result-deliverer.go`, `delivery/result-deliverer_test.go`, `CHANGELOG.md`. + ## v0.145.0 - fix(delivery): **stamp `last_result_status` on the Kafka delivery path too.** `applyStatusFrontmatter` (the content-generator path) has stamped it since v0.141.1, and that release's entry claimed "every delivered result" — but `applyResultFrontmatter` (`result-deliverer.go`), which is the path a Go agent publishing over Kafka actually takes, merged only status, assignee and phase. So the key kept whatever the spawn notification wrote (empty) on the majority path, and `agent-task-executor`'s succeeded-Job park could not tell a deliberate park from a run that reported `failed` and exited 0; its `recentlySucceeded` bypass, which keys on this field, never fired. Measured 2026-11-14 on `bborbe/trading`'s strategy-prototype agent after it was bumped to `bborbe/agent v0.141.1`: the run carried `build commit=a49ee51` and still left `last_result_status: ""` while publishing `{"Status":"failed"}`. The value mapping is the shared `lastResultStatusValue` helper, not a second implementation, so the two paths cannot drift into different notions of the same field. Covered by `stamps last_result_status from the result status` in `delivery/result-deliverer_test.go` — the test the missing stamp would have failed, and the assertion whose absence let the claim and the code disagree for a release. Change set: `delivery/result-deliverer.go`, `delivery/result-deliverer_test.go`, `CHANGELOG.md`. diff --git a/agent_runner.go b/agent_runner.go index 7f1d0e79..e0f2481c 100644 --- a/agent_runner.go +++ b/agent_runner.go @@ -68,13 +68,17 @@ func (r *StepRunner) Run(ctx context.Context, md *Markdown) (*Result, error) { } if err := r.deliverer.DeliverResult(ctx, AgentResultInfo{ - Status: result.Status, - Output: newContent, - Message: result.Message, - NextPhase: result.NextPhase, - ContinueToNext: result.ContinueToNext, - AgentTurns: result.AgentTurns, - InteractionCount: result.InteractionCount, + Status: result.Status, + Output: newContent, + Message: result.Message, + NextPhase: result.NextPhase, + ContinueToNext: result.ContinueToNext, + AgentTurns: result.AgentTurns, + InteractionCount: result.InteractionCount, + InputTokens: result.InputTokens, + OutputTokens: result.OutputTokens, + CacheReadTokens: result.CacheReadTokens, + CacheCreationTokens: result.CacheCreationTokens, }); err != nil { return result, errors.Wrapf(ctx, err, "step %q deliver", s.Name()) } diff --git a/agent_status.go b/agent_status.go index 5ee7a4e4..2fdbd47e 100644 --- a/agent_status.go +++ b/agent_status.go @@ -59,4 +59,19 @@ type AgentResultInfo struct { // transcript was read and contained no human-authored entry. Absence is never // substituted with a zero. InteractionCount *int64 + // InputTokens is the count of fresh (non-cached) input tokens the run consumed, + // from the CLI's own end-of-run usage summary. Nil means no usage summary was + // reported and the entry is omitted; a non-nil value (including a zero on a run + // that reported a positive turn count) is an observation. + InputTokens *int64 + // OutputTokens is the count of output tokens the run produced. Nil means no usage + // summary was reported. + OutputTokens *int64 + // CacheReadTokens is the count of input tokens served from the prompt cache. Nil + // means no usage summary was reported; a non-nil zero is a real observation (the + // run had no cache hits). + CacheReadTokens *int64 + // CacheCreationTokens is the count of input tokens written into the prompt cache. + // Nil means no usage summary was reported. + CacheCreationTokens *int64 } diff --git a/agent_step.go b/agent_step.go index 555edba8..0d7f0cc3 100644 --- a/agent_step.go +++ b/agent_step.go @@ -74,4 +74,22 @@ type Result struct { // transcript was read and contained no human-authored entry. Absence is never // substituted with a zero. InteractionCount *int64 + + // InputTokens is the count of fresh (non-cached) input tokens the run consumed, + // from the CLI's own end-of-run usage summary. Nil means no usage summary was + // reported. + InputTokens *int64 + + // OutputTokens is the count of output tokens the run produced. Nil means no usage + // summary was reported. + OutputTokens *int64 + + // CacheReadTokens is the count of input tokens served from the prompt cache. Nil + // means no usage summary was reported; a non-nil zero is a real observation (the + // run had no cache hits). + CacheReadTokens *int64 + + // CacheCreationTokens is the count of input tokens written into the prompt cache. + // Nil means no usage summary was reported. + CacheCreationTokens *int64 } diff --git a/claude/agent-step.go b/claude/agent-step.go index ad0ba08e..af0110ca 100644 --- a/claude/agent-step.go +++ b/claude/agent-step.go @@ -174,18 +174,39 @@ func stripLeadingSectionHeading(payload string) string { return strings.TrimPrefix(rest, "\n") } +// runMetrics is the set of per-run observations carried from the CLI result onto a +// Result, with the absence rules applied: a value the CLI did not report stays nil +// rather than being turned into a zero (spec 053). +type runMetrics struct { + agentTurns *int64 + interactionCount *int64 + inputTokens *int64 + outputTokens *int64 + cacheReadTokens *int64 + cacheCreationTokens *int64 +} + // runCounts extracts the run's observed counts from the CLI result with the absence -// rules: a turn total that is not positive is no measurement (nil), and a nil -// InteractionCount (evidence unavailable) stays nil — absence is never turned into a -// zero (spec 053). -func runCounts(result *ClaudeResult) (agentTurns *int64, interactionCount *int64) { +// rules: a turn total that is not positive is no measurement (nil), a nil +// InteractionCount (evidence unavailable) stays nil, and the four token counts are +// populated only when the CLI reported a usage summary (a positive turn total) — so a +// run with no summary publishes no token entry rather than a false zero. Absence is +// never turned into a zero (spec 053). +func runCounts(result *ClaudeResult) runMetrics { if result == nil { - return nil, nil + return runMetrics{} } + m := runMetrics{interactionCount: result.InteractionCount} if result.NumTurns > 0 { - agentTurns = collection.Ptr(result.NumTurns) + m.agentTurns = collection.Ptr(result.NumTurns) + // A usage summary was reported, so the four token counts are observations — + // including a zero (e.g. a run with no cache hits). + m.inputTokens = collection.Ptr(result.InputTokens) + m.outputTokens = collection.Ptr(result.OutputTokens) + m.cacheReadTokens = collection.Ptr(result.CacheReadTokens) + m.cacheCreationTokens = collection.Ptr(result.CacheCreationTokens) } - return agentTurns, result.InteractionCount + return m } // Run marshals the task, calls Claude with the step's prompt + tools, and @@ -225,7 +246,7 @@ func (s *agentStep) Run(ctx context.Context, md *agentlib.Markdown) (*agentlib.R time.Since(runStart), ) - agentTurns, interactionCount := runCounts(result) + m := runCounts(result) // Parse the runner's result once: the same envelope decides the // needs_input/failed path below and carries the payload written on the @@ -244,10 +265,14 @@ func (s *agentStep) Run(ctx context.Context, md *agentlib.Markdown) (*agentlib.R msg = fmt.Sprintf("%s claude run returned status %s", s.cfg.Name, parsed.Status) } return &agentlib.Result{ - Status: parsed.Status, - Message: msg, - AgentTurns: agentTurns, - InteractionCount: interactionCount, + Status: parsed.Status, + Message: msg, + AgentTurns: m.agentTurns, + InteractionCount: m.interactionCount, + InputTokens: m.inputTokens, + OutputTokens: m.outputTokens, + CacheReadTokens: m.cacheReadTokens, + CacheCreationTokens: m.cacheCreationTokens, }, nil } @@ -284,9 +309,13 @@ func (s *agentStep) Run(ctx context.Context, md *agentlib.Markdown) (*agentlib.R }) return &agentlib.Result{ - Status: agentlib.AgentStatusDone, - NextPhase: s.cfg.NextPhase, - AgentTurns: agentTurns, - InteractionCount: interactionCount, + Status: agentlib.AgentStatusDone, + NextPhase: s.cfg.NextPhase, + AgentTurns: m.agentTurns, + InteractionCount: m.interactionCount, + InputTokens: m.inputTokens, + OutputTokens: m.outputTokens, + CacheReadTokens: m.cacheReadTokens, + CacheCreationTokens: m.cacheCreationTokens, }, nil } diff --git a/delivery/content-generator.go b/delivery/content-generator.go index 68682b52..13a24776 100644 --- a/delivery/content-generator.go +++ b/delivery/content-generator.go @@ -80,7 +80,11 @@ func lastResultStatusValue(status agentlib.AgentStatus) string { // the wrong one. The key is written on every result, including a re-delivered // one, so it always describes the latest run rather than the first. func applyStatusFrontmatter(content string, result agentlib.AgentResultInfo) string { - content = SetFrontmatterField(content, "last_result_status", lastResultStatusValue(result.Status)) + content = SetFrontmatterField( + content, + "last_result_status", + lastResultStatusValue(result.Status), + ) switch result.Status { case agentlib.AgentStatusDone: if result.NextPhase == "" { diff --git a/delivery/result-deliverer.go b/delivery/result-deliverer.go index adb24fb9..5d9155aa 100644 --- a/delivery/result-deliverer.go +++ b/delivery/result-deliverer.go @@ -8,6 +8,7 @@ import ( "context" "os" "sync" + "time" "github.com/bborbe/cqrs/base" cdb "github.com/bborbe/cqrs/cdb" @@ -29,6 +30,18 @@ import ( const ( metricsAgentTurnsKey = "metrics_agent_turns" metricsInteractionCountKey = "metrics_interaction_count" + // The four token counts the CLI's end-of-run usage summary reports. Same frozen + // rules as the two keys above: lowercase, snake_case, no nesting, no version + // suffix, no alias. + metricsInputTokensKey = "metrics_input_tokens" // #nosec G101 -- a frontmatter key name, not a credential + metricsOutputTokensKey = "metrics_output_tokens" // #nosec G101 -- a frontmatter key name, not a credential + metricsCacheReadTokensKey = "metrics_cache_read_tokens" // #nosec G101 -- a frontmatter key name, not a credential + metricsCacheCreationTokensKey = "metrics_cache_creation_tokens" // #nosec G101 -- a frontmatter key name, not a credential + // metricsRunsKey is the per-run history list. The four flat keys above are + // last-write-wins — a redriven task keeps only its latest run's counts — so a + // window budget needs every run recorded. Each entry carries completed_at plus the + // four counts, and vault-cli does not know this key, so it preserves it verbatim. + metricsRunsKey = "metrics_runs" ) // NewNoopResultDeliverer creates a agentlib.ResultDeliverer that does nothing. @@ -394,6 +407,7 @@ func (d *kafkaResultDeliverer) applyResultMetrics( if result.AgentTurns != nil { frontmatter[metricsAgentTurnsKey] = *result.AgentTurns } + d.applyTokenMetrics(frontmatter, result) if result.InteractionCount == nil { return } @@ -411,6 +425,61 @@ func (d *kafkaResultDeliverer) applyResultMetrics( frontmatter[metricsInteractionCountKey] = *result.InteractionCount } +// applyTokenMetrics writes the run's four token counts, each only when the run +// observed it: a nil field means the CLI reported no usage summary, and its key is +// omitted rather than written as a zero. Values are plain integers. Unlike the +// interaction count, a token count is not guarded against lowering — it describes +// this run's spend, not a high-water mark. +func (d *kafkaResultDeliverer) applyTokenMetrics( + frontmatter agentlib.TaskFrontmatter, + result agentlib.AgentResultInfo, +) { + if result.InputTokens != nil { + frontmatter[metricsInputTokensKey] = *result.InputTokens + } + if result.OutputTokens != nil { + frontmatter[metricsOutputTokensKey] = *result.OutputTokens + } + if result.CacheReadTokens != nil { + frontmatter[metricsCacheReadTokensKey] = *result.CacheReadTokens + } + if result.CacheCreationTokens != nil { + frontmatter[metricsCacheCreationTokensKey] = *result.CacheCreationTokens + } + d.appendRunHistory(frontmatter, result) +} + +// appendRunHistory records this run as one entry in metrics_runs, so the flat keys' +// last-write-wins limitation does not erase earlier runs from a window sum. The entry +// is written only when the run observed at least one token count; completed_at is the +// deliver time, which is when the spend landed. +func (d *kafkaResultDeliverer) appendRunHistory( + frontmatter agentlib.TaskFrontmatter, + result agentlib.AgentResultInfo, +) { + if result.InputTokens == nil && result.OutputTokens == nil && + result.CacheReadTokens == nil && result.CacheCreationTokens == nil { + return + } + entry := map[string]any{ + "completed_at": d.currentDateTime.Now().Time().Format(time.RFC3339), + } + if result.InputTokens != nil { + entry["input_tokens"] = *result.InputTokens + } + if result.OutputTokens != nil { + entry["output_tokens"] = *result.OutputTokens + } + if result.CacheReadTokens != nil { + entry["cache_read_tokens"] = *result.CacheReadTokens + } + if result.CacheCreationTokens != nil { + entry["cache_creation_tokens"] = *result.CacheCreationTokens + } + history, _ := frontmatter[metricsRunsKey].([]any) + frontmatter[metricsRunsKey] = append(history, entry) +} + // recordedInteractionCount reads the interaction count already recorded on the task // content the run received. ok is false when the key is absent, unparseable, or does // not hold a non-negative integer. diff --git a/delivery/result-deliverer_test.go b/delivery/result-deliverer_test.go index 7d053f96..fe72bf04 100644 --- a/delivery/result-deliverer_test.go +++ b/delivery/result-deliverer_test.go @@ -1098,6 +1098,73 @@ var _ = Describe("KafkaResultDeliverer", func() { }) }) + Context("token counts (SC1/SC2)", func() { + publishedFrontmatter := func() map[string]interface{} { + _, cmdObj := sender.SendCommandObjectArgsForCall(0) + fm, ok := cmdObj.Command.Data["frontmatter"].(map[string]interface{}) + Expect(ok).To(BeTrue()) + return fm + } + + fullResultContent := "---\nstatus: completed\nphase: done\n---\n\nBody.\n\n## Result\n\nok\n" + + It("publishes the four token keys beside the turns key", func() { + generator.GenerateReturns(fullResultContent, nil) + err := deliverer.DeliverResult(ctx, agentlib.AgentResultInfo{ + Status: agentlib.AgentStatusDone, + NextPhase: "done", + AgentTurns: collection.Ptr(int64(3)), + InputTokens: collection.Ptr(int64(1200)), + OutputTokens: collection.Ptr(int64(340)), + CacheReadTokens: collection.Ptr(int64(9000)), + CacheCreationTokens: collection.Ptr(int64(410)), + }) + Expect(err).NotTo(HaveOccurred()) + fm := publishedFrontmatter() + Expect(fm["metrics_input_tokens"]).To(BeNumerically("==", 1200)) + Expect(fm["metrics_output_tokens"]).To(BeNumerically("==", 340)) + Expect(fm["metrics_cache_read_tokens"]).To(BeNumerically("==", 9000)) + Expect(fm["metrics_cache_creation_tokens"]).To(BeNumerically("==", 410)) + }) + + It("omits every token key and the run history when no usage summary was reported", func() { + generator.GenerateReturns(fullResultContent, nil) + err := deliverer.DeliverResult(ctx, agentlib.AgentResultInfo{ + Status: agentlib.AgentStatusDone, + NextPhase: "done", + }) + Expect(err).NotTo(HaveOccurred()) + fm := publishedFrontmatter() + for _, k := range []string{ + "metrics_input_tokens", + "metrics_output_tokens", + "metrics_cache_read_tokens", + "metrics_cache_creation_tokens", + "metrics_runs", + } { + _, ok := fm[k] + Expect(ok).To(BeFalse(), k) + } + }) + + It("appends one metrics_runs entry per run carrying the counts and completed_at", func() { + generator.GenerateReturns(fullResultContent, nil) + err := deliverer.DeliverResult(ctx, agentlib.AgentResultInfo{ + Status: agentlib.AgentStatusDone, + NextPhase: "done", + InputTokens: collection.Ptr(int64(1200)), + }) + Expect(err).NotTo(HaveOccurred()) + runs, ok := publishedFrontmatter()["metrics_runs"].([]interface{}) + Expect(ok).To(BeTrue()) + Expect(runs).To(HaveLen(1)) + entry, ok := runs[0].(map[string]interface{}) + Expect(ok).To(BeTrue()) + Expect(entry["input_tokens"]).To(BeNumerically("==", 1200)) + Expect(entry).To(HaveKey("completed_at")) + }) + }) + Context("phase advanced in-process within one Job", func() { // A multi-phase Agent walks planning -> execution inside one Job with a // single deliverer. Every save after the first is generated from the From c8f25c719f5bceb37d4bfde9380e9866788f53fc Mon Sep 17 00:00:00 2001 From: Benjamin Borbe Date: Sun, 15 Nov 2026 11:23:33 +0100 Subject: [PATCH 2/2] fix(delivery): accumulate metrics_runs across a Job's deliveries; carry tokens on the failed path Review findings on #196: - metrics_runs was truncated for multi-phase Jobs: each delivery rebuilt the frontmatter from the Job-start content, so a second delivery dropped the first's entry. The list is now accumulated on the deliverer, seeded once from the Job-start content. - Token usage was discarded on the failed-run path and for partial-usage results. runCounts now reads usage before the error check (the runner carries it on the error path) and treats a summary with any token count as present even when no turn count was reported; the failed Result carries the counts. --- claude/agent-step.go | 30 +++++++++++----- delivery/result-deliverer.go | 58 +++++++++++++++++++++++++------ delivery/result-deliverer_test.go | 29 ++++++++++++++++ 3 files changed, 99 insertions(+), 18 deletions(-) diff --git a/claude/agent-step.go b/claude/agent-step.go index af0110ca..ab600d59 100644 --- a/claude/agent-step.go +++ b/claude/agent-step.go @@ -197,10 +197,15 @@ func runCounts(result *ClaudeResult) runMetrics { return runMetrics{} } m := runMetrics{interactionCount: result.InteractionCount} - if result.NumTurns > 0 { - m.agentTurns = collection.Ptr(result.NumTurns) - // A usage summary was reported, so the four token counts are observations — - // including a zero (e.g. a run with no cache hits). + // A usage summary is present when the CLI reported turns OR any token count. + // Gating on turns alone dropped tokens from a partial-usage result — a summary + // that carried tokens but no turn count. + if result.NumTurns > 0 || + result.InputTokens > 0 || result.OutputTokens > 0 || + result.CacheReadTokens > 0 || result.CacheCreationTokens > 0 { + if result.NumTurns > 0 { + m.agentTurns = collection.Ptr(result.NumTurns) + } m.inputTokens = collection.Ptr(result.InputTokens) m.outputTokens = collection.Ptr(result.OutputTokens) m.cacheReadTokens = collection.Ptr(result.CacheReadTokens) @@ -227,6 +232,11 @@ func (s *agentStep) Run(ctx context.Context, md *agentlib.Markdown) (*agentlib.R glog.Infof("%s: invoking claude runner (prompt=%d bytes)", s.cfg.Name, len(prompt)) runStart := time.Now() result, runErr := s.cfg.Runner.Run(ctx, prompt) + // The counts are read before the error check: the runner carries usage on the + // error path too (a rate-limited or failed attempt really consumed tokens), so a + // failed run must publish what it spent — otherwise the window budget would never + // count the runs most likely to burn it. + m := runCounts(result) if runErr != nil { glog.Infof( "%s: claude runner failed after %s: %v", @@ -235,8 +245,14 @@ func (s *agentStep) Run(ctx context.Context, md *agentlib.Markdown) (*agentlib.R runErr, ) return &agentlib.Result{ - Status: agentlib.AgentStatusFailed, - Message: fmt.Sprintf("%s claude run failed: %v", s.cfg.Name, runErr), + Status: agentlib.AgentStatusFailed, + Message: fmt.Sprintf("%s claude run failed: %v", s.cfg.Name, runErr), + AgentTurns: m.agentTurns, + InteractionCount: m.interactionCount, + InputTokens: m.inputTokens, + OutputTokens: m.outputTokens, + CacheReadTokens: m.cacheReadTokens, + CacheCreationTokens: m.cacheCreationTokens, }, nil } glog.Infof( @@ -246,8 +262,6 @@ func (s *agentStep) Run(ctx context.Context, md *agentlib.Markdown) (*agentlib.R time.Since(runStart), ) - m := runCounts(result) - // Parse the runner's result once: the same envelope decides the // needs_input/failed path below and carries the payload written on the // success path further down. diff --git a/delivery/result-deliverer.go b/delivery/result-deliverer.go index 5d9155aa..dd48ce2d 100644 --- a/delivery/result-deliverer.go +++ b/delivery/result-deliverer.go @@ -199,6 +199,14 @@ type kafkaResultDeliverer struct { // from. Empty means "no advance yet"; a failed result then publishes no phase // at all, so the controller's on-disk-first merge keeps the phase on disk. advancedPhase string + + // runHistory accumulates metrics_runs entries across the deliveries of one Job. + // Each DeliverResult rebuilds the frontmatter from the Job-start content, so + // reading the run list back from that frontmatter per delivery would drop every + // earlier delivery's entry from a multi-phase Job. Seeded once from the + // Job-start content so cross-Job history survives, then appended per delivery. + runHistory []any + runHistorySeeded bool } func (d *kafkaResultDeliverer) DeliverResult( @@ -450,20 +458,36 @@ func (d *kafkaResultDeliverer) applyTokenMetrics( } // appendRunHistory records this run as one entry in metrics_runs, so the flat keys' -// last-write-wins limitation does not erase earlier runs from a window sum. The entry -// is written only when the run observed at least one token count; completed_at is the -// deliver time, which is when the spend landed. +// last-write-wins limitation does not erase earlier runs from a window sum. The +// entry is written only when the run observed at least one token count; completed_at +// is the deliver time, which is when the spend landed. +// +// The list is accumulated on the deliverer, not re-read from the generated +// frontmatter: a multi-phase Job rebuilds that frontmatter from the Job-start +// content on every delivery, so a per-delivery read would drop the earlier +// deliveries' entries. It is seeded once from the Job-start content so history from +// prior Jobs survives. func (d *kafkaResultDeliverer) appendRunHistory( frontmatter agentlib.TaskFrontmatter, result agentlib.AgentResultInfo, ) { - if result.InputTokens == nil && result.OutputTokens == nil && - result.CacheReadTokens == nil && result.CacheCreationTokens == nil { - return + if !d.runHistorySeeded { + d.runHistory = runHistoryFromContent(d.originalContent) + d.runHistorySeeded = true } - entry := map[string]any{ - "completed_at": d.currentDateTime.Now().Time().Format(time.RFC3339), + if entry, ok := tokenRunEntry(d.currentDateTime.Now().Time(), result); ok { + d.runHistory = append(d.runHistory, entry) } + if len(d.runHistory) > 0 { + frontmatter[metricsRunsKey] = d.runHistory + } +} + +// tokenRunEntry builds one metrics_runs entry from the run's observed token counts. +// ok is false when the run observed no token count — an entry carrying only +// completed_at would record a run with no spend and mislead the window sum. +func tokenRunEntry(at time.Time, result agentlib.AgentResultInfo) (map[string]any, bool) { + entry := map[string]any{"completed_at": at.Format(time.RFC3339)} if result.InputTokens != nil { entry["input_tokens"] = *result.InputTokens } @@ -476,8 +500,22 @@ func (d *kafkaResultDeliverer) appendRunHistory( if result.CacheCreationTokens != nil { entry["cache_creation_tokens"] = *result.CacheCreationTokens } - history, _ := frontmatter[metricsRunsKey].([]any) - frontmatter[metricsRunsKey] = append(history, entry) + if len(entry) == 1 { + return nil, false + } + return entry, true +} + +// runHistoryFromContent reads the metrics_runs list already on the Job-start task +// content. A missing or malformed list is empty, not an error — history is +// best-effort and a parse failure must not fail the delivery. +func runHistoryFromContent(content string) []any { + fm, _ := ParseMarkdownFrontmatter(content) + if fm == nil { + return nil + } + list, _ := fm[metricsRunsKey].([]any) + return list } // recordedInteractionCount reads the interaction count already recorded on the task diff --git a/delivery/result-deliverer_test.go b/delivery/result-deliverer_test.go index fe72bf04..014e1144 100644 --- a/delivery/result-deliverer_test.go +++ b/delivery/result-deliverer_test.go @@ -1163,6 +1163,35 @@ var _ = Describe("KafkaResultDeliverer", func() { Expect(entry["input_tokens"]).To(BeNumerically("==", 1200)) Expect(entry).To(HaveKey("completed_at")) }) + + It("accumulates metrics_runs across deliveries in one Job (multi-phase)", func() { + generator.GenerateReturns(fullResultContent, nil) + Expect(deliverer.DeliverResult(ctx, agentlib.AgentResultInfo{ + Status: agentlib.AgentStatusInProgress, + InputTokens: collection.Ptr(int64(100)), + })).To(Succeed()) + Expect(deliverer.DeliverResult(ctx, agentlib.AgentResultInfo{ + Status: agentlib.AgentStatusDone, + NextPhase: "done", + InputTokens: collection.Ptr(int64(200)), + })).To(Succeed()) + _, cmdObj := sender.SendCommandObjectArgsForCall(1) + fm, ok := cmdObj.Command.Data["frontmatter"].(map[string]interface{}) + Expect(ok).To(BeTrue()) + runs, ok := fm["metrics_runs"].([]interface{}) + Expect(ok).To(BeTrue()) + Expect(runs).To(HaveLen(2)) + }) + + It("publishes token counts on a failed result", func() { + generator.GenerateReturns(fullResultContent, nil) + err := deliverer.DeliverResult(ctx, agentlib.AgentResultInfo{ + Status: agentlib.AgentStatusFailed, + InputTokens: collection.Ptr(int64(500)), + }) + Expect(err).NotTo(HaveOccurred()) + Expect(publishedFrontmatter()["metrics_input_tokens"]).To(BeNumerically("==", 500)) + }) }) Context("phase advanced in-process within one Job", func() {