From 6a50669a8d6c3329ee3560d67eee6862491cf703 Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Wed, 21 Oct 2026 11:37:55 +0200 Subject: [PATCH 1/3] feat(trace): report the generated line totals of a session Add ai_generated_lines_added and ai_generated_lines_removed to the code changes of the AI coding session evidence. They hold the lines that all the agent edits of the session added and removed, also the lines that never reached a commit (spec issue-3639, R-011, D-011). Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: d13fc871-d294-4c08-b1d4-e7bdcaef6969 --- app/cli/internal/trace/README.md | 2 +- app/cli/internal/trace/attribution/enrich.go | 4 ++ .../internal/trace/attribution/enrich_test.go | 55 +++++++++++++++++++ .../ai-coding-session-0.1.schema.json | 8 +++ .../aicodingsession/aicodingsession.go | 26 +++++---- 5 files changed, 84 insertions(+), 11 deletions(-) diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 034a7f810..ee6a336fc 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -56,7 +56,7 @@ Which lines of a commit the AI wrote. |---------|---------------|-------------|--------|--------------|------------|---|---| | Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | | Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | -| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence does not report them yet (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | +| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence reports the totals of the session in `ai_generated_lines_added` and `ai_generated_lines_removed`. A session with no commit sends no evidence, so it reports no totals (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | | Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | No | | Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the removals of the session's commits, less the lines that a commit of the session wrote. The line counts of each file are the sum of its AI and human lines, and a file that the session created stays created when a later commit of the session changes it (spec issue-3639, issue 3674). | Partial | Partial | Partial | Partial | No | No | | Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | No | diff --git a/app/cli/internal/trace/attribution/enrich.go b/app/cli/internal/trace/attribution/enrich.go index 15f713c18..34dcc5b17 100644 --- a/app/cli/internal/trace/attribution/enrich.go +++ b/app/cli/internal/trace/attribution/enrich.go @@ -150,6 +150,10 @@ func Enrich(sessionID string, ledgers map[string]*state.AILineAttribution, commi changes.HumanLinesAdded += f.HumanLinesAdded changes.HumanLinesRemoved += f.HumanLinesRemoved } + + // All the lines that the agent edits of the session wrote, also the ones + // that never reached a commit (spec issue-3639, R-011, D-011). + changes.AIGeneratedLinesAdded, changes.AIGeneratedLinesRemoved = self.GeneratedAdded, self.GeneratedRemoved } // fileSplit is the AI/human split of the changed lines of one file. diff --git a/app/cli/internal/trace/attribution/enrich_test.go b/app/cli/internal/trace/attribution/enrich_test.go index 7c48dc1c9..58f5d6fad 100644 --- a/app/cli/internal/trace/attribution/enrich_test.go +++ b/app/cli/internal/trace/attribution/enrich_test.go @@ -189,6 +189,61 @@ func TestEnrich_Legacy(t *testing.T) { }) } +func TestEnrich_GeneratedLines(t *testing.T) { + withGenerated := func(attr *state.AILineAttribution, added, removed int) *state.AILineAttribution { + attr.GeneratedAdded, attr.GeneratedRemoved = added, removed + + return attr + } + + testCases := []struct { + name string + ledgers map[string]*state.AILineAttribution + wantAdded, wantRemoved int + }{ + { + // The agent wrote 10 lines and removed 3. Only 2 lines reached + // the commit; the session still reports all of them (spec + // issue-3639, D-011). + name: "reports all the lines of the agent edits, also the ones that no commit holds", + ledgers: map[string]*state.AILineAttribution{ + testSession: withGenerated(ledger(testSession, testFile, []string{"a", "b"}, nil), 10, 3), + }, + wantAdded: 10, + wantRemoved: 3, + }, + { + name: "does not add the lines of another session", + ledgers: map[string]*state.AILineAttribution{ + testSession: withGenerated(ledger(testSession, testFile, []string{"a", "b"}, nil), 2, 0), + otherSession: withGenerated(ledger(otherSession, testFile, []string{"c"}, nil), 7, 1), + }, + wantAdded: 2, + }, + { + name: "a session with no ledger reports no lines", + ledgers: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + changes := &aicodingsession.CodeChanges{ + LinesAdded: 2, + Files: []aicodingsession.FileChange{{Path: testFile, LinesAdded: 2}}, + } + commits := []CommitLines{{SessionIDs: []string{testSession}, Files: []tracegit.FileLines{ + {Path: testFile, Added: []string{"a", "b"}}, + }}} + + Enrich(testSession, tc.ledgers, commits, nil, changes) + + assert.Equal(t, tc.wantAdded, changes.AIGeneratedLinesAdded) + assert.Equal(t, tc.wantRemoved, changes.AIGeneratedLinesRemoved) + }) + } +} + func TestEnrich_Lines(t *testing.T) { type split struct{ aiAdded, aiRemoved, humanAdded, humanRemoved int } diff --git a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json index fbfaf42ad..a7d69b1e4 100644 --- a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json +++ b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json @@ -192,6 +192,14 @@ "type": "integer", "description": "Lines removed attributed to human" }, + "ai_generated_lines_added": { + "type": "integer", + "description": "Lines added by all the agent edits of the session, including lines that never reached a commit" + }, + "ai_generated_lines_removed": { + "type": "integer", + "description": "Lines removed by all the agent edits of the session, including lines that never reached a commit" + }, "files": { "type": "array", "description": "Per-file change details", diff --git a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go index fbe4a7ae7..258860e0e 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go +++ b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go @@ -252,16 +252,22 @@ type FileChange struct { // CodeChanges summarizes code modifications made during the session. type CodeChanges struct { - FilesModified int `json:"files_modified,omitempty"` - FilesCreated int `json:"files_created,omitempty"` - FilesDeleted int `json:"files_deleted,omitempty"` - LinesAdded int `json:"lines_added,omitempty"` - LinesRemoved int `json:"lines_removed,omitempty"` - AILinesAdded int `json:"ai_lines_added,omitempty"` - AILinesRemoved int `json:"ai_lines_removed,omitempty"` - HumanLinesAdded int `json:"human_lines_added,omitempty"` - HumanLinesRemoved int `json:"human_lines_removed,omitempty"` - Files []FileChange `json:"files,omitempty"` + FilesModified int `json:"files_modified,omitempty"` + FilesCreated int `json:"files_created,omitempty"` + FilesDeleted int `json:"files_deleted,omitempty"` + LinesAdded int `json:"lines_added,omitempty"` + LinesRemoved int `json:"lines_removed,omitempty"` + AILinesAdded int `json:"ai_lines_added,omitempty"` + AILinesRemoved int `json:"ai_lines_removed,omitempty"` + HumanLinesAdded int `json:"human_lines_added,omitempty"` + HumanLinesRemoved int `json:"human_lines_removed,omitempty"` + // AIGeneratedLinesAdded and AIGeneratedLinesRemoved are the lines that all + // the agent edits of the session added and removed, also the ones that + // never reached a commit. Compare them with AILinesAdded and + // AILinesRemoved to see how much of the generated code the session kept. + AIGeneratedLinesAdded int `json:"ai_generated_lines_added,omitempty"` + AIGeneratedLinesRemoved int `json:"ai_generated_lines_removed,omitempty"` + Files []FileChange `json:"files,omitempty"` } // Model holds information about the AI models used in the session. From 35749ca262f3d6d9e5c3499632ef7f7572be2b82 Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Wed, 21 Oct 2026 11:44:56 +0200 Subject: [PATCH 2/3] docs(trace): describe net and generated line counts of a session Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: d13fc871-d294-4c08-b1d4-e7bdcaef6969 --- .../ai-coding-session-0.1.schema.json | 16 +++++++------- .../aicodingsession/aicodingsession.go | 22 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json index a7d69b1e4..7489ee8d8 100644 --- a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json +++ b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json @@ -170,35 +170,35 @@ }, "lines_added": { "type": "integer", - "description": "Total lines added" + "description": "Net lines added by the session's commits: AI plus human lines" }, "lines_removed": { "type": "integer", - "description": "Total lines removed" + "description": "Net lines removed by the session's commits: AI plus human lines" }, "ai_lines_added": { "type": "integer", - "description": "Lines added attributed to AI" + "description": "Net committed lines added, attributed to AI" }, "ai_lines_removed": { "type": "integer", - "description": "Lines removed attributed to AI" + "description": "Net committed lines removed, attributed to AI" }, "human_lines_added": { "type": "integer", - "description": "Lines added attributed to human" + "description": "Net committed lines added, attributed to human" }, "human_lines_removed": { "type": "integer", - "description": "Lines removed attributed to human" + "description": "Net committed lines removed, attributed to human" }, "ai_generated_lines_added": { "type": "integer", - "description": "Lines added by all the agent edits of the session, including lines that never reached a commit" + "description": "All lines added by the agent edits of the session, committed or not; not net" }, "ai_generated_lines_removed": { "type": "integer", - "description": "Lines removed by all the agent edits of the session, including lines that never reached a commit" + "description": "All lines removed by the agent edits of the session, committed or not; not net" }, "files": { "type": "array", diff --git a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go index 258860e0e..1a427caee 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go +++ b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go @@ -252,19 +252,19 @@ type FileChange struct { // CodeChanges summarizes code modifications made during the session. type CodeChanges struct { - FilesModified int `json:"files_modified,omitempty"` - FilesCreated int `json:"files_created,omitempty"` - FilesDeleted int `json:"files_deleted,omitempty"` - LinesAdded int `json:"lines_added,omitempty"` - LinesRemoved int `json:"lines_removed,omitempty"` - AILinesAdded int `json:"ai_lines_added,omitempty"` - AILinesRemoved int `json:"ai_lines_removed,omitempty"` + FilesModified int `json:"files_modified,omitempty"` + FilesCreated int `json:"files_created,omitempty"` + FilesDeleted int `json:"files_deleted,omitempty"` + // Net lines of the session's commits: the AI lines plus the human lines. + LinesAdded int `json:"lines_added,omitempty"` + LinesRemoved int `json:"lines_removed,omitempty"` + // Net committed lines attributed to the AI. + AILinesAdded int `json:"ai_lines_added,omitempty"` + AILinesRemoved int `json:"ai_lines_removed,omitempty"` + // Net committed lines attributed to a human. HumanLinesAdded int `json:"human_lines_added,omitempty"` HumanLinesRemoved int `json:"human_lines_removed,omitempty"` - // AIGeneratedLinesAdded and AIGeneratedLinesRemoved are the lines that all - // the agent edits of the session added and removed, also the ones that - // never reached a commit. Compare them with AILinesAdded and - // AILinesRemoved to see how much of the generated code the session kept. + // All lines of the session's agent edits, committed or not; not net. AIGeneratedLinesAdded int `json:"ai_generated_lines_added,omitempty"` AIGeneratedLinesRemoved int `json:"ai_generated_lines_removed,omitempty"` Files []FileChange `json:"files,omitempty"` From a635ea89eb094c3b148fbba151c74d85dbbe0ccb Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Wed, 21 Oct 2026 11:49:29 +0200 Subject: [PATCH 3/3] test(trace): check the generated line counts in the e2e suite Also note that the session line totals are raw diff totals when a commit cannot be read. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: d13fc871-d294-4c08-b1d4-e7bdcaef6969 --- app/cli/internal/trace/e2e/checks.go | 21 ++++++++++++++++--- .../ai-coding-session-0.1.schema.json | 4 ++-- .../aicodingsession/aicodingsession.go | 2 +- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go index 726379afb..e6c5470c5 100644 --- a/app/cli/internal/trace/e2e/checks.go +++ b/app/cli/internal/trace/e2e/checks.go @@ -99,9 +99,7 @@ var features = map[string]Feature{ "Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}}, "Partial commits": {Checks: []Check{{scenarioPartial, checkBothCommitsCredited}}}, "Line ranges": {Checks: []Check{{scenarioEditAndCommit, checkLineRanges("greeting.txt", 3)}}}, - "Generated line counts": { - NotCovered: "neither the user nor the evidence can see the generated line counts yet", - }, + "Generated line counts": {Checks: []Check{{scenarioEditAndCommit, checkGeneratedLineCounts}}}, // User experience "Welcome message": {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}}, @@ -196,6 +194,23 @@ func checkFileAttributedToAI(path string) func(s *Session) error { } } +// checkGeneratedLineCounts verifies that the evidence reports the lines of +// all the agent edits. Each net AI line came from an agent edit, so the +// generated lines are at least the net AI lines. +func checkGeneratedLineCounts(s *Session) error { + cc := evidence(s).CodeChanges + if cc == nil { + return errors.New("the evidence has no code changes") + } + if cc.AIGeneratedLinesAdded == 0 { + return errors.New("the evidence has no generated lines added") + } + if cc.AIGeneratedLinesAdded < cc.AILinesAdded { + return fmt.Errorf("%d generated lines added, fewer than the %d net AI lines added", cc.AIGeneratedLinesAdded, cc.AILinesAdded) + } + return nil +} + // hookMessage returns the first hook message of an event, or errNotCovered // when the harness does not report hook messages. func hookMessage(s *Session, event string) (*HookMessage, error) { diff --git a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json index 7489ee8d8..02ad74465 100644 --- a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json +++ b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json @@ -170,11 +170,11 @@ }, "lines_added": { "type": "integer", - "description": "Net lines added by the session's commits: AI plus human lines" + "description": "Net lines added by the session's commits (AI plus human); raw diff totals if a commit is unreadable" }, "lines_removed": { "type": "integer", - "description": "Net lines removed by the session's commits: AI plus human lines" + "description": "Net lines removed by the session's commits (AI plus human); raw diff totals if a commit is unreadable" }, "ai_lines_added": { "type": "integer", diff --git a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go index 1a427caee..60d131822 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go +++ b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go @@ -255,7 +255,7 @@ type CodeChanges struct { FilesModified int `json:"files_modified,omitempty"` FilesCreated int `json:"files_created,omitempty"` FilesDeleted int `json:"files_deleted,omitempty"` - // Net lines of the session's commits: the AI lines plus the human lines. + // Net lines of the session's commits (AI plus human); raw diff totals if a commit is unreadable. LinesAdded int `json:"lines_added,omitempty"` LinesRemoved int `json:"lines_removed,omitempty"` // Net committed lines attributed to the AI.