diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 034a7f810..ee6a336fc 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -56,7 +56,7 @@ Which lines of a commit the AI wrote. |---------|---------------|-------------|--------|--------------|------------|---|---| | Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | | Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | -| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence does not report them yet (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | +| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence reports the totals of the session in `ai_generated_lines_added` and `ai_generated_lines_removed`. A session with no commit sends no evidence, so it reports no totals (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | | Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | No | | Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the removals of the session's commits, less the lines that a commit of the session wrote. The line counts of each file are the sum of its AI and human lines, and a file that the session created stays created when a later commit of the session changes it (spec issue-3639, issue 3674). | Partial | Partial | Partial | Partial | No | No | | Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | No | diff --git a/app/cli/internal/trace/attribution/enrich.go b/app/cli/internal/trace/attribution/enrich.go index 15f713c18..34dcc5b17 100644 --- a/app/cli/internal/trace/attribution/enrich.go +++ b/app/cli/internal/trace/attribution/enrich.go @@ -150,6 +150,10 @@ func Enrich(sessionID string, ledgers map[string]*state.AILineAttribution, commi changes.HumanLinesAdded += f.HumanLinesAdded changes.HumanLinesRemoved += f.HumanLinesRemoved } + + // All the lines that the agent edits of the session wrote, also the ones + // that never reached a commit (spec issue-3639, R-011, D-011). + changes.AIGeneratedLinesAdded, changes.AIGeneratedLinesRemoved = self.GeneratedAdded, self.GeneratedRemoved } // fileSplit is the AI/human split of the changed lines of one file. diff --git a/app/cli/internal/trace/attribution/enrich_test.go b/app/cli/internal/trace/attribution/enrich_test.go index 7c48dc1c9..58f5d6fad 100644 --- a/app/cli/internal/trace/attribution/enrich_test.go +++ b/app/cli/internal/trace/attribution/enrich_test.go @@ -189,6 +189,61 @@ func TestEnrich_Legacy(t *testing.T) { }) } +func TestEnrich_GeneratedLines(t *testing.T) { + withGenerated := func(attr *state.AILineAttribution, added, removed int) *state.AILineAttribution { + attr.GeneratedAdded, attr.GeneratedRemoved = added, removed + + return attr + } + + testCases := []struct { + name string + ledgers map[string]*state.AILineAttribution + wantAdded, wantRemoved int + }{ + { + // The agent wrote 10 lines and removed 3. Only 2 lines reached + // the commit; the session still reports all of them (spec + // issue-3639, D-011). + name: "reports all the lines of the agent edits, also the ones that no commit holds", + ledgers: map[string]*state.AILineAttribution{ + testSession: withGenerated(ledger(testSession, testFile, []string{"a", "b"}, nil), 10, 3), + }, + wantAdded: 10, + wantRemoved: 3, + }, + { + name: "does not add the lines of another session", + ledgers: map[string]*state.AILineAttribution{ + testSession: withGenerated(ledger(testSession, testFile, []string{"a", "b"}, nil), 2, 0), + otherSession: withGenerated(ledger(otherSession, testFile, []string{"c"}, nil), 7, 1), + }, + wantAdded: 2, + }, + { + name: "a session with no ledger reports no lines", + ledgers: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + changes := &aicodingsession.CodeChanges{ + LinesAdded: 2, + Files: []aicodingsession.FileChange{{Path: testFile, LinesAdded: 2}}, + } + commits := []CommitLines{{SessionIDs: []string{testSession}, Files: []tracegit.FileLines{ + {Path: testFile, Added: []string{"a", "b"}}, + }}} + + Enrich(testSession, tc.ledgers, commits, nil, changes) + + assert.Equal(t, tc.wantAdded, changes.AIGeneratedLinesAdded) + assert.Equal(t, tc.wantRemoved, changes.AIGeneratedLinesRemoved) + }) + } +} + func TestEnrich_Lines(t *testing.T) { type split struct{ aiAdded, aiRemoved, humanAdded, humanRemoved int } diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go index 726379afb..e6c5470c5 100644 --- a/app/cli/internal/trace/e2e/checks.go +++ b/app/cli/internal/trace/e2e/checks.go @@ -99,9 +99,7 @@ var features = map[string]Feature{ "Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}}, "Partial commits": {Checks: []Check{{scenarioPartial, checkBothCommitsCredited}}}, "Line ranges": {Checks: []Check{{scenarioEditAndCommit, checkLineRanges("greeting.txt", 3)}}}, - "Generated line counts": { - NotCovered: "neither the user nor the evidence can see the generated line counts yet", - }, + "Generated line counts": {Checks: []Check{{scenarioEditAndCommit, checkGeneratedLineCounts}}}, // User experience "Welcome message": {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}}, @@ -196,6 +194,23 @@ func checkFileAttributedToAI(path string) func(s *Session) error { } } +// checkGeneratedLineCounts verifies that the evidence reports the lines of +// all the agent edits. Each net AI line came from an agent edit, so the +// generated lines are at least the net AI lines. +func checkGeneratedLineCounts(s *Session) error { + cc := evidence(s).CodeChanges + if cc == nil { + return errors.New("the evidence has no code changes") + } + if cc.AIGeneratedLinesAdded == 0 { + return errors.New("the evidence has no generated lines added") + } + if cc.AIGeneratedLinesAdded < cc.AILinesAdded { + return fmt.Errorf("%d generated lines added, fewer than the %d net AI lines added", cc.AIGeneratedLinesAdded, cc.AILinesAdded) + } + return nil +} + // hookMessage returns the first hook message of an event, or errNotCovered // when the harness does not report hook messages. func hookMessage(s *Session, event string) (*HookMessage, error) { diff --git a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json index fbfaf42ad..02ad74465 100644 --- a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json +++ b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json @@ -170,27 +170,35 @@ }, "lines_added": { "type": "integer", - "description": "Total lines added" + "description": "Net lines added by the session's commits (AI plus human); raw diff totals if a commit is unreadable" }, "lines_removed": { "type": "integer", - "description": "Total lines removed" + "description": "Net lines removed by the session's commits (AI plus human); raw diff totals if a commit is unreadable" }, "ai_lines_added": { "type": "integer", - "description": "Lines added attributed to AI" + "description": "Net committed lines added, attributed to AI" }, "ai_lines_removed": { "type": "integer", - "description": "Lines removed attributed to AI" + "description": "Net committed lines removed, attributed to AI" }, "human_lines_added": { "type": "integer", - "description": "Lines added attributed to human" + "description": "Net committed lines added, attributed to human" }, "human_lines_removed": { "type": "integer", - "description": "Lines removed attributed to human" + "description": "Net committed lines removed, attributed to human" + }, + "ai_generated_lines_added": { + "type": "integer", + "description": "All lines added by the agent edits of the session, committed or not; not net" + }, + "ai_generated_lines_removed": { + "type": "integer", + "description": "All lines removed by the agent edits of the session, committed or not; not net" }, "files": { "type": "array", diff --git a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go index fbe4a7ae7..60d131822 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go +++ b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go @@ -252,16 +252,22 @@ type FileChange struct { // CodeChanges summarizes code modifications made during the session. type CodeChanges struct { - FilesModified int `json:"files_modified,omitempty"` - FilesCreated int `json:"files_created,omitempty"` - FilesDeleted int `json:"files_deleted,omitempty"` - LinesAdded int `json:"lines_added,omitempty"` - LinesRemoved int `json:"lines_removed,omitempty"` - AILinesAdded int `json:"ai_lines_added,omitempty"` - AILinesRemoved int `json:"ai_lines_removed,omitempty"` - HumanLinesAdded int `json:"human_lines_added,omitempty"` - HumanLinesRemoved int `json:"human_lines_removed,omitempty"` - Files []FileChange `json:"files,omitempty"` + FilesModified int `json:"files_modified,omitempty"` + FilesCreated int `json:"files_created,omitempty"` + FilesDeleted int `json:"files_deleted,omitempty"` + // Net lines of the session's commits (AI plus human); raw diff totals if a commit is unreadable. + LinesAdded int `json:"lines_added,omitempty"` + LinesRemoved int `json:"lines_removed,omitempty"` + // Net committed lines attributed to the AI. + AILinesAdded int `json:"ai_lines_added,omitempty"` + AILinesRemoved int `json:"ai_lines_removed,omitempty"` + // Net committed lines attributed to a human. + HumanLinesAdded int `json:"human_lines_added,omitempty"` + HumanLinesRemoved int `json:"human_lines_removed,omitempty"` + // All lines of the session's agent edits, committed or not; not net. + AIGeneratedLinesAdded int `json:"ai_generated_lines_added,omitempty"` + AIGeneratedLinesRemoved int `json:"ai_generated_lines_removed,omitempty"` + Files []FileChange `json:"files,omitempty"` } // Model holds information about the AI models used in the session.