Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/cli/internal/trace/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ Which lines of a commit the AI wrote.
|---------|---------------|-------------|--------|--------------|------------|---|---|
| Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | No |
| Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | No |
| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence does not report them yet (spec issue-3639). | Yes | Partial | Yes | Yes | No | No |
| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence reports the totals of the session in `ai_generated_lines_added` and `ai_generated_lines_removed`. A session with no commit sends no evidence, so it reports no totals (spec issue-3639). | Yes | Partial | Yes | Yes | No | No |
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | No |
| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the removals of the session's commits, less the lines that a commit of the session wrote. The line counts of each file are the sum of its AI and human lines, and a file that the session created stays created when a later commit of the session changes it (spec issue-3639, issue 3674). | Partial | Partial | Partial | Partial | No | No |
| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | No |
Expand Down
4 changes: 4 additions & 0 deletions app/cli/internal/trace/attribution/enrich.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,10 @@ func Enrich(sessionID string, ledgers map[string]*state.AILineAttribution, commi
changes.HumanLinesAdded += f.HumanLinesAdded
changes.HumanLinesRemoved += f.HumanLinesRemoved
}

// All the lines that the agent edits of the session wrote, also the ones
// that never reached a commit (spec issue-3639, R-011, D-011).
changes.AIGeneratedLinesAdded, changes.AIGeneratedLinesRemoved = self.GeneratedAdded, self.GeneratedRemoved
}

// fileSplit is the AI/human split of the changed lines of one file.
Expand Down
55 changes: 55 additions & 0 deletions app/cli/internal/trace/attribution/enrich_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,61 @@ func TestEnrich_Legacy(t *testing.T) {
})
}

func TestEnrich_GeneratedLines(t *testing.T) {
withGenerated := func(attr *state.AILineAttribution, added, removed int) *state.AILineAttribution {
attr.GeneratedAdded, attr.GeneratedRemoved = added, removed

return attr
}

testCases := []struct {
name string
ledgers map[string]*state.AILineAttribution
wantAdded, wantRemoved int
}{
{
// The agent wrote 10 lines and removed 3. Only 2 lines reached
// the commit; the session still reports all of them (spec
// issue-3639, D-011).
name: "reports all the lines of the agent edits, also the ones that no commit holds",
ledgers: map[string]*state.AILineAttribution{
testSession: withGenerated(ledger(testSession, testFile, []string{"a", "b"}, nil), 10, 3),
},
wantAdded: 10,
wantRemoved: 3,
},
{
name: "does not add the lines of another session",
ledgers: map[string]*state.AILineAttribution{
testSession: withGenerated(ledger(testSession, testFile, []string{"a", "b"}, nil), 2, 0),
otherSession: withGenerated(ledger(otherSession, testFile, []string{"c"}, nil), 7, 1),
},
wantAdded: 2,
},
{
name: "a session with no ledger reports no lines",
ledgers: nil,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
changes := &aicodingsession.CodeChanges{
LinesAdded: 2,
Files: []aicodingsession.FileChange{{Path: testFile, LinesAdded: 2}},
}
commits := []CommitLines{{SessionIDs: []string{testSession}, Files: []tracegit.FileLines{
{Path: testFile, Added: []string{"a", "b"}},
}}}

Enrich(testSession, tc.ledgers, commits, nil, changes)

assert.Equal(t, tc.wantAdded, changes.AIGeneratedLinesAdded)
assert.Equal(t, tc.wantRemoved, changes.AIGeneratedLinesRemoved)
})
}
}

func TestEnrich_Lines(t *testing.T) {
type split struct{ aiAdded, aiRemoved, humanAdded, humanRemoved int }

Expand Down
21 changes: 18 additions & 3 deletions app/cli/internal/trace/e2e/checks.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,7 @@ var features = map[string]Feature{
"Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}},
"Partial commits": {Checks: []Check{{scenarioPartial, checkBothCommitsCredited}}},
"Line ranges": {Checks: []Check{{scenarioEditAndCommit, checkLineRanges("greeting.txt", 3)}}},
"Generated line counts": {
NotCovered: "neither the user nor the evidence can see the generated line counts yet",
},
"Generated line counts": {Checks: []Check{{scenarioEditAndCommit, checkGeneratedLineCounts}}},

// User experience
"Welcome message": {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}},
Expand Down Expand Up @@ -196,6 +194,23 @@ func checkFileAttributedToAI(path string) func(s *Session) error {
}
}

// checkGeneratedLineCounts verifies that the evidence reports the lines of
// all the agent edits. Each net AI line came from an agent edit, so the
// generated lines are at least the net AI lines.
func checkGeneratedLineCounts(s *Session) error {
cc := evidence(s).CodeChanges
if cc == nil {
return errors.New("the evidence has no code changes")
}
if cc.AIGeneratedLinesAdded == 0 {
return errors.New("the evidence has no generated lines added")
}
if cc.AIGeneratedLinesAdded < cc.AILinesAdded {
return fmt.Errorf("%d generated lines added, fewer than the %d net AI lines added", cc.AIGeneratedLinesAdded, cc.AILinesAdded)
}
return nil
}

// hookMessage returns the first hook message of an event, or errNotCovered
// when the harness does not report hook messages.
func hookMessage(s *Session, event string) (*HookMessage, error) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,27 +170,35 @@
},
"lines_added": {
"type": "integer",
"description": "Total lines added"
"description": "Net lines added by the session's commits (AI plus human); raw diff totals if a commit is unreadable"
},
"lines_removed": {
"type": "integer",
"description": "Total lines removed"
"description": "Net lines removed by the session's commits (AI plus human); raw diff totals if a commit is unreadable"
},
"ai_lines_added": {
"type": "integer",
"description": "Lines added attributed to AI"
"description": "Net committed lines added, attributed to AI"
},
"ai_lines_removed": {
"type": "integer",
"description": "Lines removed attributed to AI"
"description": "Net committed lines removed, attributed to AI"
},
"human_lines_added": {
"type": "integer",
"description": "Lines added attributed to human"
"description": "Net committed lines added, attributed to human"
},
"human_lines_removed": {
"type": "integer",
"description": "Lines removed attributed to human"
"description": "Net committed lines removed, attributed to human"
},
"ai_generated_lines_added": {
"type": "integer",
"description": "All lines added by the agent edits of the session, committed or not; not net"
},
"ai_generated_lines_removed": {
"type": "integer",
"description": "All lines removed by the agent edits of the session, committed or not; not net"
},
"files": {
"type": "array",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -252,16 +252,22 @@ type FileChange struct {

// CodeChanges summarizes code modifications made during the session.
type CodeChanges struct {
FilesModified int `json:"files_modified,omitempty"`
FilesCreated int `json:"files_created,omitempty"`
FilesDeleted int `json:"files_deleted,omitempty"`
LinesAdded int `json:"lines_added,omitempty"`
LinesRemoved int `json:"lines_removed,omitempty"`
AILinesAdded int `json:"ai_lines_added,omitempty"`
AILinesRemoved int `json:"ai_lines_removed,omitempty"`
HumanLinesAdded int `json:"human_lines_added,omitempty"`
HumanLinesRemoved int `json:"human_lines_removed,omitempty"`
Files []FileChange `json:"files,omitempty"`
FilesModified int `json:"files_modified,omitempty"`
FilesCreated int `json:"files_created,omitempty"`
FilesDeleted int `json:"files_deleted,omitempty"`
// Net lines of the session's commits (AI plus human); raw diff totals if a commit is unreadable.
LinesAdded int `json:"lines_added,omitempty"`
LinesRemoved int `json:"lines_removed,omitempty"`
// Net committed lines attributed to the AI.
AILinesAdded int `json:"ai_lines_added,omitempty"`
AILinesRemoved int `json:"ai_lines_removed,omitempty"`
// Net committed lines attributed to a human.
HumanLinesAdded int `json:"human_lines_added,omitempty"`
HumanLinesRemoved int `json:"human_lines_removed,omitempty"`
// All lines of the session's agent edits, committed or not; not net.
AIGeneratedLinesAdded int `json:"ai_generated_lines_added,omitempty"`
AIGeneratedLinesRemoved int `json:"ai_generated_lines_removed,omitempty"`
Files []FileChange `json:"files,omitempty"`
}

// Model holds information about the AI models used in the session.
Expand Down
Loading