Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions app/cli/internal/trace/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,14 @@ Which lines of a commit the AI wrote.
| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial |
| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the sum of the removals of the session's commits (spec issue-3639). | Partial | Partial | Partial | Partial |
| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial |
| Renames and rewrites | The AI/human split survives a file rename, an amend, a rebase, a squash, and a cherry-pick. The lines match the trace state by content, and blame follows a renamed file to its lines from before the rename (spec issue-3639). | Partial | Partial | Partial | Partial |
Comment thread
jiparis marked this conversation as resolved.

Why not Yes:

- **Line attribution, partial commits and generated line counts, Cursor:** Cursor has only the `afterFileEdit` hook, which gives the edits but no snapshot before the edit. The provider rebuilds the content before the edit from the edits.
- **Net contribution, all agents:** the push finds the last writer of each line with `git blame`. A later commit that only re-indents a line takes the line, because blame sees a change. For Cursor, the limits of line attribution apply too.
- **Session updates, all agents:** the update works only in the clone where the work happened, while the CLI keeps the trace state of the session (30 days), and only when the commits of the session are on the branch that the push sends.
- **Renames and rewrites, all agents:** this works only in the clone where the work happened. Git finds a rename only when the file keeps at least half of its content. Two pushed commits can rename a file one after the other. A later change to the file under its middle name does not send the session again. For Cursor, the limits of line attribution apply too.
- **Shell changes attribution, all agents:** the hook before a shell command records only a hash of each file, not its content. The file at the `HEAD` commit from before the command stands in for the content before the command. Uncommitted lines that an earlier agent edit recorded are not recorded again, but uncommitted human lines of a file that the command changes count as AI.
- **Shell changes attribution, Cursor:** the provider installs the `preToolUse` and `postToolUse` hooks for the `Shell` tool. These hooks are verified for the Cursor IDE only. It is not verified that the Cursor CLI sends them.

Expand Down
124 changes: 102 additions & 22 deletions app/cli/internal/trace/attribution/enrich.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
package attribution

import (
"maps"
"slices"

"github.com/chainloop-dev/chainloop/app/cli/internal/trace"
tracegit "github.com/chainloop-dev/chainloop/app/cli/internal/trace/git"
"github.com/chainloop-dev/chainloop/app/cli/internal/trace/state"
Expand Down Expand Up @@ -57,6 +60,12 @@ type CommitLines struct {
// back to the sum of the added lines of each commit. Removed lines are always
// the sum over the session's own commits.
//
// A file that was renamed after a commit of the session has the lines of its
// new path in tip (R-007). A line there counts for the path that its commit
// changed, so a session that changed the file under both paths counts each
// line once. A commit that renamed a file matches the ledger of the new path,
// then of the old one.
//
// A file with an edit recorded before line fingerprints existed keeps the
// earlier rule: AI lines are the recorded ranges, capped at the file's diff.
func Enrich(sessionID string, ledgers map[string]*state.AILineAttribution, commits []CommitLines, tip map[string][]tracegit.BlameLine, changes *aicodingsession.CodeChanges) {
Expand All @@ -75,15 +84,25 @@ func Enrich(sessionID string, ledgers map[string]*state.AILineAttribution, commi
s.aiAdded, s.humanAdded = net.aiAdded, net.humanAdded
}

oldPaths := make(map[string][]string)
for _, c := range commits {
for _, fl := range c.Files {
if fl.OldPath != "" && !slices.Contains(oldPaths[fl.Path], fl.OldPath) {
oldPaths[fl.Path] = append(oldPaths[fl.Path], fl.OldPath)
}
}
}

changes.AILinesAdded, changes.AILinesRemoved = 0, 0
changes.HumanLinesAdded, changes.HumanLinesRemoved = 0, 0
for i := range changes.Files {
f := &changes.Files[i]

// The marker keeps its earlier meaning, a file that the session
// edited, for platforms that do not read the split (R-009). An empty
// ranges slice is valid: the agent only deleted lines.
if aiRanges, ok := self.Files[f.Path]; ok {
// edited, also under the path that a commit renamed, for platforms
// that do not read the split (R-009). An empty ranges slice is valid:
// the agent only deleted lines.
if aiRanges, ok := editedRanges(self, f.Path, oldPaths[f.Path]); ok {
f.Attribution = trace.AttributionAI
f.LineRanges = aiRanges
f.SessionIDs = []string{sessionID}
Expand Down Expand Up @@ -117,6 +136,18 @@ type fileSplit struct {
aiAdded, aiRemoved, humanAdded, humanRemoved int
}

// editedRanges returns the recorded ranges of the session's edits of a file,
// under its path or, when a commit renamed it, under an old path.
func editedRanges(l *state.AILineAttribution, path string, oldPaths []string) ([]aicodingsession.LineRange, bool) {
for _, p := range append([]string{path}, oldPaths...) {
if ranges, ok := l.Files[p]; ok {
return ranges, true
}
}

return nil, false
}

// splitByPath matches the lines of each commit against the ledgers, and
// returns the split of each file that the commits changed.
func splitByPath(sessionID string, self *state.AILineAttribution, ledgers map[string]*state.AILineAttribution, commits []CommitLines) map[string]*fileSplit {
Expand All @@ -142,19 +173,19 @@ func splitByPath(sessionID string, self *state.AILineAttribution, ledgers map[st
for _, line := range fl.Added {
fp := Fingerprint(line)
switch {
case written.take(self, fl.Path, fp):
case written.takeFile(self, fl, fp):
s.aiAdded++
case written.takeAny(others, fl.Path, fp):
case written.takeAny(others, fl, fp):
default:
s.humanAdded++
}
}
for _, line := range fl.Removed {
fp := Fingerprint(line)
switch {
case erased.take(self, fl.Path, fp):
case erased.takeFile(self, fl, fp):
s.aiRemoved++
case erased.takeAny(others, fl.Path, fp):
case erased.takeAny(others, fl, fp):
default:
s.humanRemoved++
}
Expand All @@ -178,37 +209,39 @@ func netAdded(sessionID string, self *state.AILineAttribution, ledgers map[strin
written := linePool{pick: func(a *state.AILineAttribution) map[string]state.LineCounts { return a.Written }}
joined := make(map[[2]string]state.LineCounts)

// Two paths of the session can have the same lines in tip, after a
// rename. The order makes the split of such lines the same on each push.
out := make(map[string]*fileSplit, len(tip))
for path, lines := range tip {
for _, path := range slices.Sorted(maps.Keys(tip)) {
s := &fileSplit{}
out[path] = s

for _, l := range lines {
for _, l := range tip[path] {
commit, ok := bySHA[l.SHA]
if !ok {
continue
}
fp := Fingerprint(l.Text)

if commit.Joined != nil {
key := [2]string{commit.SHA, path}
pool, ok := joined[key]
if !ok {
pool = state.LineCounts{}
pool.Add(commit.Joined[path]...)
joined[key] = pool
}
if pool.Take(fp) {
if joinedPool(joined, commit, path, tip).Take(fp) {
s.humanAdded++
}

continue
}

// A line that the commit wrote under another path of the
// session counts for that path.
fl, ok := commit.file(path)
if !ok {
continue
}

switch {
case written.take(self, path, fp):
case written.takeFile(self, fl, fp):
s.aiAdded++
case written.takeAny(otherLedgers(sessionID, ledgers, commit.SessionIDs), path, fp):
case written.takeAny(otherLedgers(sessionID, ledgers, commit.SessionIDs), fl, fp):
default:
s.humanAdded++
}
Expand All @@ -218,6 +251,47 @@ func netAdded(sessionID string, self *state.AILineAttribution, ledgers map[strin
return out
}

// file returns the lines that the commit changed in path.
func (c CommitLines) file(path string) (tracegit.FileLines, bool) {
for _, fl := range c.Files {
if fl.Path == path {
return fl, true
}
}

return tracegit.FileLines{}, false
}

// joinedPool returns the human lines that a joined commit added to path, one
// copy per match. A joined commit that changed the file under a path that tip
// does not hold, because a later commit renamed it, shares one pool of those
// lines across the paths of tip.
func joinedPool(pools map[[2]string]state.LineCounts, commit CommitLines, path string, tip map[string][]tracegit.BlameLine) state.LineCounts {
key := [2]string{commit.SHA, path}
if _, direct := commit.Joined[path]; !direct {
key[1] = ""
}

pool, ok := pools[key]
if ok {
return pool
}

pool = state.LineCounts{}
if key[1] != "" {
pool.Add(commit.Joined[path]...)
} else {
for p, fps := range commit.Joined {
Comment thread
jiparis marked this conversation as resolved.
if _, inTip := tip[p]; !inTip {
pool.Add(fps...)
}
}
}
pools[key] = pool

return pool
}

// otherLedgers returns the ledgers of the sessions in ids other than
// sessionID.
func otherLedgers(sessionID string, ledgers map[string]*state.AILineAttribution, ids []string) []*state.AILineAttribution {
Expand Down Expand Up @@ -256,10 +330,16 @@ func (p *linePool) take(l *state.AILineAttribution, path, fp string) bool {
return c.Take(fp)
}

// takeAny takes fp from the first ledger that holds it.
func (p *linePool) takeAny(ledgers []*state.AILineAttribution, path, fp string) bool {
// takeFile takes fp from the lines of ledger l for the file, under its path,
// and then under its old path when the commit renamed it.
func (p *linePool) takeFile(l *state.AILineAttribution, fl tracegit.FileLines, fp string) bool {
return p.take(l, fl.Path, fp) || (fl.OldPath != "" && p.take(l, fl.OldPath, fp))
}

// takeAny takes fp from the first ledger that holds it for the file.
func (p *linePool) takeAny(ledgers []*state.AILineAttribution, fl tracegit.FileLines, fp string) bool {
for _, l := range ledgers {
if p.take(l, path, fp) {
if p.takeFile(l, fl, fp) {
return true
}
}
Expand Down
57 changes: 57 additions & 0 deletions app/cli/internal/trace/attribution/enrich_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,17 @@ func ledger(sessionID, file string, written, erased []string) *state.AILineAttri
}
}

// oldFile is the path of testFile before a rename.
const oldFile = "old.go"

// counts holds the fingerprints of lines.
func counts(lines ...string) state.LineCounts {
c := state.LineCounts{}
c.Add(fps(lines...)...)

return c
}

func TestEnrich_Legacy(t *testing.T) {
t.Run("no AI data attributes all changes to human", func(t *testing.T) {
changes := &aicodingsession.CodeChanges{
Expand Down Expand Up @@ -253,6 +264,16 @@ func TestEnrich_Lines(t *testing.T) {
},
want: split{aiAdded: 2, humanAdded: 1, humanRemoved: 1},
},
{
// R-007: the agent edited the file under its old name, and the
// commit renamed it.
name: "a renamed file matches the ledger of its old path",
ledgers: map[string]*state.AILineAttribution{testSession: ledger(testSession, oldFile, []string{"a"}, []string{"x"})},
commits: []CommitLines{{SessionIDs: []string{testSession}, Files: []tracegit.FileLines{
{Path: testFile, OldPath: oldFile, Added: []string{"a", "b"}, Removed: []string{"x", "y"}},
}}},
want: split{aiAdded: 1, aiRemoved: 1, humanAdded: 1, humanRemoved: 1},
},
}

for _, tc := range testCases {
Expand Down Expand Up @@ -353,6 +374,42 @@ func TestEnrich_Net(t *testing.T) {
tip: map[string][]tracegit.BlameLine{testFile: nil},
want: split{aiRemoved: 1, humanRemoved: 5},
},
{
// R-007: the session wrote old.go, a commit renamed it to the
// file, and a later commit of the session changed the file. The
// tip of both paths is the file at HEAD, and each line counts for
// the path that its commit changed.
name: "a line counts for the path that its commit changed",
ledgers: map[string]*state.AILineAttribution{testSession: {
SessionID: testSession,
Written: map[string]state.LineCounts{
oldFile: counts(five...),
testFile: counts("f"),
},
}},
commits: []CommitLines{
{SHA: c1, SessionIDs: []string{testSession}, Files: []tracegit.FileLines{{Path: oldFile, Added: five}}},
ownCommit(c2, []string{"f"}, nil),
},
tip: map[string][]tracegit.BlameLine{
oldFile: tipOf(c1+":a", c1+":b", c1+":c", c1+":d", c1+":e", c2+":f"),
testFile: tipOf(c1+":a", c1+":b", c1+":c", c1+":d", c1+":e", c2+":f"),
},
want: split{aiAdded: 1},
},
{
// R-007: a commit in no session renamed the file and replaced a
// line of the session in it.
name: "a joined commit counts in a renamed file",
ledgers: map[string]*state.AILineAttribution{testSession: ledger(testSession, testFile, five, nil)},
commits: []CommitLines{
ownCommit(c1, five, nil),
{SHA: c2, Files: []tracegit.FileLines{{Path: "new.go", OldPath: testFile, Added: []string{"C"}, Removed: []string{"c"}}},
Joined: map[string][]string{"new.go": fps("C")}},
},
tip: map[string][]tracegit.BlameLine{testFile: tipOf(c1+":a", c1+":b", c2+":C", c1+":d", c1+":e")},
want: split{aiAdded: 4, humanAdded: 1},
},
}

for _, tc := range testCases {
Expand Down
11 changes: 9 additions & 2 deletions app/cli/internal/trace/git/git.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@ const (
// FileLines holds the lines that one commit added to and removed from a file,
// without their line ends.
type FileLines struct {
Path string
Path string
// OldPath is the path of the file in the parent, set when the commit
// renamed the file.
OldPath string
Added []string
Removed []string
// Hunks holds the same lines, split into the hunks of the diff.
Expand Down Expand Up @@ -91,8 +94,12 @@ type Client interface {
CodeChangesForCommits(repoRoot string, shas []string) (*aicodingsession.CodeChanges, error)
// CommitLines returns, for each file that the commit changed against its
// first parent, the content of the lines it added and removed. Binary
// files are left out. Renames show as a removal and an addition.
// files are left out. A renamed file shows the lines that changed, under
// its new path, with its old path.
CommitLines(repoRoot, sha string) ([]FileLines, error)
// Renames returns, for each file that was renamed between two revisions,
// its new path by its old path.
Renames(repoRoot, from, to string) (map[string]string, error)
// FileAt returns the content of a repo-relative path at a commit, or nil
// when the commit does not hold the path.
FileAt(repoRoot, sha, path string) ([]byte, error)
Expand Down
Loading
Loading