diff --git a/app/cli/internal/trace/claude/hooks.go b/app/cli/internal/trace/claude/hooks.go index 2b975c01c..89560326b 100644 --- a/app/cli/internal/trace/claude/hooks.go +++ b/app/cli/internal/trace/claude/hooks.go @@ -57,6 +57,10 @@ var commandTools = []string{"Bash"} // kind fired. var hookToolMatcher = strings.Join(append(append([]string{}, fileWritingTools...), commandTools...), "|") +// postHookToolMatcher also covers the Skill tool. The skill is loaded only +// after the call, so the post hook is the first one that can copy its folder. +var postHookToolMatcher = hookToolMatcher + "|" + skillTool + type hookEvent struct { event string command string @@ -67,7 +71,7 @@ var hookEvents = []hookEvent{ {eventSessionStart, "chainloop trace hook claude session-start", ""}, {eventUserPromptSubmit, "chainloop trace hook claude user-prompt-submit", ""}, {eventPreToolUse, "chainloop trace hook claude pre-tool-use", hookToolMatcher}, - {eventPostToolUse, "chainloop trace hook claude post-tool-use", hookToolMatcher}, + {eventPostToolUse, "chainloop trace hook claude post-tool-use", postHookToolMatcher}, {eventPostToolUseFailure, "chainloop trace hook claude post-tool-use", hookToolMatcher}, {eventSessionEnd, "chainloop trace hook claude session-end", ""}, } diff --git a/app/cli/internal/trace/claude/hooks_test.go b/app/cli/internal/trace/claude/hooks_test.go index 5f666f451..ac7855648 100644 --- a/app/cli/internal/trace/claude/hooks_test.go +++ b/app/cli/internal/trace/claude/hooks_test.go @@ -64,12 +64,13 @@ func TestInstallHooks(t *testing.T) { preEntry := preEntries[0].(map[string]any) assert.Equal(t, "Edit|Write|MultiEdit|Bash", preEntry["matcher"]) - // PostToolUse should have matcher + // PostToolUse also matches Skill, to copy the folder of a loaded skill postEntries := hooks["PostToolUse"].([]any) postEntry := postEntries[0].(map[string]any) - assert.Equal(t, "Edit|Write|MultiEdit|Bash", postEntry["matcher"]) + assert.Equal(t, "Edit|Write|MultiEdit|Bash|Skill", postEntry["matcher"]) - // PostToolUseFailure should have the same matcher + // PostToolUseFailure keeps the edit matcher: a failed Skill call loads + // no skill failureEntries := hooks["PostToolUseFailure"].([]any) failureEntry := failureEntries[0].(map[string]any) assert.Equal(t, "Edit|Write|MultiEdit|Bash", failureEntry["matcher"]) diff --git a/app/cli/internal/trace/claude/skills.go b/app/cli/internal/trace/claude/skills.go new file mode 100644 index 000000000..48cab1805 --- /dev/null +++ b/app/cli/internal/trace/claude/skills.go @@ -0,0 +1,442 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package claude + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" +) + +// Compile-time check that Provider records skills. +var _ trace.SkillTracker = (*Provider)(nil) + +// skillTool is the tool that the model calls to load a skill. +const skillTool = "Skill" + +// mainTranscriptKey names the main transcript in the scan offsets of a +// session. Subagent transcripts use their file name. +const mainTranscriptKey = "main" + +// commandNameRe finds the slash command of a user message, as Claude Code +// writes it: /graphify. +var commandNameRe = regexp.MustCompile(`\s*/?([^<\s]+)\s*`) + +// skillRecord holds the fields of a transcript record that tell a skill use. +type skillRecord struct { + Type string `json:"type"` + UUID string `json:"uuid"` + ParentUUID string `json:"parentUuid"` + Timestamp string `json:"timestamp"` + IsMeta bool `json:"isMeta"` + // SourceToolUseID links the message that holds a loaded skill to the + // tool call that loaded it. A skill that the user loaded has none. + SourceToolUseID string `json:"sourceToolUseID"` + Message *struct { + Content json.RawMessage `json:"content"` + } `json:"message"` +} + +// skillBlock is a content block of a transcript message. +type skillBlock struct { + Type string `json:"type"` + Text string `json:"text"` + ID string `json:"id"` + Name string `json:"name"` + Input struct { + Skill string `json:"skill"` + } `json:"input"` +} + +// text returns the text of a message, whether the content is a string or a +// list of blocks. +func (r *skillRecord) text() string { + if r.Message == nil { + return "" + } + + var s string + if json.Unmarshal(r.Message.Content, &s) == nil { + return s + } + + var b strings.Builder + for _, block := range r.blocks() { + if block.Type == "text" { + b.WriteString(block.Text) + b.WriteString("\n") + } + } + + return b.String() +} + +func (r *skillRecord) blocks() []skillBlock { + if r.Message == nil { + return nil + } + + var blocks []skillBlock + if json.Unmarshal(r.Message.Content, &blocks) != nil { + return nil + } + + return blocks +} + +// mayHoldSkillUse is a cheap test that a scan runs before it decodes a line. +// Only the lines that start or load a skill pass it: the meta messages, the +// skill tool calls and the slash commands. The command marker is matched +// without its brackets, which a JSON encoder can escape. +func mayHoldSkillUse(line []byte) bool { + return bytes.Contains(line, []byte(`"isMeta":true`)) || + bytes.Contains(line, []byte(`"name":"`+skillTool+`"`)) || + bytes.Contains(line, []byte("command-name")) +} + +// start is the record that started a skill use: a tool call or a slash +// command. +type start struct { + name string + timestamp string +} + +// transcriptSkills counts the skill uses of one transcript. +// +// Claude Code writes a meta message each time it loads a skill. A meta message +// that names a skill tool call is a model start, and one whose parent is a +// slash command is a user start. A built-in command such as /clear gets no +// meta message, and a failed tool call loads nothing, so neither counts. Each +// start counts once, also when the transcript holds its meta message more than +// once. +// +// The meta message of a skill with a folder starts with "Base directory for +// this skill:". A skill built into Claude Code, such as /simplify, and a +// custom command have no folder: their use is returned with no Dir, and is +// not recorded. +func transcriptSkills(path string, subagent bool) (map[string]*trace.SkillUse, error) { + records, err := readSkillRecords(path) + if err != nil { + return nil, err + } + + toolCalls, commands := skillStarts(records) + + uses := make(map[string]*trace.SkillUse) + counted := make(map[string]bool) + for i := range records { + r := &records[i] + if r.Type != recordTypeUser || !r.IsMeta { + continue + } + + dir, _ := skill.BaseDir(r.text()) + + s, key, byModel, ok := startOf(r, dir, toolCalls, commands) + if !ok || counted[key] { + continue + } + counted[key] = true + + at := normalizeTimestamp(s.timestamp) + if at == "" { + at = normalizeTimestamp(r.Timestamp) + } + + use, ok := uses[s.name] + if !ok { + use = &trace.SkillUse{Name: s.name} + uses[s.name] = use + } + if use.Dir == "" { + use.Dir = dir + } + use.Record(at, byModel, subagent) + } + + return uses, nil +} + +// readSkillRecords returns the records of a transcript that can start or +// load a skill. +func readSkillRecords(path string) ([]skillRecord, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + + var records []skillRecord + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 0, 64*1024), maxScannerBuffer) + for scanner.Scan() { + line := scanner.Bytes() + if !mayHoldSkillUse(line) { + continue + } + + var r skillRecord + if json.Unmarshal(line, &r) != nil { + continue + } + records = append(records, r) + } + + return records, scanner.Err() +} + +// skillStarts returns the skill tool calls by tool call ID, and the slash +// commands by message ID. +func skillStarts(records []skillRecord) (toolCalls, commands map[string]start) { + toolCalls = make(map[string]start) + commands = make(map[string]start) + for i := range records { + r := &records[i] + switch r.Type { + case recordTypeAssistant: + for _, b := range r.blocks() { + if b.Type == "tool_use" && b.Name == skillTool && b.ID != "" { + toolCalls[b.ID] = start{name: strings.TrimSpace(b.Input.Skill), timestamp: r.Timestamp} + } + } + case recordTypeUser: + if r.IsMeta || r.UUID == "" { + continue + } + if m := commandNameRe.FindStringSubmatch(r.text()); m != nil { + commands[r.UUID] = start{name: m[1], timestamp: r.Timestamp} + } + } + } + + return toolCalls, commands +} + +// startOf returns the start of the skill use that a meta message records, and +// a key that is the same for each meta message of that one start. It reports +// false for a meta message that no start explains. +func startOf(r *skillRecord, dir string, toolCalls, commands map[string]start) (s start, key string, byModel, ok bool) { + if r.SourceToolUseID != "" { + s, ok = toolCalls[r.SourceToolUseID] + if !ok && dir != "" { + // The tool call is not in this transcript. The meta message still + // tells that a skill loaded, and its folder names it. + s = start{name: filepath.Base(dir), timestamp: r.Timestamp} + } + + return s, "tool:" + r.SourceToolUseID, true, s.name != "" + } + + s, ok = commands[r.ParentUUID] + + return s, "command:" + r.ParentUUID, false, ok && s.name != "" +} + +// normalizeTimestamp returns a transcript timestamp as RFC3339 in UTC, or "" +// when it cannot be read. Claude Code writes milliseconds, and the other spec +// entries have seconds. +func normalizeTimestamp(ts string) string { + t, err := time.Parse(time.RFC3339Nano, ts) + if err != nil { + return "" + } + + return t.UTC().Format(time.RFC3339) +} + +// SessionSkills returns the skills that the session used, from the main +// transcript and the subagent transcripts copied under opts.SessionDir. +func (p *Provider) SessionSkills(opts *trace.ParseOpts) ([]trace.SkillUse, []string, error) { + jsonlPath, err := findJSONLPath(opts.SessionDir, opts.SessionID) + if err != nil { + return nil, nil, err + } + + merged, err := transcriptSkills(jsonlPath, false) + if err != nil { + return nil, nil, fmt.Errorf("read session file: %w", err) + } + + var warnings []string + subagents, _ := filepath.Glob(filepath.Join(state.RawSubagentDir(opts.SessionDir, opts.SessionID), "agent-*.jsonl")) + for _, path := range subagents { + uses, err := transcriptSkills(path, true) + if err != nil { + warnings = append(warnings, fmt.Sprintf("the skills of subagent %s were not recorded: it could not be read", filepath.Base(path))) + continue + } + + for name, use := range uses { + if cur, ok := merged[name]; ok { + cur.Merge(*use) + } else { + merged[name] = use + } + } + } + + return trace.SortedSkillUses(merged), warnings, nil +} + +// NewSkillLoads reads the transcripts of the session from where the last +// hook stopped, and returns the folder of each skill that they loaded since. +// It reads the live transcripts, not the copies in the state, because the +// hooks do not refresh the copies. +func (p *Provider) NewSkillLoads(store *state.Store, input *trace.HookInput) []string { + projectsDir := claudeProjectsDir() + if projectsDir == "" || !state.ValidSessionID(input.SessionID) { + return nil + } + + sourceDir, err := transcriptSourceDir(projectsDir, input.TranscriptPath, input.SessionID) + if err != nil { + if input.Cwd == "" { + return nil + } + sourceDir = filepath.Join(projectsDir, encodeCWDForClaudePath(input.Cwd)) + } + + offsets, err := store.SkillScanOffsets(input.SessionID) + if err != nil { + offsets = map[string]int64{} + } + + // The main transcript first, then the subagents, which Glob sorts. + subagents, _ := filepath.Glob(filepath.Join(sourceDir, input.SessionID, "subagents", "agent-*.jsonl")) + paths := append([]string{filepath.Join(sourceDir, input.SessionID+".jsonl")}, subagents...) + + var dirs []string + seen := make(map[string]bool) + changed := false + for i, path := range paths { + key := mainTranscriptKey + if i > 0 { + key = filepath.Base(path) + } + + found, next, err := scanSkillLoads(path, offsets[key]) + if err != nil { + continue + } + if next != offsets[key] { + offsets[key] = next + changed = true + } + for _, dir := range found { + if !seen[dir] { + seen[dir] = true + dirs = append(dirs, dir) + } + } + } + + if changed { + // A failure costs the next hook a scan of the same lines again. + _ = store.SetSkillScanOffsets(input.SessionID, offsets) + } + + return dirs +} + +// scanSkillLoads reads path from offset and returns the skill folders that +// the complete lines name, with the offset after the last complete line. A +// line that the agent is still writing is left for the next hook. +func scanSkillLoads(path string, offset int64) ([]string, int64, error) { + f, err := os.Open(path) + if err != nil { + return nil, offset, err + } + defer func() { _ = f.Close() }() + + info, err := f.Stat() + if err != nil { + return nil, offset, err + } + // A shorter file is a new file, so it is read from the start. + if info.Size() < offset { + offset = 0 + } + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return nil, offset, err + } + + var dirs []string + reader := bufio.NewReaderSize(f, 64*1024) + for { + line, err := reader.ReadBytes('\n') + if errors.Is(err, io.EOF) { + return dirs, offset, nil + } + if err != nil { + return dirs, offset, err + } + offset += int64(len(line)) + + if !skill.HasBaseDir(line) { + continue + } + + var r skillRecord + if json.Unmarshal(line, &r) != nil || r.Type != recordTypeUser || !r.IsMeta { + continue + } + if dir, ok := skill.BaseDir(r.text()); ok { + dirs = append(dirs, dir) + } + } +} + +// SkillRoots returns the Claude Code folders that tell where a skill came +// from. +func (p *Provider) SkillRoots(repoRoot string) skill.Roots { + roots := skill.Roots{ + Organization: []string{managedSettingsDir()}, + Project: []string{repoRoot}, + } + + if home, err := os.UserHomeDir(); err == nil { + roots.Plugin = []string{filepath.Join(home, ".claude", "plugins")} + roots.User = []string{filepath.Join(home, ".claude", "skills")} + } + + return roots +} + +// managedSettingsDir is the folder where an administrator installs the +// Claude Code configuration for all users of a machine. +func managedSettingsDir() string { + switch runtime.GOOS { + case "darwin": + return "/Library/Application Support/ClaudeCode" + case "windows": + return `C:\Program Files\ClaudeCode` + default: + return "/etc/claude-code" + } +} diff --git a/app/cli/internal/trace/claude/skills_test.go b/app/cli/internal/trace/claude/skills_test.go new file mode 100644 index 000000000..6103cc11f --- /dev/null +++ b/app/cli/internal/trace/claude/skills_test.go @@ -0,0 +1,336 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package claude + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const skillsSessionID = "5c584dae-abf9-4a94-8598-4dc1ce6579d0" + +const ( + steSkill = "asd-ste100" + graphSkill = "graphify" + tenOClock = "2026-10-07T10:00:00Z" + tenOClockM = "2026-10-07T10:00:00.000Z" + tenOhOne = "2026-10-07T10:01:00Z" +) + +// fixtureRecord, fixtureMessage and fixtureBlock have the shape that Claude +// Code 2.1 writes for a skill. +type fixtureRecord struct { + Type string `json:"type"` + UUID string `json:"uuid,omitempty"` + ParentUUID string `json:"parentUuid,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + IsMeta bool `json:"isMeta,omitempty"` + SourceToolUseID string `json:"sourceToolUseID,omitempty"` + Message fixtureMessage `json:"message"` +} + +type fixtureMessage struct { + Role string `json:"role"` + // Content is a string or a list of blocks. + Content any `json:"content"` +} + +type fixtureBlock struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Input map[string]string `json:"input,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + Content string `json:"content,omitempty"` + IsError bool `json:"is_error,omitempty"` +} + +func toolCallLine(t *testing.T, uuid, toolID, skillName, ts string) string { + t.Helper() + + return mustLine(t, fixtureRecord{ + Type: recordTypeAssistant, UUID: uuid, Timestamp: ts, + Message: fixtureMessage{Role: recordTypeAssistant, Content: []fixtureBlock{{ + Type: "tool_use", ID: toolID, Name: skillTool, + Input: map[string]string{"skill": skillName, "args": "private arguments"}, + }}}, + }) +} + +func toolResultLine(t *testing.T, uuid, parent, toolID, result string, isError bool, ts string) string { + t.Helper() + + return mustLine(t, fixtureRecord{ + Type: recordTypeUser, UUID: uuid, ParentUUID: parent, Timestamp: ts, + Message: fixtureMessage{Role: recordTypeUser, Content: []fixtureBlock{{ + Type: "tool_result", ToolUseID: toolID, Content: result, IsError: isError, + }}}, + }) +} + +func commandLine(t *testing.T, uuid, command, ts string) string { + t.Helper() + + return mustLine(t, fixtureRecord{ + Type: recordTypeUser, UUID: uuid, Timestamp: ts, + Message: fixtureMessage{ + Role: recordTypeUser, + Content: "" + command + "\n/" + command + "", + }, + }) +} + +// metaLine is the message that holds a loaded skill. toolID is empty for a +// skill that the user loaded, and dir for a skill with no folder. +func metaLine(t *testing.T, uuid, parent, toolID, dir, ts string) string { + t.Helper() + + text := "Base directory for this skill: " + dir + "\n\n# The skill\n\nBody." + if dir == "" { + text = "`/simplify` Review the changed code.\n\nBody." + } + + return mustLine(t, fixtureRecord{ + Type: recordTypeUser, UUID: uuid, ParentUUID: parent, Timestamp: ts, IsMeta: true, SourceToolUseID: toolID, + Message: fixtureMessage{Role: recordTypeUser, Content: []fixtureBlock{{ + Type: "text", Text: text, + }}}, + }) +} + +// mustLine encodes a record as Claude Code does, without escaping < and >. +func mustLine(t *testing.T, record fixtureRecord) string { + t.Helper() + + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + require.NoError(t, enc.Encode(record)) + + return strings.TrimSuffix(b.String(), "\n") +} + +func writeLines(t *testing.T, path string, lines ...string) { + t.Helper() + + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600)) +} + +func TestSessionSkills(t *testing.T) { + const ( + steDir = "/home/me/.claude/skills/asd-ste100" + graphDir = "/home/me/.claude/skills/graphify" + planDir = "/home/me/.claude/plugins/cache/x/br/1.0.0/skills/plan" + ) + + modelStart := func(t *testing.T, n, skillName, dir, ts string) []string { + return []string{ + toolCallLine(t, "a"+n, "toolu_"+n, skillName, ts), + toolResultLine(t, "r"+n, "a"+n, "toolu_"+n, "Launching skill: "+skillName, false, ts), + metaLine(t, "m"+n, "r"+n, "toolu_"+n, dir, ts), + } + } + userStart := func(t *testing.T, n, command, dir, ts string) []string { + return []string{ + commandLine(t, "c"+n, command, ts), + metaLine(t, "m"+n, "c"+n, "", dir, ts), + } + } + join := func(groups ...[]string) []string { + var out []string + for _, g := range groups { + out = append(out, g...) + } + return out + } + + testCases := []struct { + name string + main []string + subagents map[string][]string + want []trace.SkillUse + }{ + { + name: "no skill", + main: []string{commandLine(t, "c1", "clear", tenOClockM)}, + want: []trace.SkillUse{}, + }, + { + name: "a model start", + main: modelStart(t, "1", steSkill, steDir, "2026-10-07T10:14:51.622Z"), + want: []trace.SkillUse{{Name: steSkill, Dir: steDir, FirstUsedAt: "2026-10-07T10:14:51Z", ByModel: 1}}, + }, + { + name: "a user start", + main: userStart(t, "1", graphSkill, graphDir, tenOClockM), + want: []trace.SkillUse{{Name: graphSkill, Dir: graphDir, FirstUsedAt: tenOClock, ByUser: 1}}, + }, + { + name: "both starts for one skill, in the order of first use", + main: join( + userStart(t, "1", graphSkill, graphDir, tenOClockM), + modelStart(t, "2", steSkill, steDir, "2026-10-07T10:05:00.000Z"), + modelStart(t, "3", graphSkill, graphDir, "2026-10-07T10:10:00.000Z"), + ), + want: []trace.SkillUse{ + {Name: graphSkill, Dir: graphDir, FirstUsedAt: tenOClock, ByModel: 1, ByUser: 1}, + {Name: steSkill, Dir: steDir, FirstUsedAt: "2026-10-07T10:05:00Z", ByModel: 1}, + }, + }, + { + name: "a failed call does not count", + main: []string{ + toolCallLine(t, "a1", "toolu_1", "missing", tenOClockM), + toolResultLine(t, "r1", "a1", "toolu_1", "Unknown skill: missing", true, tenOClockM), + }, + want: []trace.SkillUse{}, + }, + { + name: "built-in and custom commands do not count", + main: []string{ + commandLine(t, "c1", "clear", tenOClockM), + commandLine(t, "c2", "model", "2026-10-07T10:00:01.000Z"), + commandLine(t, "c3", "my-custom-command", "2026-10-07T10:00:02.000Z"), + }, + want: []trace.SkillUse{}, + }, + { + name: "one use recorded twice counts once", + main: join( + modelStart(t, "1", steSkill, steDir, tenOClockM), + []string{metaLine(t, "m1-again", "r1", "toolu_1", steDir, tenOClockM)}, + userStart(t, "2", graphSkill, graphDir, "2026-10-07T10:01:00.000Z"), + []string{metaLine(t, "m2-again", "c2", "", graphDir, "2026-10-07T10:01:00.000Z")}, + ), + want: []trace.SkillUse{ + {Name: steSkill, Dir: steDir, FirstUsedAt: tenOClock, ByModel: 1}, + {Name: graphSkill, Dir: graphDir, FirstUsedAt: tenOhOne, ByUser: 1}, + }, + }, + { + // A skill built into Claude Code, or a custom command, has no + // folder. Its use is returned without one, and is not recorded. + name: "a skill with no folder", + main: join( + userStart(t, "1", "simplify", "", tenOClockM), + modelStart(t, "2", "code-review", "", "2026-10-07T10:01:00.000Z"), + ), + want: []trace.SkillUse{ + {Name: "simplify", FirstUsedAt: tenOClock, ByUser: 1}, + {Name: "code-review", FirstUsedAt: tenOhOne, ByModel: 1}, + }, + }, + { + name: "a plugin skill keeps its prefixed name", + main: modelStart(t, "1", "br:plan", planDir, tenOClockM), + want: []trace.SkillUse{{Name: "br:plan", Dir: planDir, FirstUsedAt: tenOClock, ByModel: 1}}, + }, + { + name: "subagent uses merge into the session entry", + main: modelStart(t, "1", steSkill, steDir, "2026-10-07T10:05:00.000Z"), + subagents: map[string][]string{ + "agent-a1.jsonl": modelStart(t, "s1", steSkill, steDir, "2026-10-07T10:01:00.000Z"), + "agent-a2.jsonl": modelStart(t, "s2", graphSkill, graphDir, "2026-10-07T10:09:00.000Z"), + }, + want: []trace.SkillUse{ + {Name: steSkill, Dir: steDir, FirstUsedAt: tenOhOne, ByModel: 2, InSubagents: 1}, + {Name: graphSkill, Dir: graphDir, FirstUsedAt: "2026-10-07T10:09:00Z", ByModel: 1, InSubagents: 1}, + }, + }, + { + // A format that this version does not know gives no entry, never + // a failed push. + name: "an unknown format gives no entry", + main: []string{mustLine(t, fixtureRecord{ + Type: recordTypeUser, UUID: "x", Message: fixtureMessage{Role: recordTypeUser, Content: "Launching skill: " + steSkill}, + })}, + want: []trace.SkillUse{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + rawDir := t.TempDir() + writeLines(t, state.RawSessionPath(rawDir, skillsSessionID), tc.main...) + for name, lines := range tc.subagents { + writeLines(t, filepath.Join(state.RawSubagentDir(rawDir, skillsSessionID), name), lines...) + } + + opts := &trace.ParseOpts{SessionDir: rawDir, SessionID: skillsSessionID} + got, warnings, err := New().SessionSkills(opts) + require.NoError(t, err) + assert.Empty(t, warnings) + assert.Equal(t, tc.want, got) + + again, _, err := New().SessionSkills(opts) + require.NoError(t, err) + assert.Equal(t, got, again, "the same transcript gives the same output") + }) + } +} + +func TestNewSkillLoads(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + + projectDir := filepath.Join(home, ".claude", "projects", "-repo") + transcript := filepath.Join(projectDir, skillsSessionID+".jsonl") + subagent := filepath.Join(projectDir, skillsSessionID, "subagents", "agent-a1.jsonl") + + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + input := &trace.HookInput{SessionID: skillsSessionID, TranscriptPath: transcript} + + writeLines(t, transcript, + commandLine(t, "c1", graphSkill, tenOClockM), + metaLine(t, "m1", "c1", "", "/skills/graphify", tenOClockM), + ) + + assert.Equal(t, []string{"/skills/graphify"}, New().NewSkillLoads(store, input)) + assert.Empty(t, New().NewSkillLoads(store, input), "a line is read once") + + // The agent appends a use, and writes half of the next line. + f, err := os.OpenFile(transcript, os.O_APPEND|os.O_WRONLY, 0o600) + require.NoError(t, err) + _, err = f.WriteString(metaLine(t, "m2", "r2", "toolu_2", "/skills/asd-ste100", "2026-10-07T10:01:00.000Z") + "\n") + require.NoError(t, err) + half := metaLine(t, "m3", "r3", "toolu_3", "/skills/later", "2026-10-07T10:02:00.000Z") + _, err = f.WriteString(half[:len(half)/2]) + require.NoError(t, err) + require.NoError(t, f.Close()) + + writeLines(t, subagent, metaLine(t, "s1", "rs1", "toolu_s1", "/skills/in-subagent", "2026-10-07T10:01:30.000Z")) + + assert.ElementsMatch(t, []string{"/skills/asd-ste100", "/skills/in-subagent"}, New().NewSkillLoads(store, input), + "a line that the agent is still writing waits for the next hook") + + f, err = os.OpenFile(transcript, os.O_APPEND|os.O_WRONLY, 0o600) + require.NoError(t, err) + _, err = f.WriteString(half[len(half)/2:] + "\n") + require.NoError(t, err) + require.NoError(t, f.Close()) + + assert.Equal(t, []string{"/skills/later"}, New().NewSkillLoads(store, input)) +} diff --git a/app/cli/internal/trace/opencode/hooks.go b/app/cli/internal/trace/opencode/hooks.go index 2ac5f7247..baedebdc6 100644 --- a/app/cli/internal/trace/opencode/hooks.go +++ b/app/cli/internal/trace/opencode/hooks.go @@ -25,9 +25,11 @@ import ( "path/filepath" "regexp" "slices" + "strconv" "strings" "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" ) const ( @@ -63,6 +65,22 @@ const pluginTemplate = `import { spawn } from "node:child_process" const fileWritingTools = {{FileWritingToolsArray}} const commandTools = {{CommandToolsArray}} +const skillTool = {{SkillTool}} + +// skillDirFrom returns the folder of the skill that a skill tool call loaded: +// the dir field of the result metadata, or else the base directory line of +// the result text. Each OpenCode major gives the result in its own shape, so +// each candidate is tried in turn. +function skillDirFrom(...results: any[]): string { + for (const r of results) { + const dir = r?.metadata?.dir + if (typeof dir === "string" && dir) return dir + const text = typeof r === "string" ? r : typeof r?.output === "string" ? r.output : "" + const m = /^Base directory for this skill:\s*(.+)$/m.exec(text) + if (m) return m[1].trim() + } + return "" +} function filePathsFromArgs(args: any): string[] { if (args?.filePath) return [args.filePath] @@ -175,8 +193,14 @@ async function sessionEvent(directory: string, type: string, sessionID: string, {{SessionEndBlock}} } -async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any) { +async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any, skillDir = "") { const payload = { session_id: sessionID, hook_event_name: hookEventName, tool_name: tool } + if (tool === skillTool) { + // The skill is loaded after the call, and the result names its folder. + // The hook copies the folder for the evidence. + if (skillDir) await fire(directory, hook, { ...payload, skill_dir: skillDir }) + return + } if (commandTools.includes(tool)) { // The call ID pairs this hook with the other hook of the same call, so // overlapping commands keep their own snapshots. @@ -230,8 +254,8 @@ async function server({ directory, client }: any) { "tool.execute.before": async (input: any, output: any) => { await toolEvent(directory, "pre-tool-use", "tool.execute.before", input.sessionID, input.tool, input.callID, output.args) }, - "tool.execute.after": async (input: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args) + "tool.execute.after": async (input: any, output: any) => { + await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args, input.tool === skillTool ? skillDirFrom(output) : "") }, } } @@ -245,7 +269,7 @@ async function setup(ctx: any) { await toolEvent(directory, "pre-tool-use", "tool.execute.before", event.sessionID, event.tool, event.id, event.input) }) await ctx.tool.hook("execute.after", async (event: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input) + await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input, event.tool === skillTool ? skillDirFrom(event.output, event.result, event) : "") }) // resume: false stores the message without asking the model for an answer. @@ -342,6 +366,7 @@ func (p *Provider) writePluginFile(repoRoot string, includeSessionEnd bool) erro content := pluginTemplate content = strings.Replace(content, "{{FileWritingToolsArray}}", toolsArrayLiteral(fileWritingTools), 1) content = strings.Replace(content, "{{CommandToolsArray}}", toolsArrayLiteral(commandTools), 1) + content = strings.Replace(content, "{{SkillTool}}", strconv.Quote(skillTool), 1) if includeSessionEnd { content = strings.Replace(content, "{{SessionEndBlock}}", sessionDeletedHandler, 1) } else { @@ -400,6 +425,7 @@ func (p *Provider) ReadHookInput(r io.Reader) (*trace.HookInput, error) { ToolName string `json:"tool_name"` FilePath string `json:"file_path"` ToolUseID string `json:"tool_use_id"` + SkillDir string `json:"skill_dir"` } if err := json.Unmarshal(data, &raw); err != nil { return nil, err @@ -417,12 +443,18 @@ func (p *Provider) ReadHookInput(r io.Reader) (*trace.HookInput, error) { filePath = abs } + // The plugin takes the skill folder from the result text when the + // metadata has none, and there it is a file URL. A folder that is not + // absolute is dropped. + skillDir, _ := skill.ParseDir(raw.SkillDir) + return &trace.HookInput{ SessionID: raw.SessionID, HookEventName: raw.HookEventName, ToolName: raw.ToolName, FilePath: filePath, ToolUseID: raw.ToolUseID, + SkillDir: skillDir, }, nil } diff --git a/app/cli/internal/trace/opencode/hooks_test.go b/app/cli/internal/trace/opencode/hooks_test.go index e109ebc5d..7def22013 100644 --- a/app/cli/internal/trace/opencode/hooks_test.go +++ b/app/cli/internal/trace/opencode/hooks_test.go @@ -228,10 +228,56 @@ func TestPluginSendsShellCallID(t *testing.T) { assert.Contains(t, content, "{ ...payload, tool_use_id: callID }") // OpenCode 1.x. assert.Contains(t, content, `toolEvent(directory, "pre-tool-use", "tool.execute.before", input.sessionID, input.tool, input.callID, output.args)`) - assert.Contains(t, content, `toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args)`) + assert.Contains(t, content, `toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args, input.tool === skillTool ? skillDirFrom(output) : "")`) // OpenCode 2. assert.Contains(t, content, `toolEvent(directory, "pre-tool-use", "tool.execute.before", event.sessionID, event.tool, event.id, event.input)`) - assert.Contains(t, content, `toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input)`) + assert.Contains(t, content, `toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input, event.tool === skillTool ? skillDirFrom(event.output, event.result, event) : "")`) +} + +// TestPluginSendsSkillDir pins that the plugin sends the folder of a loaded +// skill to the post hook of the skill tool, and fires no hook for the skill +// tool otherwise. +func TestPluginSendsSkillDir(t *testing.T) { + repoRoot := t.TempDir() + require.NoError(t, New().InstallHooks(repoRoot)) + + data, err := os.ReadFile(filepath.Join(repoRoot, settingsFile)) + require.NoError(t, err) + content := string(data) + + assert.Contains(t, content, `const skillTool = "skill"`) + assert.Contains(t, content, "if (skillDir) await fire(directory, hook, { ...payload, skill_dir: skillDir })") +} + +func TestReadHookInputParsesSkillDir(t *testing.T) { + testCases := []struct { + name string + payload string + want string + }{ + { + name: "a path from the tool metadata", + payload: `{"session_id":"ses_1","hook_event_name":"tool.execute.after","tool_name":"skill","skill_dir":"/home/me/.config/opencode/skills/git-release"}`, + want: "/home/me/.config/opencode/skills/git-release", + }, + { + name: "a file URL from the tool output", + payload: `{"session_id":"ses_1","hook_event_name":"tool.execute.after","tool_name":"skill","skill_dir":"file:///home/me/repo/.opencode/skills/git-release"}`, + want: "/home/me/repo/.opencode/skills/git-release", + }, + { + name: "a relative folder is dropped", + payload: `{"session_id":"ses_1","hook_event_name":"tool.execute.after","tool_name":"skill","skill_dir":"skills/git-release"}`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + input, err := New().ReadHookInput(bytes.NewBufferString(tc.payload)) + require.NoError(t, err) + assert.Equal(t, tc.want, input.SkillDir) + }) + } } // The plugin sends opencode's callID as tool_use_id on shell hooks, so that diff --git a/app/cli/internal/trace/opencode/parse.go b/app/cli/internal/trace/opencode/parse.go index 7a2c582da..fe6cce86a 100644 --- a/app/cli/internal/trace/opencode/parse.go +++ b/app/cli/internal/trace/opencode/parse.go @@ -36,10 +36,24 @@ const ( partTool = "tool" ) +// toolStatusCompleted is the status of a tool call that ran to the end. +const toolStatusCompleted = "completed" + // parseExport reads and parses the opencode export JSON at path into // structured evidence. It reads both the OpenCode 1.x and the OpenCode 2 // export format. Unknown fields are ignored. func parseExport(path string) (*aicodingsession.Evidence, error) { + export, err := readExport(path) + if err != nil { + return nil, err + } + + return buildEvidence(export), nil +} + +// readExport reads an export file in either format, in the OpenCode 1.x +// shape. +func readExport(path string) (*exportData, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("read export file: %w", err) @@ -51,7 +65,7 @@ func parseExport(path string) (*aicodingsession.Evidence, error) { return nil, fmt.Errorf("parse export JSON: %w", err) } - return buildEvidence(export.toExportData()), nil + return export.toExportData(), nil } var export exportData @@ -59,7 +73,7 @@ func parseExport(path string) (*aicodingsession.Evidence, error) { return nil, fmt.Errorf("parse export JSON: %w", err) } - return buildEvidence(&export), nil + return &export, nil } // isExportV2 reports whether data is an OpenCode 2 export. Only the OpenCode 2 @@ -166,7 +180,7 @@ func buildEvidence(export *exportData) *aicodingsession.Evidence { // Count completed tool invocations from tool parts. for _, part := range msg.Parts { - if part.Type == partTool && part.State != nil && part.State.Status == "completed" { + if part.Type == partTool && part.State != nil && part.State.Status == toolStatusCompleted { name := part.Tool if name == "" { name = unknownValue @@ -309,7 +323,7 @@ func buildRawSessionEntries(messages []messageEntry) []json.RawMessage { blocks = append(blocks, trace.RawSessionTextBlock{Type: partText, Text: p.Text}) } case partTool: - if p.State == nil || p.State.Status != "completed" { + if p.State == nil || p.State.Status != toolStatusCompleted { continue } toolName := p.Tool diff --git a/app/cli/internal/trace/opencode/skills.go b/app/cli/internal/trace/opencode/skills.go new file mode 100644 index 000000000..9d3c4d670 --- /dev/null +++ b/app/cli/internal/trace/opencode/skills.go @@ -0,0 +1,143 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package opencode + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" +) + +// Compile-time check that Provider records skills. +var _ trace.SkillTracker = (*Provider)(nil) + +// skillTool is the native OpenCode tool that loads a skill. Its one input is +// the name of the skill. +const skillTool = "skill" + +// NewSkillLoads returns the skill folder that the plugin put in the payload +// of the hook after a skill tool call. OpenCode tells the folder in the tool +// result, so no transcript is read. +func (p *Provider) NewSkillLoads(_ *state.Store, input *trace.HookInput) []string { + if input.ToolName != skillTool || input.SkillDir == "" { + return nil + } + + return []string{input.SkillDir} +} + +// SessionSkills returns the skills that the session used, from the copied +// export. A model start is a completed call to the skill tool. OpenCode has +// no slash command that starts a skill, so the user count stays at zero. A +// subagent runs in a child session, which has its own export, so the count of +// subagent uses stays at zero too. +func (p *Provider) SessionSkills(opts *trace.ParseOpts) ([]trace.SkillUse, []string, error) { + path, err := state.FindRawSessionFile(opts.SessionDir, opts.SessionID) + if err != nil { + return nil, nil, fmt.Errorf("locate opencode export: %w", err) + } + + export, err := readExport(path) + if err != nil { + return nil, nil, err + } + + return exportSkills(export), nil, nil +} + +// exportSkills counts the completed skill tool calls of an export. +func exportSkills(export *exportData) []trace.SkillUse { + uses := make(map[string]*trace.SkillUse) + for _, msg := range export.Messages { + at := msgTimeRFC3339(msg.Info.Time) + + for _, part := range msg.Parts { + if part.Type != partTool || part.Tool != skillTool || part.State == nil || part.State.Status != toolStatusCompleted { + continue + } + + var input struct { + Name string `json:"name"` + } + if json.Unmarshal(part.State.Input, &input) != nil || strings.TrimSpace(input.Name) == "" { + continue + } + name := strings.TrimSpace(input.Name) + + use, ok := uses[name] + if !ok { + use = &trace.SkillUse{Name: name} + uses[name] = use + } + if use.Dir == "" { + use.Dir = toolSkillDir(part.State) + } + use.Record(at, true, false) + } + } + + return trace.SortedSkillUses(uses) +} + +// toolSkillDir returns the skill folder of a skill tool call: from the tool +// metadata, or else from the base directory line of the tool output. +func toolSkillDir(st *toolState) string { + if st.Metadata != nil && filepath.IsAbs(st.Metadata.Dir) { + return filepath.Clean(st.Metadata.Dir) + } + + var output string + if json.Unmarshal(st.Output, &output) != nil { + return "" + } + + dir, _ := skill.BaseDir(output) + + return dir +} + +// SkillRoots returns the OpenCode folders that tell where a skill came from. +// OpenCode reads project skills from .opencode/skills, .claude/skills and +// .agents/skills in the repository, and user skills from the same folders in +// the user's configuration. +func (p *Provider) SkillRoots(repoRoot string) skill.Roots { + roots := skill.Roots{Project: []string{repoRoot}} + + home, err := os.UserHomeDir() + if err != nil { + return roots + } + + configHome := os.Getenv("XDG_CONFIG_HOME") + if configHome == "" { + configHome = filepath.Join(home, ".config") + } + + roots.User = []string{ + filepath.Join(configHome, "opencode", "skills"), + filepath.Join(home, ".opencode", "skills"), + filepath.Join(home, ".claude", "skills"), + filepath.Join(home, ".agents", "skills"), + } + + return roots +} diff --git a/app/cli/internal/trace/opencode/skills_test.go b/app/cli/internal/trace/opencode/skills_test.go new file mode 100644 index 000000000..430c1c73c --- /dev/null +++ b/app/cli/internal/trace/opencode/skills_test.go @@ -0,0 +1,114 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package opencode + +import ( + "os" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSessionSkills(t *testing.T) { + const sessionID = "ses_skills" + + // skillPart is a skill tool call as OpenCode 1.x exports it. The tool + // output holds the folder as a file URL, and the metadata as a path. + skillPart := func(name, status, metadata, output string) string { + return `{"type":"tool","tool":"skill","state":{"status":"` + status + `","input":{"name":"` + name + `"}` + + metadata + `,"output":` + output + `}}` + } + message := func(created string, parts ...string) string { + joined := "" + for i, p := range parts { + if i > 0 { + joined += "," + } + joined += p + } + return `{"info":{"role":"assistant","id":"msg","time":{"created":` + created + `}},"parts":[` + joined + `]}` + } + export := func(messages ...string) string { + joined := "" + for i, m := range messages { + if i > 0 { + joined += "," + } + joined += m + } + return `{"info":{"id":"` + sessionID + `","directory":"/repo","time":{"created":1},"version":"1.18.0"},"messages":[` + joined + `]}` + } + + testCases := []struct { + name string + export string + want []trace.SkillUse + }{ + { + name: "no skill", + export: export(message("1791367200000", `{"type":"text","text":"hello"}`)), + want: []trace.SkillUse{}, + }, + { + name: "completed calls count, a failed call does not", + export: export( + message("1791367200000", skillPart("git-release", "completed", `,"metadata":{"dir":"/home/me/.config/opencode/skills/git-release"}`, `""`)), + message("1791367260000", skillPart("missing", "error", "", `"Skill not found"`)), + message("1791367320000", skillPart("git-release", "completed", "", `"Base directory for this skill: file:///home/me/.config/opencode/skills/git-release\n"`)), + message("1791367380000", skillPart("pdf", "completed", "", `"Base directory for this skill: file:///repo/.opencode/skills/pdf\n"`)), + ), + want: []trace.SkillUse{ + {Name: "git-release", Dir: "/home/me/.config/opencode/skills/git-release", FirstUsedAt: "2026-10-07T10:00:00Z", ByModel: 2}, + {Name: "pdf", Dir: "/repo/.opencode/skills/pdf", FirstUsedAt: "2026-10-07T10:03:00Z", ByModel: 1}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + rawDir := t.TempDir() + require.NoError(t, os.WriteFile(state.RawSessionPath(rawDir, sessionID), []byte(tc.export), 0o600)) + + got, warnings, err := New().SessionSkills(&trace.ParseOpts{SessionDir: rawDir, SessionID: sessionID}) + require.NoError(t, err) + assert.Empty(t, warnings) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestNewSkillLoads(t *testing.T) { + const pdfDir = "/skills/pdf" + + testCases := []struct { + name string + input trace.HookInput + want []string + }{ + {name: "a skill call with a folder", input: trace.HookInput{ToolName: "skill", SkillDir: pdfDir}, want: []string{pdfDir}}, + {name: "a skill call without a folder", input: trace.HookInput{ToolName: "skill"}}, + {name: "another tool", input: trace.HookInput{ToolName: "bash", SkillDir: pdfDir}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, New().NewSkillLoads(nil, &tc.input)) + }) + } +} diff --git a/app/cli/internal/trace/opencode/testdata/plugin_full.ts b/app/cli/internal/trace/opencode/testdata/plugin_full.ts index f23a9d010..2eb4e23d2 100644 --- a/app/cli/internal/trace/opencode/testdata/plugin_full.ts +++ b/app/cli/internal/trace/opencode/testdata/plugin_full.ts @@ -2,6 +2,22 @@ import { spawn } from "node:child_process" const fileWritingTools = ["edit","write","apply_patch","patch"] const commandTools = ["bash","shell"] +const skillTool = "skill" + +// skillDirFrom returns the folder of the skill that a skill tool call loaded: +// the dir field of the result metadata, or else the base directory line of +// the result text. Each OpenCode major gives the result in its own shape, so +// each candidate is tried in turn. +function skillDirFrom(...results: any[]): string { + for (const r of results) { + const dir = r?.metadata?.dir + if (typeof dir === "string" && dir) return dir + const text = typeof r === "string" ? r : typeof r?.output === "string" ? r.output : "" + const m = /^Base directory for this skill:\s*(.+)$/m.exec(text) + if (m) return m[1].trim() + } + return "" +} function filePathsFromArgs(args: any): string[] { if (args?.filePath) return [args.filePath] @@ -117,8 +133,14 @@ async function sessionEvent(directory: string, type: string, sessionID: string, } } -async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any) { +async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any, skillDir = "") { const payload = { session_id: sessionID, hook_event_name: hookEventName, tool_name: tool } + if (tool === skillTool) { + // The skill is loaded after the call, and the result names its folder. + // The hook copies the folder for the evidence. + if (skillDir) await fire(directory, hook, { ...payload, skill_dir: skillDir }) + return + } if (commandTools.includes(tool)) { // The call ID pairs this hook with the other hook of the same call, so // overlapping commands keep their own snapshots. @@ -172,8 +194,8 @@ async function server({ directory, client }: any) { "tool.execute.before": async (input: any, output: any) => { await toolEvent(directory, "pre-tool-use", "tool.execute.before", input.sessionID, input.tool, input.callID, output.args) }, - "tool.execute.after": async (input: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args) + "tool.execute.after": async (input: any, output: any) => { + await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args, input.tool === skillTool ? skillDirFrom(output) : "") }, } } @@ -187,7 +209,7 @@ async function setup(ctx: any) { await toolEvent(directory, "pre-tool-use", "tool.execute.before", event.sessionID, event.tool, event.id, event.input) }) await ctx.tool.hook("execute.after", async (event: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input) + await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input, event.tool === skillTool ? skillDirFrom(event.output, event.result, event) : "") }) // resume: false stores the message without asking the model for an answer. diff --git a/app/cli/internal/trace/opencode/testdata/plugin_tracerun.ts b/app/cli/internal/trace/opencode/testdata/plugin_tracerun.ts index 1a20fd518..43e717716 100644 --- a/app/cli/internal/trace/opencode/testdata/plugin_tracerun.ts +++ b/app/cli/internal/trace/opencode/testdata/plugin_tracerun.ts @@ -2,6 +2,22 @@ import { spawn } from "node:child_process" const fileWritingTools = ["edit","write","apply_patch","patch"] const commandTools = ["bash","shell"] +const skillTool = "skill" + +// skillDirFrom returns the folder of the skill that a skill tool call loaded: +// the dir field of the result metadata, or else the base directory line of +// the result text. Each OpenCode major gives the result in its own shape, so +// each candidate is tried in turn. +function skillDirFrom(...results: any[]): string { + for (const r of results) { + const dir = r?.metadata?.dir + if (typeof dir === "string" && dir) return dir + const text = typeof r === "string" ? r : typeof r?.output === "string" ? r.output : "" + const m = /^Base directory for this skill:\s*(.+)$/m.exec(text) + if (m) return m[1].trim() + } + return "" +} function filePathsFromArgs(args: any): string[] { if (args?.filePath) return [args.filePath] @@ -113,8 +129,14 @@ async function sessionEvent(directory: string, type: string, sessionID: string, } } -async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any) { +async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any, skillDir = "") { const payload = { session_id: sessionID, hook_event_name: hookEventName, tool_name: tool } + if (tool === skillTool) { + // The skill is loaded after the call, and the result names its folder. + // The hook copies the folder for the evidence. + if (skillDir) await fire(directory, hook, { ...payload, skill_dir: skillDir }) + return + } if (commandTools.includes(tool)) { // The call ID pairs this hook with the other hook of the same call, so // overlapping commands keep their own snapshots. @@ -168,8 +190,8 @@ async function server({ directory, client }: any) { "tool.execute.before": async (input: any, output: any) => { await toolEvent(directory, "pre-tool-use", "tool.execute.before", input.sessionID, input.tool, input.callID, output.args) }, - "tool.execute.after": async (input: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args) + "tool.execute.after": async (input: any, output: any) => { + await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args, input.tool === skillTool ? skillDirFrom(output) : "") }, } } @@ -183,7 +205,7 @@ async function setup(ctx: any) { await toolEvent(directory, "pre-tool-use", "tool.execute.before", event.sessionID, event.tool, event.id, event.input) }) await ctx.tool.hook("execute.after", async (event: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input) + await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input, event.tool === skillTool ? skillDirFrom(event.output, event.result, event) : "") }) // resume: false stores the message without asking the model for an answer. diff --git a/app/cli/internal/trace/opencode/types.go b/app/cli/internal/trace/opencode/types.go index b73daaa42..638e66850 100644 --- a/app/cli/internal/trace/opencode/types.go +++ b/app/cli/internal/trace/opencode/types.go @@ -115,6 +115,14 @@ type toolState struct { Status string `json:"status"` Input json.RawMessage `json:"input,omitempty"` Output json.RawMessage `json:"output,omitempty"` + // Metadata is read for the skill tool only, which puts the skill folder + // in it. + Metadata *toolMetadata `json:"metadata,omitempty"` +} + +// toolMetadata holds the fields of the tool metadata that the provider reads. +type toolMetadata struct { + Dir string `json:"dir,omitempty"` } // exportDataV2 is the top-level shape of `opencode session export`, the diff --git a/app/cli/internal/trace/provider.go b/app/cli/internal/trace/provider.go index 114cd9b56..d3c579fc7 100644 --- a/app/cli/internal/trace/provider.go +++ b/app/cli/internal/trace/provider.go @@ -19,7 +19,9 @@ import ( "context" "errors" "io" + "sort" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" ) @@ -158,6 +160,95 @@ type Provider interface { AnnounceToUser(msg string) error } +// SkillTracker is implemented by a provider that records the skills that a +// session used. A provider without it records no skills, and for its sessions +// no skill entry means "not recorded", not "no skill used". +type SkillTracker interface { + // NewSkillLoads returns the folders of the skills that the session loaded + // since the last call for the session, as far as the hook can tell. The + // caller copies each folder. It never fails: a hook that cannot tell + // returns nothing, and the push copies the folder later. + NewSkillLoads(store *state.Store, input *HookInput) []string + + // SessionSkills returns the skills that the session used, from the + // session data under opts.SessionDir, in the order of their first use. + // The warnings are for uses that could not be read. + SessionSkills(opts *ParseOpts) ([]SkillUse, []string, error) + + // SkillRoots returns the folders that tell where a skill came from, for a + // session in repoRoot. + SkillRoots(repoRoot string) skill.Roots +} + +// SkillUse is one skill that a session used, with the counts of its uses. +type SkillUse struct { + // Name is the name that the agent used for the skill. + Name string + // Dir is the folder that the agent loaded the skill from, or "" when the + // skill has no folder (a skill built into the agent) or the session data + // does not tell. A use with no folder is not recorded. + Dir string + // FirstUsedAt is the time of the first use, RFC3339. + FirstUsedAt string + // ByModel counts the uses that the model started with a tool call. + ByModel int + // ByUser counts the uses that the user started with a slash command. + ByUser int + // InSubagents counts the uses, of either start, that came from subagents. + InSubagents int +} + +// Record adds one use at time at, RFC3339, to the counts. The first use is +// the earliest time recorded, whatever the order of the calls. +func (u *SkillUse) Record(at string, byModel, subagent bool) { + u.noteTime(at) + if byModel { + u.ByModel++ + } else { + u.ByUser++ + } + if subagent { + u.InSubagents++ + } +} + +// Merge adds the uses of other, of the same skill, to u. +func (u *SkillUse) Merge(other SkillUse) { + u.noteTime(other.FirstUsedAt) + u.ByModel += other.ByModel + u.ByUser += other.ByUser + u.InSubagents += other.InSubagents + if u.Dir == "" { + u.Dir = other.Dir + } +} + +func (u *SkillUse) noteTime(at string) { + if at != "" && (u.FirstUsedAt == "" || at < u.FirstUsedAt) { + u.FirstUsedAt = at + } +} + +// SortedSkillUses returns the uses in the order of their first use. The name +// breaks ties, so that the same session data always gives the same order: +// the limit on skill entries keeps the first ones. +func SortedSkillUses(uses map[string]*SkillUse) []SkillUse { + out := make([]SkillUse, 0, len(uses)) + for _, use := range uses { + out = append(out, *use) + } + + sort.Slice(out, func(i, j int) bool { + if out[i].FirstUsedAt != out[j].FirstUsedAt { + return out[i].FirstUsedAt < out[j].FirstUsedAt + } + + return out[i].Name < out[j].Name + }) + + return out +} + // HookInput represents parsed hook invocation data from an AI agent. type HookInput struct { // SessionID is the agent-assigned identifier for the session. @@ -199,6 +290,9 @@ type HookInput struct { // only emit post-edit events (e.g., Cursor's afterFileEdit) populate it so // consumers can reconstruct the "before" content via reverse application. Edits []HookEdit `json:"-"` + // SkillDir is the folder of the skill that the tool call loaded, when the + // hook payload gives it. Empty otherwise. + SkillDir string `json:"-"` // ToolFailed reports that the hook fires after a tool call that failed // (Claude's PostToolUseFailure). The tool can still have changed files, // so its changes are recorded. But the agent expects a different hook diff --git a/app/cli/internal/trace/skill/skill.go b/app/cli/internal/trace/skill/skill.go new file mode 100644 index 000000000..b63957fcb --- /dev/null +++ b/app/cli/internal/trace/skill/skill.go @@ -0,0 +1,552 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package skill keeps a copy of each skill folder that a coding session used, +// and turns the copy into the materials that the evidence holds. +// +// The copy is made when the session uses the skill, not when the session is +// pushed, so the evidence holds the skill that ran also when its files change +// before the push. A copy is never replaced: a later use that finds different +// content only marks the copy as changed. +// +// A copy holds the files of the folder as they are. Secrets are removed only +// when a package is made from the copy, by the same redaction as the other +// spec sources. +package skill + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" + "unicode/utf8" +) + +const ( + // DefinitionFile is the file that defines a skill. A folder without it is + // not a skill, and is never read. + DefinitionFile = "SKILL.md" + + // MaxPackageSize is the largest archive that is uploaded. A larger skill + // keeps its definition and its counts, and loses its package only. + MaxPackageSize = 5 << 20 + + // maxCopySize bounds the bytes that a hook copies. The archive is + // compressed, so a folder can be larger than MaxPackageSize and still + // give a package under the limit. A folder over this bound cannot. + maxCopySize = 50 << 20 + + // filesDir holds the copied files inside a copy. + filesDir = "files" + // infoFile records what the copy was made from. + infoFile = "info.json" + // changedFile marks a copy whose folder changed after the first use. + changedFile = "changed" + // packageDir holds the redacted definition and archive, made at the first + // push that needs them. + packageDir = "package" + packageArchive = "package.tar.gz" + // oversizeFile marks a copy whose archive is over MaxPackageSize. + oversizeFile = "oversize" + + // baseDirMarker opens the line that tells where an agent loaded a skill + // from. Claude Code writes it with a path, and OpenCode with a file URL. + baseDirMarker = "Base directory for this skill:" +) + +// vcsDirs are the version-control folders that a package leaves out. +var vcsDirs = []string{".git", ".hg", ".svn", ".bzr", ".jj"} + +// ErrNotSkill is returned for a folder that holds no SKILL.md file. +var ErrNotSkill = errors.New("the folder holds no " + DefinitionFile) + +// Info describes a copy. +type Info struct { + // Digest identifies the content of the folder when it was copied. The + // same content gives the same digest in every session. + Digest string `json:"digest"` + // Truncated reports a folder over the copy bound. Only its definition was + // copied, and it gets no package. + Truncated bool `json:"truncated,omitempty"` + // SkippedLinks counts the links that point out of the folder, or to a + // folder, and so were left out. + SkippedLinks int `json:"skipped_links,omitempty"` + // Changed reports that a later use found different content in the folder. + // The copy keeps the content of the first use. + Changed bool `json:"-"` +} + +// file is one file of a skill folder that goes into a copy. +type file struct { + // rel is the path inside the folder, with forward slashes. + rel string + // path is where the content is read from. For a link inside the folder, + // it is the target of the link. + path string + // executable keeps the one mode bit a script needs. + executable bool + size int64 +} + +// Copy copies the skill folder src into dst, which must not exist yet, and +// returns what it copied. When another hook renamed its copy into dst first, +// Copy returns that copy unchanged. +func Copy(src, dst string) (Info, error) { + files, skipped, err := listFiles(src) + if err != nil { + return Info{}, err + } + + info := Info{SkippedLinks: skipped} + + var total int64 + for _, f := range files { + total += f.size + } + // A folder over the bound keeps only its definition. Its digest is still + // that of the whole folder, for the comparison at a later use. + info.Truncated = total > maxCopySize + + if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + return Info{}, fmt.Errorf("create skill copy directory: %w", err) + } + + // The copy is made in a sibling folder and renamed into place, so that a + // reader never sees half of it and two hooks cannot mix their files. + tmp, err := os.MkdirTemp(filepath.Dir(dst), "."+filepath.Base(dst)+"-*") + if err != nil { + return Info{}, fmt.Errorf("create skill copy directory: %w", err) + } + defer func() { _ = os.RemoveAll(tmp) }() + + h := sha256.New() + for _, f := range files { + content, err := os.ReadFile(f.path) + if err != nil { + return Info{}, fmt.Errorf("read skill file %q: %w", f.rel, err) + } + + writeDigestEntry(h, f.rel, f.executable, content) + if info.Truncated && f.rel != DefinitionFile { + continue + } + + mode := os.FileMode(0o600) + if f.executable { + mode = 0o700 + } + + // The path comes from a walk of the folder, so it cannot leave the + // copy. The check keeps that true if the walk ever changes. + target := filepath.Join(tmp, filesDir, filepath.FromSlash(f.rel)) + if !Within(target, filepath.Join(tmp, filesDir)) { + return Info{}, fmt.Errorf("skill file %q is outside the skill folder", f.rel) + } + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return Info{}, fmt.Errorf("copy skill file %q: %w", f.rel, err) + } + //nolint:gosec // target is checked above to stay inside the copy + if err := os.WriteFile(target, content, mode); err != nil { + return Info{}, fmt.Errorf("copy skill file %q: %w", f.rel, err) + } + } + + info.Digest = "sha256:" + hex.EncodeToString(h.Sum(nil)) + + data, err := json.Marshal(info) + if err != nil { + return Info{}, fmt.Errorf("encode skill copy info: %w", err) + } + if err := os.WriteFile(filepath.Join(tmp, infoFile), data, 0o600); err != nil { + return Info{}, fmt.Errorf("write skill copy info: %w", err) + } + + if err := os.Rename(tmp, dst); err != nil { + // Another hook renamed its copy into place first. Its copy is of the + // same use, so it stands. + if existing, loadErr := Load(dst); loadErr == nil { + return existing, nil + } + + return Info{}, fmt.Errorf("store skill copy: %w", err) + } + + return info, nil +} + +// Load returns the description of an existing copy. +func Load(dst string) (Info, error) { + data, err := os.ReadFile(filepath.Join(dst, infoFile)) + if err != nil { + return Info{}, err + } + + var info Info + if err := json.Unmarshal(data, &info); err != nil { + return Info{}, fmt.Errorf("parse skill copy info: %w", err) + } + + if _, err := os.Stat(filepath.Join(dst, changedFile)); err == nil { + info.Changed = true + } + + return info, nil +} + +// MarkChanged records that a later use found content in the folder that is +// not the content of the copy. +func MarkChanged(dst string) error { + return os.WriteFile(filepath.Join(dst, changedFile), nil, 0o600) +} + +// ContentDigest returns the digest of the content of a skill folder, as Copy +// computes it, without copying anything. +func ContentDigest(src string) (string, error) { + files, _, err := listFiles(src) + if err != nil { + return "", err + } + + return contentDigest(files) +} + +func contentDigest(files []file) (string, error) { + h := sha256.New() + for _, f := range files { + content, err := os.ReadFile(f.path) + if err != nil { + return "", fmt.Errorf("read skill file %q: %w", f.rel, err) + } + writeDigestEntry(h, f.rel, f.executable, content) + } + + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + +// writeDigestEntry adds one file to a content digest. Each field has its +// length first, so that no two lists of files give the same input. +func writeDigestEntry(w io.Writer, rel string, executable bool, content []byte) { + sum := sha256.Sum256(content) + _, _ = fmt.Fprintf(w, "%d:%s:%t:%x\n", len(rel), rel, executable, sum) +} + +// listFiles returns the files of a skill folder that go into a copy, sorted by +// path, and the number of links that it left out. +// +// The folder itself can be a link: an agent often installs a skill as a link +// to a checkout. The links inside it are followed only to a file inside the +// folder. A link out of the folder, or to a folder, is left out, so a skill +// cannot pull other files of the user into the evidence, and a loop of links +// cannot stop the walk. +func listFiles(src string) (files []file, skipped int, err error) { + root, err := filepath.EvalSymlinks(src) + if err != nil { + return nil, 0, fmt.Errorf("resolve skill folder: %w", err) + } + + if info, err := os.Stat(filepath.Join(root, DefinitionFile)); err != nil || !info.Mode().IsRegular() { + return nil, 0, ErrNotSkill + } + + err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + if path != root && slices.Contains(vcsDirs, d.Name()) { + return filepath.SkipDir + } + + return nil + } + + // A git worktree or submodule keeps a .git file, not a folder. + if slices.Contains(vcsDirs, d.Name()) { + return nil + } + + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + + source := path + if d.Type()&fs.ModeSymlink != 0 { + target, ok := linkInside(root, path) + if !ok { + skipped++ + return nil + } + source = target + } + + info, err := os.Stat(source) + if err != nil { + return err + } + if !info.Mode().IsRegular() { + if info.IsDir() { + skipped++ + } + + return nil + } + + files = append(files, file{ + rel: filepath.ToSlash(rel), + path: source, + executable: info.Mode().Perm()&0o111 != 0, + size: info.Size(), + }) + + return nil + }) + if err != nil { + return nil, 0, fmt.Errorf("read skill folder: %w", err) + } + + sort.Slice(files, func(i, j int) bool { return files[i].rel < files[j].rel }) + + return files, skipped, nil +} + +// linkInside resolves a link and reports whether its target is a file or a +// folder inside root. +func linkInside(root, link string) (string, bool) { + target, err := filepath.EvalSymlinks(link) + if err != nil { + return "", false + } + + return target, Within(target, root) +} + +// Within reports whether path is root or a path below it. +func Within(path, root string) bool { + rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) + if err != nil { + return false + } + + return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))) +} + +// Package is what the evidence holds for one skill. +type Package struct { + // Definition is the redacted SKILL.md file. + Definition []byte + // Archive is a reproducible gzip-compressed tar archive of the redacted + // folder, or nil when it is over MaxPackageSize or the copy was + // truncated. + Archive []byte + // Oversize reports that the archive was left out for its size. + Oversize bool +} + +// Redactor removes secrets from the text of one file. +type Redactor func(doc []byte) ([]byte, error) + +// MakePackage returns the package of a copy. The first call redacts each text +// file and keeps the result in the copy, so a later push of the session does +// not redact an unchanged skill again. Binary files are stored as they are. +// +// Redaction fails closed: a file that could not be scanned fails the package, +// and nothing of the skill is uploaded. +func MakePackage(dst string, redact Redactor) (Package, error) { + info, err := Load(dst) + if err != nil { + return Package{}, fmt.Errorf("read skill copy: %w", err) + } + + cache := filepath.Join(dst, packageDir) + if pkg, ok := loadPackage(cache); ok { + return pkg, nil + } + + files, _, err := listFiles(filepath.Join(dst, filesDir)) + if err != nil { + return Package{}, err + } + + var pkg Package + var buf bytes.Buffer + + gz, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) + if err != nil { + return Package{}, err + } + tw := tar.NewWriter(gz) + + for _, f := range files { + content, err := os.ReadFile(f.path) + if err != nil { + return Package{}, fmt.Errorf("read skill file %q: %w", f.rel, err) + } + + if utf8.Valid(content) { + if content, err = redact(content); err != nil { + return Package{}, fmt.Errorf("scanning %q for secrets: %w", f.rel, err) + } + } + + if f.rel == DefinitionFile { + pkg.Definition = content + } + + if err := writeTarEntry(tw, f, content); err != nil { + return Package{}, err + } + } + + if err := tw.Close(); err != nil { + return Package{}, fmt.Errorf("close skill archive: %w", err) + } + if err := gz.Close(); err != nil { + return Package{}, fmt.Errorf("close skill archive: %w", err) + } + + if info.Truncated || buf.Len() > MaxPackageSize { + pkg.Oversize = true + } else { + pkg.Archive = buf.Bytes() + } + + // A cache we cannot write costs the next push a redaction, never this + // push its package. + _ = storePackage(cache, pkg) + + return pkg, nil +} + +// writeTarEntry adds one file with fixed metadata, so the same content gives +// the same archive on every machine: no time, no owner, and a mode that keeps +// only the executable bit. +func writeTarEntry(tw *tar.Writer, f file, content []byte) error { + mode := int64(0o644) + if f.executable { + mode = 0o755 + } + + hdr := &tar.Header{ + Typeflag: tar.TypeReg, + Name: f.rel, + Mode: mode, + Size: int64(len(content)), + ModTime: time.Unix(0, 0).UTC(), + Format: tar.FormatPAX, + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("write skill archive: %w", err) + } + if _, err := tw.Write(content); err != nil { + return fmt.Errorf("write skill archive: %w", err) + } + + return nil +} + +func loadPackage(cache string) (Package, bool) { + definition, err := os.ReadFile(filepath.Join(cache, DefinitionFile)) + if err != nil { + return Package{}, false + } + + pkg := Package{Definition: definition} + if _, err := os.Stat(filepath.Join(cache, oversizeFile)); err == nil { + pkg.Oversize = true + return pkg, true + } + + archive, err := os.ReadFile(filepath.Join(cache, packageArchive)) + if err != nil { + return Package{}, false + } + pkg.Archive = archive + + return pkg, true +} + +// storePackage writes the package next to the copy. The definition goes last, +// because loadPackage takes its presence to mean that the rest is there. +func storePackage(cache string, pkg Package) error { + if err := os.MkdirAll(cache, 0o700); err != nil { + return err + } + + if pkg.Oversize { + if err := os.WriteFile(filepath.Join(cache, oversizeFile), nil, 0o600); err != nil { + return err + } + } else if err := os.WriteFile(filepath.Join(cache, packageArchive), pkg.Archive, 0o600); err != nil { + return err + } + + return os.WriteFile(filepath.Join(cache, DefinitionFile), pkg.Definition, 0o600) +} + +// BaseDir returns the skill folder that a "Base directory for this skill:" +// line in text names. Claude Code writes a path, and OpenCode a file URL. Only +// an absolute path is accepted. +func BaseDir(text string) (string, bool) { + for line := range strings.SplitSeq(text, "\n") { + rest, ok := strings.CutPrefix(strings.TrimSpace(line), baseDirMarker) + if !ok { + continue + } + + return ParseDir(rest) + } + + return "", false +} + +// ParseDir reads a skill folder as an agent gives it: a path or a file URL. +// Only an absolute path is accepted. +func ParseDir(s string) (string, bool) { + dir := strings.TrimSpace(s) + if strings.HasPrefix(dir, "file://") { + u, err := url.Parse(dir) + if err != nil { + return "", false + } + dir = u.Path + } + + if dir == "" || !filepath.IsAbs(dir) { + return "", false + } + + return filepath.Clean(dir), true +} + +// HasBaseDir reports whether text can hold a base directory line. It is a +// cheap test that a scan of a large transcript runs before it decodes a line. +func HasBaseDir(text []byte) bool { + return bytes.Contains(text, []byte(baseDirMarker)) +} diff --git a/app/cli/internal/trace/skill/skill_test.go b/app/cli/internal/trace/skill/skill_test.go new file mode 100644 index 000000000..31a1f25d4 --- /dev/null +++ b/app/cli/internal/trace/skill/skill_test.go @@ -0,0 +1,287 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skill + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/rand" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeSkill makes a skill folder from a map of relative paths to contents. +func writeSkill(t *testing.T, dir string, files map[string]string) string { + t.Helper() + + for rel, content := range files { + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + } + + return dir +} + +// skillDoc is the definition of most test skills, and insideDoc the content +// of a file inside a test skill. +const ( + skillDoc = "# Skill" + insideDoc = "inside" +) + +// noRedaction keeps every file as it is. +func noRedaction(doc []byte) ([]byte, error) { return doc, nil } + +// archiveFiles reads an archive back as a map of paths to contents. +func archiveFiles(t *testing.T, archive []byte) map[string]string { + t.Helper() + + gz, err := gzip.NewReader(bytes.NewReader(archive)) + require.NoError(t, err) + + out := make(map[string]string) + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + + assert.Zero(t, hdr.ModTime.Unix(), "an entry has no time") + assert.Zero(t, hdr.Uid) + assert.Empty(t, hdr.Uname) + + content, err := io.ReadAll(tr) + require.NoError(t, err) + out[hdr.Name] = string(content) + } + + return out +} + +func TestCopyAndPackage(t *testing.T) { + t.Run("a package holds the full folder, redacted", func(t *testing.T) { + src := writeSkill(t, t.TempDir(), map[string]string{ + DefinitionFile: "# Skill\nuse token SECRET\n", + "references/rules.md": "the rules", + "scripts/lint.py": "print('SECRET')", + ".git/HEAD": "ref: refs/heads/main", + "nested/.git": "gitdir: ../../.git/modules/nested", + "examples/before-after.txt": "before and after", + }) + + dst := filepath.Join(t.TempDir(), "copy") + info, err := Copy(src, dst) + require.NoError(t, err) + assert.False(t, info.Truncated) + assert.True(t, strings.HasPrefix(info.Digest, "sha256:")) + + redact := func(doc []byte) ([]byte, error) { + return bytes.ReplaceAll(doc, []byte("SECRET"), []byte("[REDACTED]")), nil + } + + pkg, err := MakePackage(dst, redact) + require.NoError(t, err) + + assert.Equal(t, "# Skill\nuse token [REDACTED]\n", string(pkg.Definition)) + assert.Equal(t, map[string]string{ + DefinitionFile: "# Skill\nuse token [REDACTED]\n", + "references/rules.md": "the rules", + "scripts/lint.py": "print('[REDACTED]')", + "examples/before-after.txt": "before and after", + }, archiveFiles(t, pkg.Archive), "version-control folders and files are left out") + }) + + t.Run("the same content gives the same digests in every session", func(t *testing.T) { + files := map[string]string{DefinitionFile: skillDoc, "a/b.md": "b", "c.md": "c"} + first := writeSkill(t, t.TempDir(), files) + second := writeSkill(t, t.TempDir(), files) + + dst1 := filepath.Join(t.TempDir(), "copy") + dst2 := filepath.Join(t.TempDir(), "copy") + + info1, err := Copy(first, dst1) + require.NoError(t, err) + info2, err := Copy(second, dst2) + require.NoError(t, err) + assert.Equal(t, info1.Digest, info2.Digest) + + pkg1, err := MakePackage(dst1, noRedaction) + require.NoError(t, err) + pkg2, err := MakePackage(dst2, noRedaction) + require.NoError(t, err) + assert.Equal(t, pkg1.Archive, pkg2.Archive, "the archive is reproducible") + }) + + t.Run("a copy does not change when the folder changes", func(t *testing.T) { + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: "first"}) + dst := filepath.Join(t.TempDir(), "copy") + + first, err := Copy(src, dst) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(filepath.Join(src, DefinitionFile), []byte("second"), 0o600)) + + again, err := Copy(src, dst) + require.NoError(t, err) + assert.Equal(t, first.Digest, again.Digest, "the first copy stands") + + live, err := ContentDigest(src) + require.NoError(t, err) + assert.NotEqual(t, first.Digest, live) + + require.NoError(t, MarkChanged(dst)) + loaded, err := Load(dst) + require.NoError(t, err) + assert.True(t, loaded.Changed) + + pkg, err := MakePackage(dst, noRedaction) + require.NoError(t, err) + assert.Equal(t, "first", string(pkg.Definition)) + }) + + t.Run("a folder without SKILL.md is not read", func(t *testing.T) { + src := writeSkill(t, t.TempDir(), map[string]string{"README.md": "not a skill"}) + + _, err := Copy(src, filepath.Join(t.TempDir(), "copy")) + assert.ErrorIs(t, err, ErrNotSkill) + }) + + t.Run("a link out of the folder is not followed", func(t *testing.T) { + outside := writeSkill(t, t.TempDir(), map[string]string{"secret.txt": "private"}) + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc, "docs/inside.md": insideDoc}) + require.NoError(t, os.Symlink(filepath.Join(outside, "secret.txt"), filepath.Join(src, "secret.txt"))) + require.NoError(t, os.Symlink(outside, filepath.Join(src, "outside-dir"))) + require.NoError(t, os.Symlink(filepath.Join(src, "docs", "inside.md"), filepath.Join(src, "alias.md"))) + + dst := filepath.Join(t.TempDir(), "copy") + info, err := Copy(src, dst) + require.NoError(t, err) + assert.Equal(t, 2, info.SkippedLinks) + + pkg, err := MakePackage(dst, noRedaction) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + DefinitionFile: skillDoc, + "docs/inside.md": insideDoc, + "alias.md": insideDoc, + }, archiveFiles(t, pkg.Archive)) + }) + + t.Run("a skill folder that is itself a link is read", func(t *testing.T) { + target := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc}) + link := filepath.Join(t.TempDir(), "skill") + require.NoError(t, os.Symlink(target, link)) + + dst := filepath.Join(t.TempDir(), "copy") + _, err := Copy(link, dst) + require.NoError(t, err) + + pkg, err := MakePackage(dst, noRedaction) + require.NoError(t, err) + assert.Equal(t, skillDoc, string(pkg.Definition)) + }) + + t.Run("a package over the limit keeps its definition only", func(t *testing.T) { + big := make([]byte, MaxPackageSize+1024) + _, err := rand.Read(big) + require.NoError(t, err) + + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc}) + require.NoError(t, os.WriteFile(filepath.Join(src, "model.bin"), big, 0o600)) + + dst := filepath.Join(t.TempDir(), "copy") + _, err = Copy(src, dst) + require.NoError(t, err) + + pkg, err := MakePackage(dst, noRedaction) + require.NoError(t, err) + assert.True(t, pkg.Oversize) + assert.Nil(t, pkg.Archive) + assert.Equal(t, skillDoc, string(pkg.Definition)) + + // The second push reads the stored result. + again, err := MakePackage(dst, func([]byte) ([]byte, error) { return nil, errors.New("not called") }) + require.NoError(t, err) + assert.Equal(t, pkg, again) + }) + + t.Run("a file that cannot be scanned fails the package", func(t *testing.T) { + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc}) + dst := filepath.Join(t.TempDir(), "copy") + _, err := Copy(src, dst) + require.NoError(t, err) + + _, err = MakePackage(dst, func([]byte) ([]byte, error) { return nil, errors.New("scanner down") }) + assert.Error(t, err) + }) + + t.Run("binary files are stored as they are", func(t *testing.T) { + binary := string([]byte{0xff, 0xfe, 0x00, 'S', 'E', 'C', 'R', 'E', 'T'}) + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc, "icon.bin": binary}) + dst := filepath.Join(t.TempDir(), "copy") + _, err := Copy(src, dst) + require.NoError(t, err) + + pkg, err := MakePackage(dst, func(doc []byte) ([]byte, error) { + return bytes.ReplaceAll(doc, []byte("SECRET"), []byte("x")), nil + }) + require.NoError(t, err) + assert.Equal(t, binary, archiveFiles(t, pkg.Archive)["icon.bin"]) + }) +} + +func TestBaseDir(t *testing.T) { + testCases := []struct { + name string + text string + want string + wantOK bool + }{ + { + name: "a Claude Code path", + text: "Base directory for this skill: /Users/me/.claude/skills/asd-ste100\n\n# Simplified Technical English", + want: "/Users/me/.claude/skills/asd-ste100", + wantOK: true, + }, + { + name: "an OpenCode file URL inside the tool output", + text: "\n# Skill: git-release\n\nBase directory for this skill: file:///home/me/.config/opencode/skills/git%20release\n", + want: "/home/me/.config/opencode/skills/git release", + wantOK: true, + }, + {name: "a relative path", text: "Base directory for this skill: skills/x", wantOK: false}, + {name: "no marker", text: "Launching skill: x", wantOK: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, ok := BaseDir(tc.text) + assert.Equal(t, tc.wantOK, ok) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/app/cli/internal/trace/skill/source.go b/app/cli/internal/trace/skill/source.go new file mode 100644 index 000000000..a5dd0b45e --- /dev/null +++ b/app/cli/internal/trace/skill/source.go @@ -0,0 +1,90 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skill + +import ( + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +// Roots are the folders that tell where a skill came from. Each agent has its +// own layout, so each provider gives its own. +type Roots struct { + // Plugin holds the folders where the agent installs plugins. + Plugin []string + // Organization holds the folders of the managed configuration, which an + // administrator installs for all users. + Organization []string + // User holds the skill folders of the user's agent configuration. + User []string + // Project holds the repository and the session folder. + Project []string +} + +// Source returns where a skill came from, as one of the +// aicodingsession.SkillSource* constants. A plugin skill has a name with a +// plugin prefix (plugin:skill) or a folder in a plugin folder. Otherwise the +// first root list that holds the folder gives the source, in the order of +// the Roots fields. A folder that no root holds, or no folder, gives unknown. +func Source(name, dir string, roots Roots) string { + if strings.Contains(name, ":") { + return aicodingsession.SkillSourcePlugin + } + + if dir == "" { + return aicodingsession.SkillSourceUnknown + } + + for _, r := range []struct { + roots []string + source string + }{ + {roots.Plugin, aicodingsession.SkillSourcePlugin}, + {roots.Organization, aicodingsession.SkillSourceOrganization}, + {roots.User, aicodingsession.SkillSourceUser}, + {roots.Project, aicodingsession.SkillSourceProject}, + } { + for _, root := range r.roots { + if root != "" && withinResolved(dir, root) { + return r.source + } + } + } + + return aicodingsession.SkillSourceUnknown +} + +// withinResolved compares the paths as given, and again with their links +// resolved: on macOS /tmp is a link to /private/tmp, and an agent can report +// either. +func withinResolved(path, root string) bool { + if Within(path, root) { + return true + } + + resolvedPath, err := filepath.EvalSymlinks(path) + if err != nil { + return false + } + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return false + } + + return Within(resolvedPath, resolvedRoot) +} diff --git a/app/cli/internal/trace/skill/source_test.go b/app/cli/internal/trace/skill/source_test.go new file mode 100644 index 000000000..c9e79812e --- /dev/null +++ b/app/cli/internal/trace/skill/source_test.go @@ -0,0 +1,54 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skill + +import ( + "testing" + + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/stretchr/testify/assert" +) + +func TestSource(t *testing.T) { + roots := Roots{ + Plugin: []string{"/home/me/.claude/plugins"}, + Organization: []string{"/etc/claude-code/.claude/skills"}, + User: []string{"/home/me/.claude/skills"}, + Project: []string{"/home/me/src/repo"}, + } + + testCases := []struct { + name string + skillName string + dir string + want string + }{ + {name: "a plugin prefix", skillName: "superpowers:brainstorming", dir: "/somewhere/else", want: aicodingsession.SkillSourcePlugin}, + {name: "a plugin folder", skillName: "brainstorming", dir: "/home/me/.claude/plugins/cache/x/1.0.0/skills/brainstorming", want: aicodingsession.SkillSourcePlugin}, + {name: "a managed folder", skillName: "policy", dir: "/etc/claude-code/.claude/skills/policy", want: aicodingsession.SkillSourceOrganization}, + {name: "a user folder", skillName: "asd-ste100", dir: "/home/me/.claude/skills/asd-ste100", want: aicodingsession.SkillSourceUser}, + {name: "a repository folder", skillName: "release", dir: "/home/me/src/repo/.claude/skills/release", want: aicodingsession.SkillSourceProject}, + {name: "a folder next to the repository", skillName: "release", dir: "/home/me/src/repo-other/.claude/skills/release", want: aicodingsession.SkillSourceUnknown}, + {name: "a bundled skill", skillName: "claude-api", dir: "/tmp/bundled-skills/2.1.280/claude-api", want: aicodingsession.SkillSourceUnknown}, + {name: "no folder", skillName: "graphify", want: aicodingsession.SkillSourceUnknown}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, Source(tc.skillName, tc.dir, roots)) + }) + } +} diff --git a/app/cli/internal/trace/state/skills.go b/app/cli/internal/trace/state/skills.go new file mode 100644 index 000000000..1cf380aa5 --- /dev/null +++ b/app/cli/internal/trace/state/skills.go @@ -0,0 +1,103 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +const ( + // traceDirSkills holds, for each session, a copy of each skill folder + // that the session used. A copy lives for the session, not for one push. + traceDirSkills = "skills" + // skillScanName records, inside a session's skills directory, how far + // the hooks have read each transcript of the session. + skillScanName = "scan.json" +) + +// SkillCopyDir returns the directory that holds the copy of the skill folder +// dir for a session. The name is a digest of the folder path, so the path +// itself is not in the name, and two folders never share a copy. +func (s *Store) SkillCopyDir(sessionID, dir string) string { + sum := sha256.Sum256([]byte(filepath.Clean(dir))) + + return filepath.Join(s.skillsDir(sessionID), hex.EncodeToString(sum[:16])) +} + +func (s *Store) skillsDir(sessionID string) string { + return filepath.Join(s.traceDirPath(), traceDirSkills, SanitizeID(sessionID)) +} + +// SkillScanOffsets returns how many bytes of each transcript of a session the +// hooks have read for skill uses, by a key that the provider chooses. A +// session that no hook has scanned has none. +func (s *Store) SkillScanOffsets(sessionID string) (map[string]int64, error) { + data, err := os.ReadFile(filepath.Join(s.skillsDir(sessionID), skillScanName)) + if errors.Is(err, os.ErrNotExist) { + return map[string]int64{}, nil + } + if err != nil { + return nil, fmt.Errorf("read skill scan offsets: %w", err) + } + + offsets := map[string]int64{} + if err := json.Unmarshal(data, &offsets); err != nil { + return nil, fmt.Errorf("parse skill scan offsets: %w", err) + } + + return offsets, nil +} + +// SetSkillScanOffsets replaces the offsets of a session. +func (s *Store) SetSkillScanOffsets(sessionID string, offsets map[string]int64) error { + if err := writeJSONAtomic(filepath.Join(s.skillsDir(sessionID), skillScanName), offsets); err != nil { + return fmt.Errorf("write skill scan offsets: %w", err) + } + + return nil +} + +// gcSkillCopies drops the skill copies of each session that has no record +// any more. Nothing else drops them: a push after the session end still needs +// them, and the session end never runs when the agent is killed. +func (s *Store) gcSkillCopies() error { + dir := filepath.Join(s.traceDirPath(), traceDirSkills) + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read %s dir: %w", traceDirSkills, err) + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + record := filepath.Join(s.traceDirPath(), traceDirSessions, entry.Name()+sessionRecordExt) + if _, err := os.Stat(record); errors.Is(err, os.ErrNotExist) { + _ = os.RemoveAll(filepath.Join(dir, entry.Name())) + } + } + + return nil +} diff --git a/app/cli/internal/trace/state/skills_test.go b/app/cli/internal/trace/state/skills_test.go new file mode 100644 index 000000000..bee4ecb95 --- /dev/null +++ b/app/cli/internal/trace/state/skills_test.go @@ -0,0 +1,83 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSkillCopyDir(t *testing.T) { + store := NewGitStore(filepath.Join(t.TempDir(), ".git")) + + a := store.SkillCopyDir("sess-a", "/home/me/.claude/skills/asd-ste100") + assert.Equal(t, a, store.SkillCopyDir("sess-a", "/home/me/.claude/skills/asd-ste100/"), "the path is cleaned first") + assert.NotEqual(t, a, store.SkillCopyDir("sess-a", "/home/me/.claude/skills/graphify")) + assert.NotEqual(t, a, store.SkillCopyDir("sess-b", "/home/me/.claude/skills/asd-ste100")) + assert.False(t, strings.Contains(a, "asd-ste100"), "the folder path is not in the state") +} + +func TestSkillScanOffsets(t *testing.T) { + store := NewGitStore(filepath.Join(t.TempDir(), ".git")) + + got, err := store.SkillScanOffsets("sess-a") + require.NoError(t, err) + assert.Empty(t, got, "a session that no hook scanned has no offsets") + + require.NoError(t, store.SetSkillScanOffsets("sess-a", map[string]int64{"main": 120, "agent-a1.jsonl": 40})) + + got, err = store.SkillScanOffsets("sess-a") + require.NoError(t, err) + assert.Equal(t, map[string]int64{"main": 120, "agent-a1.jsonl": 40}, got) + + other, err := store.SkillScanOffsets("sess-b") + require.NoError(t, err) + assert.Empty(t, other) +} + +// TestGCOrphansDropsSkillCopies covers the skill copies of a session. They +// stay after the session end, for a push that comes later, and go with the +// session records. +func TestGCOrphansDropsSkillCopies(t *testing.T) { + store := NewGitStore(filepath.Join(t.TempDir(), ".git")) + require.NoError(t, store.InitTraceDir()) + + const ( + live = "skills-live" + orphan = "skills-orphan" + committed = "2026-10-07T00:00:00Z" + ) + + require.NoError(t, store.SaveCommitRecord(&CommitRecord{SHA: "sha-skills-live", SessionIDs: []string{live}, Timestamp: committed})) + require.NoError(t, store.SaveCommitRecord(&CommitRecord{SHA: "sha-skills-orphan", SessionIDs: []string{orphan}, Timestamp: committed})) + for _, id := range []string{live, orphan} { + require.NoError(t, store.SaveSessionRecord(&SessionRecord{SessionID: id, Active: true, StartedAt: time.Now().UTC().Format(time.RFC3339)})) + copyDir := store.SkillCopyDir(id, "/skills/x") + require.NoError(t, os.MkdirAll(copyDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(copyDir, "info.json"), []byte("{}"), 0o600)) + } + + require.NoError(t, store.GCOrphans(map[string]bool{"sha-skills-live": true})) + + assert.FileExists(t, filepath.Join(store.SkillCopyDir(live, "/skills/x"), "info.json")) + assert.NoDirExists(t, store.SkillCopyDir(orphan, "/skills/x")) +} diff --git a/app/cli/internal/trace/state/state.go b/app/cli/internal/trace/state/state.go index 7689b888a..f73564214 100644 --- a/app/cli/internal/trace/state/state.go +++ b/app/cli/internal/trace/state/state.go @@ -136,35 +136,42 @@ func (s *Store) SetRecordedSpecFiles(sessionID string, names []string) error { return removeIfExists(path) } - data, err := json.Marshal(names) + if err := writeJSONAtomic(path, names); err != nil { + return fmt.Errorf("write recorded spec files: %w", err) + } + + return nil +} + +// writeJSONAtomic writes v as JSON to path, creating its directory. The data +// goes to a temporary file that is then renamed over path, so a reader never +// sees half of it. +func writeJSONAtomic(path string, v any) error { + data, err := json.Marshal(v) if err != nil { - return fmt.Errorf("encode recorded spec files: %w", err) + return err } - dir := s.SpecRedactionDir(sessionID) + dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o700); err != nil { - return fmt.Errorf("create spec redaction directory: %w", err) + return err } - tmp, err := os.CreateTemp(dir, "."+recordedSpecFilesName+"-*") + tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+"-*") if err != nil { - return fmt.Errorf("write recorded spec files: %w", err) + return err } defer func() { _ = os.Remove(tmp.Name()) }() if _, err := tmp.Write(data); err != nil { _ = tmp.Close() - return fmt.Errorf("write recorded spec files: %w", err) + return err } if err := tmp.Close(); err != nil { - return fmt.Errorf("write recorded spec files: %w", err) - } - - if err := os.Rename(tmp.Name(), path); err != nil { - return fmt.Errorf("write recorded spec files: %w", err) + return err } - return nil + return os.Rename(tmp.Name(), path) } func (s *Store) recordedSpecFilesPath(sessionID string) string { @@ -283,7 +290,8 @@ func (s *Store) GCOrphans(liveSHAs map[string]bool) error { _ = os.RemoveAll(filepath.Join(redactionsDir, entry.Name())) } } - return nil + + return s.gcSkillCopies() } // staleSessionIDs returns the sanitized IDs of session records older than diff --git a/app/cli/pkg/action/trace_agent_hook.go b/app/cli/pkg/action/trace_agent_hook.go index 188916518..ace543ffc 100644 --- a/app/cli/pkg/action/trace_agent_hook.go +++ b/app/cli/pkg/action/trace_agent_hook.go @@ -65,6 +65,10 @@ func HandleAgentSessionEnd(provider trace.Provider, log zerolog.Logger) error { log.Debug().Err(err).Msg("copy session data failed") } + // The last turn can have used a skill that no hook saw yet. The copies + // stay after the session end, for a push that comes later. + captureSkillLoads(provider, store, input, log) + setSessionActive(store, sessionID, false, log) // The spec stays on disk for the whole session, so that every push of it @@ -218,12 +222,16 @@ func HandleAgentPromptSubmit(provider trace.Provider, log zerolog.Logger) error log.Debug().Str("session_id", input.SessionID).Msg("prompt-submit hook invoked") - _, repoRoot, err := state.Locate() + store, repoRoot, err := state.Locate() if err != nil { log.Debug().Err(err).Msg("prompt-submit: no trace state located") return nil } + // A skill that the user started with a slash command fires no tool hook. + // This is the first hook after it when the skill calls no tool. + captureSkillLoads(provider, store, input, log) + reminder := sessionSpecReminder(repoRoot, input.SessionID, log) if err := provider.AnnouncePromptSubmit(reminder); err != nil { log.Debug().Err(err).Msg("prompt-submit: failed to send the spec reminder") @@ -270,6 +278,7 @@ func HandleAgentPreToolUse(provider trace.Provider, log zerolog.Logger) error { } ensureSessionTracked(provider, store, repoRoot, input, log) + captureSkillLoads(provider, store, input, log) switch { case provider.IsCommandTool(input.ToolName): @@ -573,6 +582,12 @@ func HandleAgentPostToolUse(provider trace.Provider, log zerolog.Logger) error { isCommand := provider.IsCommandTool(input.ToolName) isFileWriting := provider.IsFileWritingTool(input.ToolName) if !isCommand && !isFileWriting { + // The hook fires for the skill tool too, whose skill is loaded only + // after the call. The skill is the one thing to record for it. + if store, _, err := state.Locate(); err == nil { + captureSkillLoads(provider, store, input, log) + } + log.Debug().Str("tool", input.ToolName).Msg("post-tool-use: not a tracked tool, skipping") return nil } diff --git a/app/cli/pkg/action/trace_hook_handler.go b/app/cli/pkg/action/trace_hook_handler.go index fee48bf25..1a435449a 100644 --- a/app/cli/pkg/action/trace_hook_handler.go +++ b/app/cli/pkg/action/trace_hook_handler.go @@ -639,6 +639,139 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts log.Debug().Int("session_count", len(sessionCommits)).Msg("grouped commits by session") + sessions := buildSessionEvidence(ctx, store, repoRoot, gitClient, sessionCommits, sessionRecords, sessionMode, log) + + if len(sessions) == 0 { + log.Debug().Msg("no session evidence could be generated") + + return nil + } + + // Create attestation using local state file to avoid conflicts with other attestations + localStatePath := store.AttestationStatePath() + defer func() { _ = os.Remove(localStatePath) }() + + projectName, organization, workflowName := resolvePushIdentity(repoRoot, opts) + if projectName == "" { + if opts.IgnoreYAML { + return fmt.Errorf("no project name provided") + } + + return fmt.Errorf("no project name found (pass via options or .chainloop.yml)") + } + + log.Debug().Str("project", projectName).Msg("resolved project for trace push") + + execOpts := []ExecutorOption{WithLocalStatePath(localStatePath), WithLogger(log)} + if organization != "" { + log.Debug().Str("forced_org", organization).Msg("forcing organization for trace push") + execOpts = append(execOpts, WithForcedOrganization(organization)) + } + executor, err := NewAttestationExecutor(opts.ActionOpts, opts.CLIVersion, execOpts...) + if err != nil { + // Warn (not debug): a misconfiguration here means no attestation is + // sent at all, and the pre-push hook swallows the returned error + // unless require-trace is on. + log.Warn().Err(err).Msg("skipping trace attestation") + + return err + } + defer func() { _ = executor.Close() }() + + attestationID, err := executor.Init(ctx, workflowName, projectName, opts.ProjectVersion) + if err != nil { + return fmt.Errorf("attestation init: %w", err) + } + + log.Debug().Str("attestation_id", attestationID).Msg("attestation initialized") + + // Add evidence for each session: its spec materials first, so that the + // session material can record their digests, then the session itself. + attestedSessions, attestedSpecs := attachSessionEvidence(ctx, executor, store, sessions, log) + + if len(attestedSessions) == 0 { + log.Debug().Msg("no evidence successfully added, resetting attestation") + _ = executor.Reset(ctx, AttestationResetTriggerCancelled, "no CHAINLOOP_AI_CODING_SESSION evidence added") + + return nil + } + + // Push attestation + log.Debug().Msg("pushing attestation") + res, err := executor.Push(ctx) + if err != nil { + return fmt.Errorf("attestation push: %w", err) + } + + // Tell the user where each session landed. The organization comes from the + // control plane rather than opts.Organization, which is empty whenever the + // CLI's current org is used. + links := logAttestedSessions(log, res.UIDashboardURL, res.GetOrganization(), attestedSessions) + + // Hand the links to the agent hook that runs after this push. When the + // push was driven by a coding agent's shell tool, the log line above is + // captured into that tool's output rather than shown to the user, so the + // hook is what actually puts the link in front of them. A failure here + // costs a notification, never the attestation that already succeeded. + // + // The caller tells us whether to skip, rather than us inferring it from + // on-disk state: the trace-run sentinel outlives a killed run, and + // reading it here would silently suppress every later notification in + // that repository. + if opts.SkipAgentNotification { + log.Debug().Msg("caller already showed the links; not recording them for an agent hook") + } else if err := store.SavePendingLinks(links); err != nil { + log.Debug().Err(err).Msg("could not record session links for the agent hook") + } + + log.Debug().Msg("attestation pushed, wiping single-use trace state") + + // A later push of each session keeps the spec files recorded here, so + // that the limit never drops a spec that is already in the evidence. + for sessionID, fileNames := range attestedSpecs { + recordPushedSpecs(store, sessionID, fileNames, log) + } + + // Mark every AI commit included in this attestation as tracked so that a + // subsequent `git push` with no new commits short-circuits at the skip + // check above. Save errors are non-fatal: the attestation already went + // out, and at worst we re-attest the same commits on the next push. + for _, c := range aiCommits { + if c.Tracked { + continue + } + c.Tracked = true + if err := store.SaveCommitRecord(c); err != nil { + log.Debug().Err(err).Str("sha", c.SHA).Msg("could not mark commit record as tracked") + } + } + + _ = store.WipeTraceDir() + if liveSHAs, err := gitClient.LocalReachableSHAs(repoRoot); err == nil { + if err := store.GCOrphans(liveSHAs); err != nil { + log.Debug().Err(err).Msg("orphan GC failed; trace state is intact but not pruned") + } + } else { + log.Debug().Err(err).Msg("could not enumerate local branch SHAs; skipping orphan GC") + } + + return nil +} + +// sessionEvidence is the evidence of one session before it is added to the +// attestation. It is written out only once the attestation exists, because it +// records the digests of the spec and skill materials added to it first. +type sessionEvidence struct { + sessionID string + evidence *aicodingsession.Evidence + specs []spec.Capture + skills []sessionSkill +} + +// buildSessionEvidence parses each session of the push into its evidence, +// with the specs and the skills that go into the attestation next to it. A +// session that cannot be read is left out, with a warning in the log. +func buildSessionEvidence(ctx context.Context, store *state.Store, repoRoot string, gitClient tracegit.Client, sessionCommits map[string][]*state.CommitRecord, sessionRecords map[string]*state.SessionRecord, sessionMode string, log zerolog.Logger) []sessionEvidence { // Collect repo-level context once (same for all sessions) rawDir := store.RawSessionDir() @@ -649,14 +782,6 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts isGenerated := gitClient.GeneratedMatcher(repoRoot) - // Build evidence for each session. It is written out only once the - // attestation exists, because it records the digests of the spec materials - // added to it first. - type sessionEvidence struct { - sessionID string - evidence *aicodingsession.Evidence - specs []spec.Capture - } var sessions []sessionEvidence for sessionID, commits := range sessionCommits { @@ -708,6 +833,11 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts captures, specWarnings := readSessionSpecs(store, repoRoot, sessionID, log) result.Data.Warnings = append(result.Data.Warnings, specWarnings...) + // The skills that the session used, each with the copy of its folder + // that a hook made when the session used it. + skills, skillWarnings := readSessionSkills(provider, store, parseOpts, repoRoot, log) + result.Data.Warnings = append(result.Data.Warnings, skillWarnings...) + // Apply repo-wide context with per-session commit overrides if gitCtxErr == nil { sessionCtx := *gitCtx @@ -746,68 +876,37 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts sessionID: sessionID, evidence: result, specs: captures, + skills: skills, }) } - if len(sessions) == 0 { - log.Debug().Msg("no session evidence could be generated") - - return nil - } - - // Create attestation using local state file to avoid conflicts with other attestations - localStatePath := store.AttestationStatePath() - defer func() { _ = os.Remove(localStatePath) }() - - projectName, organization, workflowName := resolvePushIdentity(repoRoot, opts) - if projectName == "" { - if opts.IgnoreYAML { - return fmt.Errorf("no project name provided") - } - - return fmt.Errorf("no project name found (pass via options or .chainloop.yml)") - } - - log.Debug().Str("project", projectName).Msg("resolved project for trace push") - - execOpts := []ExecutorOption{WithLocalStatePath(localStatePath), WithLogger(log)} - if organization != "" { - log.Debug().Str("forced_org", organization).Msg("forcing organization for trace push") - execOpts = append(execOpts, WithForcedOrganization(organization)) - } - executor, err := NewAttestationExecutor(opts.ActionOpts, opts.CLIVersion, execOpts...) - if err != nil { - // Warn (not debug): a misconfiguration here means no attestation is - // sent at all, and the pre-push hook swallows the returned error - // unless require-trace is on. - log.Warn().Err(err).Msg("skipping trace attestation") - - return err - } - defer func() { _ = executor.Close() }() - - attestationID, err := executor.Init(ctx, workflowName, projectName, opts.ProjectVersion) - if err != nil { - return fmt.Errorf("attestation init: %w", err) - } - - log.Debug().Str("attestation_id", attestationID).Msg("attestation initialized") + return sessions +} - // Add evidence for each session: its spec materials first, so that the - // session material can record their digests, then the session itself. - attestedSessions := make([]string, 0, len(sessions)) - attestedSpecs := make(map[string][]string, len(sessions)) +// attachSessionEvidence adds the materials of each session to the attestation: +// its spec and skill materials first, so that the session material can record +// their digests, then the session itself. It returns the sessions that were +// added, and the spec files that each of them stored. +func attachSessionEvidence(ctx context.Context, adder specMaterialAdder, store *state.Store, sessions []sessionEvidence, log zerolog.Logger) (attestedSessions []string, attestedSpecs map[string][]string) { + attestedSessions = make([]string, 0, len(sessions)) + attestedSpecs = make(map[string][]string, len(sessions)) // One allocator for the whole attestation: names taken from the start of // a session ID can repeat across sessions, and a repeated name would // replace an earlier material. names := materials.NewNameAllocator(nil) for _, se := range sessions { - entries, warnings, stored := attachSpecs(ctx, executor, newSpecRedactor(store.SpecRedactionDir(se.sessionID)), names, se.sessionID, se.specs, log) - se.evidence.Data.Spec = entries + redactor := newSpecRedactor(store.SpecRedactionDir(se.sessionID)) + entries, warnings, stored := attachSpecs(ctx, adder, redactor, names, se.sessionID, se.specs, log) se.evidence.Data.Warnings = append(se.evidence.Data.Warnings, warnings...) + // The skill entries come after the sources that the agent wrote. + skillEntries, skillWarnings := attachSkills(ctx, adder, redactor, names, se.sessionID, se.skills, log) + entries = append(entries, skillEntries...) + se.evidence.Data.Spec = entries + se.evidence.Data.Warnings = append(se.evidence.Data.Warnings, skillWarnings...) + name := evidenceName(se.sessionID) - if err := addSessionEvidence(ctx, executor, name, se.evidence); err != nil { + if err := addSessionEvidence(ctx, adder, name, se.evidence); err != nil { // Warn, not debug: the session is left out of the attestation, // and this is the only place that says why. log.Warn().Err(err).Str("session", se.sessionID).Msg("could not add the evidence of an AI session; it is left out of the attestation") @@ -818,73 +917,7 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts log.Debug().Str("session", se.sessionID).Str("name", name).Int("spec_entries", len(entries)).Msg("evidence added") } - if len(attestedSessions) == 0 { - log.Debug().Msg("no evidence successfully added, resetting attestation") - _ = executor.Reset(ctx, AttestationResetTriggerCancelled, "no CHAINLOOP_AI_CODING_SESSION evidence added") - - return nil - } - - // Push attestation - log.Debug().Msg("pushing attestation") - res, err := executor.Push(ctx) - if err != nil { - return fmt.Errorf("attestation push: %w", err) - } - - // Tell the user where each session landed. The organization comes from the - // control plane rather than opts.Organization, which is empty whenever the - // CLI's current org is used. - links := logAttestedSessions(log, res.UIDashboardURL, res.GetOrganization(), attestedSessions) - - // Hand the links to the agent hook that runs after this push. When the - // push was driven by a coding agent's shell tool, the log line above is - // captured into that tool's output rather than shown to the user, so the - // hook is what actually puts the link in front of them. A failure here - // costs a notification, never the attestation that already succeeded. - // - // The caller tells us whether to skip, rather than us inferring it from - // on-disk state: the trace-run sentinel outlives a killed run, and - // reading it here would silently suppress every later notification in - // that repository. - if opts.SkipAgentNotification { - log.Debug().Msg("caller already showed the links; not recording them for an agent hook") - } else if err := store.SavePendingLinks(links); err != nil { - log.Debug().Err(err).Msg("could not record session links for the agent hook") - } - - log.Debug().Msg("attestation pushed, wiping single-use trace state") - - // A later push of each session keeps the spec files recorded here, so - // that the limit never drops a spec that is already in the evidence. - for sessionID, fileNames := range attestedSpecs { - recordPushedSpecs(store, sessionID, fileNames, log) - } - - // Mark every AI commit included in this attestation as tracked so that a - // subsequent `git push` with no new commits short-circuits at the skip - // check above. Save errors are non-fatal: the attestation already went - // out, and at worst we re-attest the same commits on the next push. - for _, c := range aiCommits { - if c.Tracked { - continue - } - c.Tracked = true - if err := store.SaveCommitRecord(c); err != nil { - log.Debug().Err(err).Str("sha", c.SHA).Msg("could not mark commit record as tracked") - } - } - - _ = store.WipeTraceDir() - if liveSHAs, err := gitClient.LocalReachableSHAs(repoRoot); err == nil { - if err := store.GCOrphans(liveSHAs); err != nil { - log.Debug().Err(err).Msg("orphan GC failed; trace state is intact but not pruned") - } - } else { - log.Debug().Err(err).Msg("could not enumerate local branch SHAs; skipping orphan GC") - } - - return nil + return attestedSessions, attestedSpecs } // logAttestedSessions reports one line per attested session. When the @@ -918,7 +951,7 @@ func sessionLinkMessage(url string) string { // addSessionEvidence writes one session's evidence to a temporary file and adds // it to the attestation. -func addSessionEvidence(ctx context.Context, executor *AttestationExecutor, name string, evidence *aicodingsession.Evidence) error { +func addSessionEvidence(ctx context.Context, adder specMaterialAdder, name string, evidence *aicodingsession.Evidence) error { tmpFile, err := os.CreateTemp("", "chainloop-trace-*.json") if err != nil { return fmt.Errorf("create temp file: %w", err) @@ -936,7 +969,7 @@ func addSessionEvidence(ctx context.Context, executor *AttestationExecutor, name return fmt.Errorf("write evidence: %w", err) } - _, err = executor.AddMaterial(ctx, name, tmpFile.Name(), "CHAINLOOP_AI_CODING_SESSION", nil) + _, err = adder.AddMaterial(ctx, name, tmpFile.Name(), "CHAINLOOP_AI_CODING_SESSION", nil) return err } diff --git a/app/cli/pkg/action/trace_skills.go b/app/cli/pkg/action/trace_skills.go new file mode 100644 index 000000000..828c2f4f3 --- /dev/null +++ b/app/cli/pkg/action/trace_skills.go @@ -0,0 +1,286 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package action + +import ( + "context" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/rs/zerolog" +) + +// maxSkillEntries is the limit of skill entries for each session. It is apart +// from the limit of the spec sources that the agent writes, so that many +// skills cannot push out the task and its spec. +const maxSkillEntries = 25 + +// specAnnotationSkillContent tells the two materials of a skill apart. +const specAnnotationSkillContent = "chainloop.spec.skill.content" + +// The values of specAnnotationSkillContent. +const ( + skillContentDefinition = "definition" + skillContentPackage = "package" +) + +// captureSkillLoads copies the folder of each skill that the session loaded +// since the last hook, into the trace state. A hook runs it, so that the +// evidence holds the skill as it was when it ran, also when its files change +// before the push. +// +// A copy is never replaced. A later use that finds different content only +// marks the copy, and the push records a warning. +// +// Best effort throughout: a skill that cannot be copied here is copied at the +// push, and nothing here may slow down or fail the agent's tool call. +func captureSkillLoads(provider trace.Provider, store *state.Store, input *trace.HookInput, log zerolog.Logger) { + tracker, ok := provider.(trace.SkillTracker) + if !ok { + return + } + + for _, dir := range tracker.NewSkillLoads(store, input) { + copySkill(store, input.SessionID, dir, log) + } +} + +// copySkill makes the copy of a skill folder for a session, or checks an +// existing copy against the folder. +func copySkill(store *state.Store, sessionID, dir string, log zerolog.Logger) { + dst := store.SkillCopyDir(sessionID, dir) + + info, err := skill.Load(dst) + if err != nil { + if _, err := skill.Copy(dir, dst); err != nil { + log.Debug().Err(err).Str("session_id", sessionID).Msg("could not copy a skill folder; the push copies it") + } + + return + } + + if info.Changed { + return + } + + live, err := skill.ContentDigest(dir) + if err != nil || live == info.Digest { + return + } + + if err := skill.MarkChanged(dst); err != nil { + log.Debug().Err(err).Str("session_id", sessionID).Msg("could not mark a changed skill folder") + } +} + +// sessionSkill is one used skill, ready to be stored: its counts, where it +// came from, and the copy of its folder. +type sessionSkill struct { + use trace.SkillUse + source string + copyDir string +} + +// readSessionSkills returns the skills that a session used, each with a copy +// of its folder, and a warning for each skill that is left out or not as it +// was when it ran. +// +// A provider without skill support gives no skills and no warning. A failure +// to read the skills is never a reason to lose the session. +func readSessionSkills(provider trace.Provider, store *state.Store, opts *trace.ParseOpts, repoRoot string, log zerolog.Logger) ([]sessionSkill, []string) { + tracker, ok := provider.(trace.SkillTracker) + if !ok { + return nil, nil + } + + uses, warnings, err := tracker.SessionSkills(opts) + if err != nil { + // The error stays in the local log: its text carries local paths. + log.Warn().Err(err).Str("session", opts.SessionID).Msg("could not read the skills that the session used; the session is attested without them") + return nil, []string{"the skills that the session used were not recorded: the session data could not be read"} + } + + // A skill with no folder, such as a skill built into the agent, has + // nothing to store. It is left out with a warning in the local log only. + uses = slices.DeleteFunc(uses, func(use trace.SkillUse) bool { + if use.Dir != "" { + return false + } + + log.Warn().Str("session", opts.SessionID).Str("skill", use.Name).Msg("a skill that the session used has no skill folder, so it is not recorded") + + return true + }) + + if len(uses) > maxSkillEntries { + // The skills that the session used first are kept. + warnings = append(warnings, fmt.Sprintf("%d skill entries beyond the first %d were dropped", len(uses)-maxSkillEntries, maxSkillEntries)) + uses = uses[:maxSkillEntries] + } + + roots := tracker.SkillRoots(repoRoot) + + skills := make([]sessionSkill, 0, len(uses)) + for _, use := range uses { + dst := store.SkillCopyDir(opts.SessionID, use.Dir) + info, err := skill.Load(dst) + switch { + case err != nil: + // No hook ran after the use, so the folder is copied now. It can + // differ from the folder that the session used. + if info, err = skill.Copy(use.Dir, dst); err != nil { + log.Warn().Err(err).Str("session", opts.SessionID).Str("skill", use.Name).Msg("could not copy a skill folder") + warnings = append(warnings, fmt.Sprintf("skill %q was not recorded: its folder could not be read", use.Name)) + continue + } + warnings = append(warnings, fmt.Sprintf("skill %q was copied at push time, not when the session used it", use.Name)) + case info.Changed: + warnings = append(warnings, fmt.Sprintf("skill %q changed after its first use; the evidence holds the skill as it was at the first use", use.Name)) + } + + if info.SkippedLinks > 0 { + warnings = append(warnings, fmt.Sprintf("skill %q: %d links out of the skill folder, or to a folder, were left out", use.Name, info.SkippedLinks)) + } + + skills = append(skills, sessionSkill{ + use: use, + source: skill.Source(use.Name, use.Dir, roots), + copyDir: dst, + }) + } + + return skills, warnings +} + +// attachSkills stores the materials of each used skill and returns the spec +// entries that the session material records for them. +// +// Each skill gives two EVIDENCE materials: its redacted SKILL.md file, which +// the digest of the entry points to so that a viewer can show it, and a +// reproducible archive of the full redacted folder, which the metadata points +// to as the download. A skill that cannot be stored is left out with a +// warning, as a spec source is. +func attachSkills(ctx context.Context, adder specMaterialAdder, redactor *specRedactor, names *materials.NameAllocator, sessionID string, skills []sessionSkill, log zerolog.Logger) (entries []aicodingsession.SpecEntry, warnings []string) { + if len(skills) == 0 { + return nil, nil + } + + tmpDir, err := os.MkdirTemp("", "chainloop-trace-skill-*") + if err != nil { + return nil, []string{fmt.Sprintf("%d skill entries were not recorded: %v", len(skills), err)} + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + for i, s := range skills { + entry, warning, err := storeSkill(ctx, adder, redactor, names, filepath.Join(tmpDir, strconv.Itoa(i)), sessionID, s) + if err != nil { + // The error stays in the local log. The warning goes into the + // uploaded evidence, and error text can carry local paths. + log.Warn().Err(err).Str("session", sessionID).Str("skill", s.use.Name).Msg("could not record a skill entry") + warnings = append(warnings, fmt.Sprintf("skill %q was not recorded", s.use.Name)) + + continue + } + if warning != "" { + warnings = append(warnings, warning) + } + + entries = append(entries, entry) + } + + return entries, warnings +} + +// storeSkill adds the materials of one skill and returns its spec entry. A +// package that cannot be stored costs the package only, with a warning: the +// definition and the counts still go into the evidence. +func storeSkill(ctx context.Context, adder specMaterialAdder, redactor *specRedactor, names *materials.NameAllocator, dir, sessionID string, s sessionSkill) (aicodingsession.SpecEntry, string, error) { + pkg, err := skill.MakePackage(s.copyDir, func(doc []byte) ([]byte, error) { + return redactor.Redact(ctx, doc) + }) + if err != nil { + return aicodingsession.SpecEntry{}, "", err + } + + if err := os.MkdirAll(dir, 0o700); err != nil { + return aicodingsession.SpecEntry{}, "", fmt.Errorf("staging the skill: %w", err) + } + + // spec--skill-. The + // allocator makes each name unique within the attestation. + stem := materialStem(s.use.Name, "skill") + base := specMaterialPrefix(sessionID) + "skill-" + stem + annotations := func(content string) map[string]string { + return map[string]string{ + specAnnotationSession: sessionID, + specAnnotationKind: aicodingsession.SpecKindSkill, + specAnnotationTitle: s.use.Name, + specAnnotationSkillContent: content, + } + } + + definitionPath := filepath.Join(dir, skill.DefinitionFile) + if err := os.WriteFile(definitionPath, pkg.Definition, 0o600); err != nil { + return aicodingsession.SpecEntry{}, "", fmt.Errorf("staging the skill: %w", err) + } + + digest, err := adder.AddMaterial(ctx, names.AllocateNamed(base), definitionPath, specMaterialKind, annotations(skillContentDefinition)) + if err != nil { + return aicodingsession.SpecEntry{}, "", err + } + + var packageDigest, warning string + if pkg.Oversize { + warning = fmt.Sprintf("the package of skill %q is larger than %d MB and was not uploaded", s.use.Name, skill.MaxPackageSize>>20) + } else { + // The archive is stored under the name of the skill. + archivePath := filepath.Join(dir, stem+".tar.gz") + if err = os.WriteFile(archivePath, pkg.Archive, 0o600); err == nil { + packageDigest, err = adder.AddMaterial(ctx, names.AllocateNamed(base+"-pkg"), archivePath, specMaterialKind, annotations(skillContentPackage)) + } + if err != nil { + warning = fmt.Sprintf("the package of skill %q was not uploaded", s.use.Name) + } + } + + capturedAt := s.use.FirstUsedAt + if capturedAt == "" { + capturedAt = state.NowTimestamp() + } + + return aicodingsession.SpecEntry{ + Kind: aicodingsession.SpecKindSkill, + Title: s.use.Name, + Digest: digest, + CapturedAt: capturedAt, + Metadata: &aicodingsession.SpecMetadata{ + Source: s.source, + PackageDigest: packageDigest, + InvocationCount: s.use.ByModel + s.use.ByUser, + ByModel: s.use.ByModel, + ByUser: s.use.ByUser, + InSubagents: s.use.InSubagents, + }, + }, warning, nil +} diff --git a/app/cli/pkg/action/trace_skills_scenario_test.go b/app/cli/pkg/action/trace_skills_scenario_test.go new file mode 100644 index 000000000..8ddc867c6 --- /dev/null +++ b/app/cli/pkg/action/trace_skills_scenario_test.go @@ -0,0 +1,227 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package action + +import ( + "context" + "encoding/json" + "maps" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + tracegit "github.com/chainloop-dev/chainloop/app/cli/internal/trace/git" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/internal/schemavalidators" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// steLintScript is the script of the STE test skill. +const steLintScript = "scripts/ste-lint.py" + +// transcript appends Claude Code transcript records to the session file, in +// the shape that Claude Code 2.1 writes them. +type transcript struct { + t *testing.T + path string +} + +func (tr transcript) append(records ...string) { + tr.t.Helper() + + f, err := os.OpenFile(tr.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + require.NoError(tr.t, err) + defer func() { _ = f.Close() }() + + for _, r := range records { + _, err := f.WriteString(r + "\n") + require.NoError(tr.t, err) + } +} + +// TestSkillScenario follows two skills from their use in a Claude Code session +// to the materials that the pre-push hook adds to the attestation. The model +// starts one skill with the Skill tool, and the user starts the other with a +// slash command. The hooks run as Claude Code runs them, with the payload on +// stdin, and the push runs every step up to the upload. +func TestSkillScenario(t *testing.T) { + const ( + sessionID = "0f3a6c1e-2b4d-4e8f-9a1b-2c3d4e5f6a7b" + // Assembled from fragments so that nothing credential-shaped is + // committed. + pat = "ghp_erOZlZv0B1e3amrQ" + "ugdwZ8Ro2W4kDql9WPTf" + ) + + home := t.TempDir() + t.Setenv("HOME", home) + + repoDir := initTempGitRepo(t) + require.NoError(t, state.NewGitStore(filepath.Join(repoDir, ".git")).InitTraceDir()) + t.Chdir(repoDir) + + // A user skill with a script and a version-control folder, and a project + // skill in the repository. + userSkill := makeSkillAt(t, filepath.Join(home, ".claude", "skills", "asd-ste100"), map[string]string{ + skill.DefinitionFile: "# STE\nAs it ran. Token " + pat + "\n", + steLintScript: "TOKEN = '" + pat + "'\n", + ".git/HEAD": "ref: refs/heads/main\n", + }) + projectSkill := makeSkillAt(t, filepath.Join(repoDir, ".claude", "skills", "release"), map[string]string{ + skill.DefinitionFile: "# Release\nCut a release.\n", + }) + + projectDir := claudeProjectDir(home, repoDir) + require.NoError(t, os.MkdirAll(projectDir, 0o700)) + tr := transcript{t: t, path: filepath.Join(projectDir, sessionID+".jsonl")} + + quote := func(s string) string { + b, err := json.Marshal(s) + require.NoError(t, err) + return string(b) + } + + // hookPayload is what Claude Code writes to the stdin of a hook. + hookPayload := func(event, tool string) string { + b, err := json.Marshal(struct { + SessionID string `json:"session_id"` + HookEventName string `json:"hook_event_name"` + TranscriptPath string `json:"transcript_path"` + Cwd string `json:"cwd"` + ToolName string `json:"tool_name,omitempty"` + }{sessionID, event, tr.path, repoDir, tool}) + require.NoError(t, err) + return string(b) + } + + provider := claude.New() + + // The session starts. + tr.append(`{"type":"user","uuid":"u1","timestamp":"2026-10-07T10:00:00.000Z","sessionId":"` + sessionID + `","message":{"role":"user","content":"Rewrite the PR description in STE."}}`) + withStdin(t, hookPayload("SessionStart", "")) + captureStdout(t, func() { require.NoError(t, HandleAgentSessionStart(provider, zerolog.Nop())) }) + + // The model calls the Skill tool, Claude Code loads the skill, and the + // post-tool-use hook fires. + tr.append( + `{"type":"assistant","uuid":"a1","parentUuid":"u1","timestamp":"2026-10-07T10:01:00.000Z","message":{"model":"claude-opus-5-5","role":"assistant","content":[{"type":"tool_use","id":"toolu_1","name":"Skill","input":{"skill":"asd-ste100","args":"the PR description"}}],"usage":{"input_tokens":10,"output_tokens":5}}}`, + `{"type":"user","uuid":"r1","parentUuid":"a1","timestamp":"2026-10-07T10:01:01.000Z","toolUseResult":{"success":true,"commandName":"asd-ste100"},"message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"Launching skill: asd-ste100"}]}}`, + `{"type":"user","uuid":"m1","parentUuid":"r1","isMeta":true,"sourceToolUseID":"toolu_1","timestamp":"2026-10-07T10:01:01.000Z","message":{"role":"user","content":[{"type":"text","text":`+quote("Base directory for this skill: "+userSkill+"\n\n# STE")+`}]}}`, + ) + withStdin(t, hookPayload("PostToolUse", "Skill")) + require.NoError(t, HandleAgentPostToolUse(provider, zerolog.Nop())) + + // The skill changes on disk after it ran. The evidence must hold the + // skill as it ran. + require.NoError(t, os.WriteFile(filepath.Join(userSkill, skill.DefinitionFile), []byte("# STE\nEdited after the use.\n"), 0o600)) + + // The user starts the project skill with a slash command, and the next + // prompt fires the prompt-submit hook. + tr.append( + `{"type":"user","uuid":"c1","timestamp":"2026-10-07T10:05:00.000Z","message":{"role":"user","content":"release\n/release"}}`, + `{"type":"user","uuid":"m2","parentUuid":"c1","isMeta":true,"timestamp":"2026-10-07T10:05:00.000Z","message":{"role":"user","content":[{"type":"text","text":`+quote("Base directory for this skill: "+projectSkill+"\n\n# Release")+`}]}}`, + `{"type":"user","uuid":"u2","timestamp":"2026-10-07T10:06:00.000Z","message":{"role":"user","content":"thanks"}}`, + ) + withStdin(t, hookPayload("UserPromptSubmit", "")) + captureStdout(t, func() { require.NoError(t, HandleAgentPromptSubmit(provider, zerolog.Nop())) }) + + // The pre-push hook: the push builds the session evidence and adds the + // materials, as RunTracePush does once the attestation exists. + ctx := context.Background() + store, repoRoot, err := state.Locate() + require.NoError(t, err) + records, err := store.LoadAllSessionRecords() + require.NoError(t, err) + + sessions := buildSessionEvidence(ctx, store, repoRoot, tracegit.NewGoGitClient(), + map[string][]*state.CommitRecord{sessionID: nil}, records, aicodingsession.ModeCoding, zerolog.Nop()) + require.Len(t, sessions, 1) + + adder := &fakeMaterialAdder{realDigests: true} + attested, _ := attachSessionEvidence(ctx, adder, store, sessions, zerolog.Nop()) + require.Equal(t, []string{sessionID}, attested) + + // Each skill gives its definition and its package, then the session + // material comes last. + assert.Equal(t, []string{ + "spec-0f3a6c-skill-asd-ste100", + "spec-0f3a6c-skill-asd-ste100-pkg", + "spec-0f3a6c-skill-release", + "spec-0f3a6c-skill-release-pkg", + "ai-coding-session-0f3a6c", + }, adder.names()) + + definition := adder.byName("spec-0f3a6c-skill-asd-ste100") + assert.Equal(t, "EVIDENCE", definition.kind) + assert.Equal(t, skill.DefinitionFile, definition.fileName) + assert.Contains(t, definition.content, "As it ran.", "the evidence holds the skill as it ran") + assert.NotContains(t, definition.content, pat, "the definition is redacted") + assert.Equal(t, skillContentDefinition, definition.annotations[specAnnotationSkillContent]) + + pkg := adder.byName("spec-0f3a6c-skill-asd-ste100-pkg") + assert.Equal(t, "EVIDENCE", pkg.kind) + assert.Equal(t, "asd-ste100.tar.gz", pkg.fileName) + assert.Equal(t, skillContentPackage, pkg.annotations[specAnnotationSkillContent]) + files := untar(t, pkg.content) + assert.ElementsMatch(t, []string{skill.DefinitionFile, steLintScript}, slices.Collect(maps.Keys(files)), "the package holds the full folder, without .git") + assert.Equal(t, definition.content, files[skill.DefinitionFile]) + assert.NotContains(t, files[steLintScript], pat, "each file of the package is redacted") + + // The session material holds the skill entries, which point to the + // materials above, and passes the schema. + session := adder.byName("ai-coding-session-0f3a6c") + assert.Equal(t, "CHAINLOOP_AI_CODING_SESSION", session.kind) + + var evidence aicodingsession.Evidence + require.NoError(t, json.Unmarshal([]byte(session.content), &evidence)) + require.Len(t, evidence.Data.Spec, 2) + + digestOf := func(name string) string { return adder.byName(name).digest } + + assert.Equal(t, aicodingsession.SpecEntry{ + Kind: aicodingsession.SpecKindSkill, Title: "asd-ste100", + Digest: digestOf("spec-0f3a6c-skill-asd-ste100"), CapturedAt: "2026-10-07T10:01:00Z", + Metadata: &aicodingsession.SpecMetadata{ + Source: aicodingsession.SkillSourceUser, PackageDigest: digestOf("spec-0f3a6c-skill-asd-ste100-pkg"), + InvocationCount: 1, ByModel: 1, + }, + }, evidence.Data.Spec[0]) + assert.Equal(t, aicodingsession.SpecEntry{ + Kind: aicodingsession.SpecKindSkill, Title: "release", + Digest: digestOf("spec-0f3a6c-skill-release"), CapturedAt: "2026-10-07T10:05:00Z", + Metadata: &aicodingsession.SpecMetadata{ + Source: aicodingsession.SkillSourceProject, PackageDigest: digestOf("spec-0f3a6c-skill-release-pkg"), + InvocationCount: 1, ByUser: 1, + }, + }, evidence.Data.Spec[1]) + + for _, w := range evidence.Data.Warnings { + assert.NotContains(t, w, "skill", "the hooks copied both skills at their use") + } + + // The material crafter validates the data of the envelope, decoded as + // generic JSON so that an unknown field fails the schema. + var envelope struct { + Data any `json:"data"` + } + require.NoError(t, json.Unmarshal([]byte(session.content), &envelope)) + require.NoError(t, schemavalidators.ValidateAICodingSession(envelope.Data, schemavalidators.AICodingSessionVersion0_1)) +} diff --git a/app/cli/pkg/action/trace_skills_test.go b/app/cli/pkg/action/trace_skills_test.go new file mode 100644 index 000000000..b644306ee --- /dev/null +++ b/app/cli/pkg/action/trace_skills_test.go @@ -0,0 +1,332 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package action + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + skillSessionID = "7412a0c2-1a2b-4c3d-8e9f-0a1b2c3d4e5f" + skillDoc = "# Skill" + skillTenOClock = "2026-10-07T10:00:00Z" + steSkillName = "asd-ste100" +) + +// skillProvider is a provider that records skills, with uses and loads set by +// the test. +type skillProvider struct { + trace.Provider + + uses []trace.SkillUse + loads []string + err error + roots skill.Roots +} + +func (p *skillProvider) NewSkillLoads(_ *state.Store, _ *trace.HookInput) []string { + return p.loads +} + +func (p *skillProvider) SessionSkills(_ *trace.ParseOpts) ([]trace.SkillUse, []string, error) { + return p.uses, nil, p.err +} + +func (p *skillProvider) SkillRoots(_ string) skill.Roots { + return p.roots +} + +// noSkillProvider is a provider without skill support, such as Cursor. +type noSkillProvider struct { + trace.Provider +} + +func makeSkill(t *testing.T, files map[string]string) string { + t.Helper() + + return makeSkillAt(t, t.TempDir(), files) +} + +// makeSkillAt writes a skill folder at dir from a map of relative paths to +// contents. +func makeSkillAt(t *testing.T, dir string, files map[string]string) string { + t.Helper() + + for rel, content := range files { + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + } + + return dir +} + +func untar(t *testing.T, archive string) map[string]string { + t.Helper() + + gz, err := gzip.NewReader(bytes.NewReader([]byte(archive))) + require.NoError(t, err) + + out := make(map[string]string) + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return out + } + require.NoError(t, err) + + content, err := io.ReadAll(tr) + require.NoError(t, err) + out[hdr.Name] = string(content) + } +} + +func TestCaptureSkillLoads(t *testing.T) { + t.Run("a skill file that changes after the use does not change the package", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: "as it ran"}) + provider := &skillProvider{ + loads: []string{dir}, + uses: []trace.SkillUse{{Name: steSkillName, Dir: dir, FirstUsedAt: skillTenOClock, ByModel: 2}}, + } + input := &trace.HookInput{SessionID: skillSessionID} + + captureSkillLoads(provider, store, input, zerolog.Nop()) + require.NoError(t, os.WriteFile(filepath.Join(dir, skill.DefinitionFile), []byte("changed later"), 0o600)) + // A later use finds the changed folder. + captureSkillLoads(provider, store, input, zerolog.Nop()) + + skills, warnings := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + require.Len(t, skills, 1) + assert.Equal(t, []string{`skill "asd-ste100" changed after its first use; the evidence holds the skill as it was at the first use`}, warnings) + + adder := &fakeMaterialAdder{} + _, _ = attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, skills, zerolog.Nop()) + require.Len(t, adder.added, 2) + assert.Equal(t, "as it ran", adder.added[0].content) + }) + + t.Run("a provider without skill support copies nothing", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + captureSkillLoads(&noSkillProvider{}, store, &trace.HookInput{SessionID: skillSessionID}, zerolog.Nop()) + + skills, warnings := readSessionSkills(&noSkillProvider{}, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + assert.Empty(t, skills) + assert.Empty(t, warnings, "no skill entry means not recorded, with no warning") + }) +} + +func TestReadSessionSkills(t *testing.T) { + t.Run("a skill that no hook copied is copied at push time, with a warning", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: skillDoc}) + provider := &skillProvider{uses: []trace.SkillUse{{Name: steSkillName, Dir: dir, ByUser: 1}}} + + skills, warnings := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + require.Len(t, skills, 1) + assert.Equal(t, []string{`skill "asd-ste100" was copied at push time, not when the session used it`}, warnings) + }) + + t.Run("a missing folder gives a warning and the other skills", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: skillDoc}) + provider := &skillProvider{uses: []trace.SkillUse{ + {Name: "gone", Dir: filepath.Join(t.TempDir(), "gone"), ByModel: 1}, + {Name: "unknown-dir", ByModel: 1}, + {Name: steSkillName, Dir: dir, ByModel: 1}, + }} + captureSkillLoads(&skillProvider{loads: []string{dir}}, store, &trace.HookInput{SessionID: skillSessionID}, zerolog.Nop()) + + var logs bytes.Buffer + skills, warnings := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.New(&logs)) + require.Len(t, skills, 1) + assert.Equal(t, steSkillName, skills[0].use.Name) + assert.Equal(t, []string{ + `skill "gone" was not recorded: its folder could not be read`, + }, warnings, "a skill with no folder is only in the local log") + assert.Contains(t, logs.String(), `"level":"warn","session":"`+skillSessionID+`","skill":"unknown-dir","message":"a skill that the session used has no skill folder, so it is not recorded"`) + for _, w := range warnings { + assert.NotContains(t, w, string(filepath.Separator)+"gone", "a warning holds no local path") + } + }) + + t.Run("skills beyond the limit are dropped, the first ones kept", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: skillDoc}) + captureSkillLoads(&skillProvider{loads: []string{dir}}, store, &trace.HookInput{SessionID: skillSessionID}, zerolog.Nop()) + + uses := make([]trace.SkillUse, 0, maxSkillEntries+2) + for i := range maxSkillEntries + 2 { + uses = append(uses, trace.SkillUse{Name: "skill-" + string(rune('a'+i)), Dir: dir, ByModel: 1}) + } + + skills, warnings := readSessionSkills(&skillProvider{uses: uses}, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + require.Len(t, skills, maxSkillEntries) + assert.Equal(t, "skill-a", skills[0].use.Name) + assert.Contains(t, warnings, "2 skill entries beyond the first 25 were dropped") + }) + + t.Run("a session data failure costs the skills only", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + skills, warnings := readSessionSkills(&skillProvider{err: errors.New("/home/me/x: no such file")}, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + assert.Empty(t, skills) + assert.Equal(t, []string{"the skills that the session used were not recorded: the session data could not be read"}, warnings) + }) + + t.Run("the source comes from the provider roots", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: skillDoc}) + captureSkillLoads(&skillProvider{loads: []string{dir}}, store, &trace.HookInput{SessionID: skillSessionID}, zerolog.Nop()) + + provider := &skillProvider{ + uses: []trace.SkillUse{{Name: steSkillName, Dir: dir, ByModel: 1}}, + roots: skill.Roots{User: []string{filepath.Dir(dir)}}, + } + + skills, _ := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + require.Len(t, skills, 1) + assert.Equal(t, aicodingsession.SkillSourceUser, skills[0].source) + }) +} + +func TestAttachSkills(t *testing.T) { + // Assembled from fragments so that nothing credential-shaped is committed. + const pat = "ghp_erOZlZv0B1e3amrQ" + "ugdwZ8Ro2W4kDql9WPTf" + + stored := func(t *testing.T, files map[string]string, use trace.SkillUse) sessionSkill { + t.Helper() + + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, files) + dst := store.SkillCopyDir(skillSessionID, dir) + _, err := skill.Copy(dir, dst) + require.NoError(t, err) + + use.Dir = dir + + return sessionSkill{use: use, source: aicodingsession.SkillSourceUser, copyDir: dst} + } + + t.Run("each skill gives a definition, a package and an entry", func(t *testing.T) { + s := stored(t, map[string]string{ + skill.DefinitionFile: "# STE\ntoken " + pat + "\n", + steLintScript: "print('" + pat + "')", + }, trace.SkillUse{Name: steSkillName, FirstUsedAt: "2026-10-07T10:14:51Z", ByModel: 2, ByUser: 1, InSubagents: 1}) + + adder := &fakeMaterialAdder{} + entries, warnings := attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + assert.Empty(t, warnings) + require.Len(t, adder.added, 2) + + definition, pkg := adder.added[0], adder.added[1] + assert.Equal(t, "spec-7412a0-skill-asd-ste100", definition.name) + assert.Equal(t, skill.DefinitionFile, definition.fileName) + assert.Equal(t, "EVIDENCE", definition.kind) + assert.NotContains(t, definition.content, pat, "the definition is redacted") + assert.Equal(t, map[string]string{ + specAnnotationSession: skillSessionID, + specAnnotationKind: aicodingsession.SpecKindSkill, + specAnnotationTitle: steSkillName, + specAnnotationSkillContent: skillContentDefinition, + }, definition.annotations) + + assert.Equal(t, "spec-7412a0-skill-asd-ste100-pkg", pkg.name) + assert.Equal(t, "asd-ste100.tar.gz", pkg.fileName) + assert.Equal(t, "EVIDENCE", pkg.kind) + assert.Equal(t, "package", pkg.annotations["chainloop.spec.skill.content"]) + + files := untar(t, pkg.content) + assert.Equal(t, definition.content, files[skill.DefinitionFile], "the definition and the packaged SKILL.md are the same bytes") + assert.NotContains(t, files[steLintScript], pat, "each text file is redacted") + + assert.Equal(t, []aicodingsession.SpecEntry{{ + Kind: aicodingsession.SpecKindSkill, + Title: steSkillName, + Digest: "sha256:spec-7412a0-skill-asd-ste100", + CapturedAt: "2026-10-07T10:14:51Z", + Metadata: &aicodingsession.SpecMetadata{ + Source: aicodingsession.SkillSourceUser, + PackageDigest: "sha256:spec-7412a0-skill-asd-ste100-pkg", + InvocationCount: 3, ByModel: 2, ByUser: 1, InSubagents: 1, + }, + }}, entries, "no role, description or source address") + }) + + t.Run("a package over the limit keeps the definition and the counts", func(t *testing.T) { + big := make([]byte, skill.MaxPackageSize+1024) + _, err := rand.Read(big) + require.NoError(t, err) + + s := stored(t, map[string]string{skill.DefinitionFile: "# Big", "model.bin": string(big)}, trace.SkillUse{Name: "big", FirstUsedAt: skillTenOClock, ByModel: 1}) + + adder := &fakeMaterialAdder{} + entries, warnings := attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + assert.Equal(t, []string{`the package of skill "big" is larger than 5 MB and was not uploaded`}, warnings) + require.Len(t, adder.added, 1) + require.Len(t, entries, 1) + assert.Empty(t, entries[0].Metadata.PackageDigest) + assert.Equal(t, 1, entries[0].Metadata.InvocationCount) + }) + + t.Run("a package that cannot be stored costs the package only", func(t *testing.T) { + s := stored(t, map[string]string{skill.DefinitionFile: skillDoc}, trace.SkillUse{Name: steSkillName, FirstUsedAt: skillTenOClock, ByModel: 1}) + + adder := &fakeMaterialAdder{failOn: map[string]bool{"spec-7412a0-skill-asd-ste100-pkg": true}} + entries, warnings := attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + assert.Equal(t, []string{`the package of skill "asd-ste100" was not uploaded`}, warnings) + require.Len(t, entries, 1) + assert.Empty(t, entries[0].Metadata.PackageDigest) + }) + + t.Run("a definition that cannot be stored leaves the skill out", func(t *testing.T) { + s := stored(t, map[string]string{skill.DefinitionFile: skillDoc}, trace.SkillUse{Name: steSkillName, FirstUsedAt: skillTenOClock, ByModel: 1}) + + adder := &fakeMaterialAdder{failOn: map[string]bool{"spec-7412a0-skill-asd-ste100": true}} + entries, warnings := attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + assert.Empty(t, entries) + assert.Equal(t, []string{`skill "asd-ste100" was not recorded`}, warnings) + }) + + t.Run("a plugin skill gets a valid material name", func(t *testing.T) { + s := stored(t, map[string]string{skill.DefinitionFile: skillDoc}, trace.SkillUse{Name: "superpowers:brainstorming", FirstUsedAt: skillTenOClock, ByModel: 1}) + + adder := &fakeMaterialAdder{} + _, _ = attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + require.Len(t, adder.added, 2) + assert.Equal(t, "spec-7412a0-skill-superpowers-brainstorming", adder.added[0].name) + assert.Equal(t, "superpowers-brainstorming.tar.gz", adder.added[1].fileName) + assert.Equal(t, "superpowers:brainstorming", adder.added[0].annotations["chainloop.spec.title"]) + }) +} diff --git a/app/cli/pkg/action/trace_spec_materials.go b/app/cli/pkg/action/trace_spec_materials.go index ab443b391..b0b8ebbda 100644 --- a/app/cli/pkg/action/trace_spec_materials.go +++ b/app/cli/pkg/action/trace_spec_materials.go @@ -261,18 +261,30 @@ func (r *specRedactor) Redact(ctx context.Context, doc []byte) ([]byte, error) { // ID>-. The allocator sanitizes it into a valid material name and // makes it unique within the attestation. func specMaterialName(sessionID, fileName string) string { + return specMaterialPrefix(sessionID) + materialStem(strings.TrimSuffix(fileName, filepath.Ext(fileName)), "source") +} + +// specMaterialPrefix is the start of the name of each spec material of a +// session: spec--. +func specMaterialPrefix(sessionID string) string { short := sessionID if len(short) > 6 { short = short[:6] } - stem := materials.SanitizeMaterialName(strings.TrimSuffix(fileName, filepath.Ext(fileName))) + return "spec-" + short + "-" +} + +// materialStem sanitizes name into the part of a material name taken from it, +// cut to maxSpecSlugLen, or returns fallback when nothing is left. +func materialStem(name, fallback string) string { + stem := materials.SanitizeMaterialName(name) if len(stem) > maxSpecSlugLen { stem = strings.TrimRight(stem[:maxSpecSlugLen], "-") } if stem == "" { - stem = "source" + return fallback } - return "spec-" + short + "-" + stem + return stem } diff --git a/app/cli/pkg/action/trace_spec_materials_test.go b/app/cli/pkg/action/trace_spec_materials_test.go index cc6030ec0..255da9cbb 100644 --- a/app/cli/pkg/action/trace_spec_materials_test.go +++ b/app/cli/pkg/action/trace_spec_materials_test.go @@ -36,14 +36,38 @@ import ( // addedMaterial is one call recorded by fakeMaterialAdder, with the file read // back at call time: the caller is free to delete it once the add returns. type addedMaterial struct { - name, fileName, kind, content string - annotations map[string]string + name, fileName, kind, content, digest string + annotations map[string]string } type fakeMaterialAdder struct { added []addedMaterial // failOn names the materials whose add fails. failOn map[string]bool + // realDigests returns the SHA-256 digest of the content, as the + // attestation does, instead of a digest made from the name. + realDigests bool +} + +// byName returns the material added under name. +func (f *fakeMaterialAdder) byName(name string) addedMaterial { + for _, m := range f.added { + if m.name == name { + return m + } + } + + return addedMaterial{} +} + +// names returns the names of the added materials, in the order of the adds. +func (f *fakeMaterialAdder) names() []string { + out := make([]string, 0, len(f.added)) + for _, m := range f.added { + out = append(out, m.name) + } + + return out } func (f *fakeMaterialAdder) AddMaterial(_ context.Context, name, path, kind string, annotations map[string]string) (string, error) { @@ -56,11 +80,17 @@ func (f *fakeMaterialAdder) AddMaterial(_ context.Context, name, path, kind stri return "", err } + digest := "sha256:" + name + if f.realDigests { + sum := sha256.Sum256(content) + digest = "sha256:" + hex.EncodeToString(sum[:]) + } + f.added = append(f.added, addedMaterial{ - name: name, fileName: filepath.Base(path), kind: kind, content: string(content), annotations: annotations, + name: name, fileName: filepath.Base(path), kind: kind, content: string(content), digest: digest, annotations: annotations, }) - return "sha256:" + name, nil + return digest, nil } // specCapture builds a capture the way the spec folder reader does: parsed from diff --git a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json index 70654051e..b3bf2d908 100644 --- a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json +++ b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json @@ -80,7 +80,7 @@ "properties": { "kind": { "type": "string", - "description": "What this entry was resolved from: 'ticket', 'document', 'image', or 'text' for a spec stated in the session itself. Left unconstrained so further kinds need no schema version bump; a producer normalises anything it does not recognise to 'text'." + "description": "What this entry was resolved from: 'ticket', 'document', 'image', 'text' for a spec stated in the session itself, or 'skill' for a skill that the session used. Left unconstrained so further kinds need no schema version bump; a producer normalises anything it does not recognise to 'text'." }, "role": { "type": "string", @@ -106,7 +106,43 @@ "captured_at": { "type": "string", "minLength": 1, - "description": "ISO 8601 timestamp of when the agent wrote this entry. Self-reported by the producer, not a trusted timestamp." + "description": "ISO 8601 timestamp of when the agent wrote this entry. For a 'skill' entry, the time of the first use in the transcript. Self-reported by the producer, not a trusted timestamp." + }, + "metadata": { + "type": "object", + "description": "Data that depends on the kind of the entry. Only 'skill' entries have it. For a skill, the digest of the entry points to the SKILL.md file.", + "properties": { + "source": { + "type": "string", + "description": "Where the skill came from: 'project', 'user', 'organization', 'plugin', or 'unknown'. Left unconstrained so further values need no schema version bump." + }, + "package_digest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$", + "description": "Digest of the EVIDENCE material that holds a reproducible archive of the full skill folder. Absent when the archive was not uploaded." + }, + "invocation_count": { + "type": "integer", + "minimum": 0, + "description": "Number of times the session used the skill: by_model plus by_user" + }, + "by_model": { + "type": "integer", + "minimum": 0, + "description": "Number of uses that the model started with a tool call" + }, + "by_user": { + "type": "integer", + "minimum": 0, + "description": "Number of uses that the user started with a slash command" + }, + "in_subagents": { + "type": "integer", + "minimum": 0, + "description": "Number of uses that came from subagents" + } + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go index 58ca4ae9d..638921db7 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go +++ b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go @@ -70,6 +70,11 @@ const ( SpecKindText = "text" ) +// SpecKindSkill is a skill that the agent used in the session. The CLI makes +// these entries from the transcript, never the agent, so ResolveSpecKind does +// not accept it: an agent that writes it in a spec file gets kind text. +const SpecKindSkill = "skill" + // ResolveSpecKind maps a captured kind onto the vocabulary above, so the // normalisation is written down once instead of at each producer and consumer. func ResolveSpecKind(kind string) string { @@ -137,8 +142,50 @@ type SpecEntry struct { Digest string `json:"digest"` // CapturedAt is when the agent wrote this entry, RFC3339. It comes from the // file's modification time, so it is the last write rather than the first, - // and it is trivially forgeable: not a trusted timestamp. + // and it is trivially forgeable: not a trusted timestamp. For a skill, it + // is the time of the first use in the transcript. CapturedAt string `json:"captured_at"` + // Metadata holds data that depends on the kind. Only skill entries have + // it. + Metadata *SpecMetadata `json:"metadata,omitempty"` +} + +// Where a skill came from. The evidence never holds the path of the skill +// folder, because the path exposes local user data. +const ( + // SkillSourceProject is a skill from the repository. + SkillSourceProject = "project" + // SkillSourceUser is a skill from the user's agent configuration. + SkillSourceUser = "user" + // SkillSourceOrganization is a skill that an administrator installs for + // all users. + SkillSourceOrganization = "organization" + // SkillSourcePlugin is a skill from an installed plugin. + SkillSourcePlugin = "plugin" + // SkillSourceUnknown is any other skill. + SkillSourceUnknown = "unknown" +) + +// SpecMetadata is the kind-specific data of a spec entry. The fields below are +// those of a skill entry. A later kind can add its own fields. +type SpecMetadata struct { + // Source is one of the SkillSource* constants. + Source string `json:"source,omitempty"` + // PackageDigest identifies the EVIDENCE material that holds an archive + // of the full skill folder, as "sha256:". Absent when the archive was + // not uploaded, for example because it was too large. + PackageDigest string `json:"package_digest,omitempty"` + // InvocationCount is the number of times the session used the skill. It + // is ByModel plus ByUser. + InvocationCount int `json:"invocation_count"` + // ByModel is the number of uses that the model started with a tool call. + ByModel int `json:"by_model"` + // ByUser is the number of uses that the user started with a slash + // command. + ByUser int `json:"by_user"` + // InSubagents is the number of uses, of either start, that came from + // subagents. + InSubagents int `json:"in_subagents"` } // Agent identifies the AI agent provider. diff --git a/pkg/attestation/crafter/materials/aicodingsession/spec_test.go b/pkg/attestation/crafter/materials/aicodingsession/spec_test.go index 376141a28..a5ae7282a 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/spec_test.go +++ b/pkg/attestation/crafter/materials/aicodingsession/spec_test.go @@ -41,6 +41,8 @@ func TestResolveSpecKind(t *testing.T) { // newer Chainloop, so a value outside the vocabulary is a mistake to // normalise away rather than a future kind to pass through. {name: "a kind outside the vocabulary is text", kind: "jira-ticket", want: SpecKindText}, + // Only the CLI makes skill entries, so an agent cannot write one. + {name: "an agent-written skill is text", kind: SpecKindSkill, want: SpecKindText}, } for _, tc := range testCases { @@ -97,6 +99,25 @@ func TestSpecEntryMatchesSchema(t *testing.T) { name: "only what the schema requires", entry: SpecEntry{Kind: SpecKindText, Digest: testSpecDigest, CapturedAt: "2026-09-16T10:12:03Z"}, }, + { + name: "a skill with its metadata", + entry: SpecEntry{ + Kind: SpecKindSkill, Title: "asd-ste100", Digest: testSpecDigest, CapturedAt: "2026-10-07T10:14:51Z", + Metadata: &SpecMetadata{ + Source: SkillSourceUser, PackageDigest: testSpecDigest, + InvocationCount: 3, ByModel: 2, ByUser: 1, InSubagents: 1, + }, + }, + }, + { + // A package over the limit is not uploaded, and a session can + // use a skill only from the model: zero counts stay in the output. + name: "a skill without a package", + entry: SpecEntry{ + Kind: SpecKindSkill, Title: "graphify", Digest: testSpecDigest, CapturedAt: "2026-10-07T10:14:51Z", + Metadata: &SpecMetadata{Source: SkillSourceUnknown, InvocationCount: 1, ByModel: 1}, + }, + }, } for _, tc := range testCases {