From 6f7eab67c654b298c4919789a540afdb3f31b03d Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Wed, 7 Oct 2026 18:03:54 +0200 Subject: [PATCH 1/4] feat(trace): record the skills that an agent used in a session Each skill that a Claude Code or OpenCode session used becomes a spec entry of kind skill. The entry holds the use counts and the skill source in a new metadata object. The redacted SKILL.md file and a reproducible archive of the full skill folder are stored as EVIDENCE materials. Hooks copy the skill folder when the session uses the skill. Implements R-001 to R-010 of docs/specs/issue-3540-trace-skills-used.md. Refs #3540 Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: ee05bc4f-cf05-4433-b614-415b89577db7 --- app/cli/internal/trace/claude/hooks.go | 6 +- app/cli/internal/trace/claude/hooks_test.go | 7 +- app/cli/internal/trace/claude/skills.go | 437 ++++++++++++++ app/cli/internal/trace/claude/skills_test.go | 317 ++++++++++ app/cli/internal/trace/opencode/hooks.go | 40 +- app/cli/internal/trace/opencode/hooks_test.go | 50 +- app/cli/internal/trace/opencode/parse.go | 22 +- app/cli/internal/trace/opencode/skills.go | 143 +++++ .../internal/trace/opencode/skills_test.go | 114 ++++ .../trace/opencode/testdata/plugin_full.ts | 30 +- .../opencode/testdata/plugin_tracerun.ts | 30 +- app/cli/internal/trace/opencode/types.go | 8 + app/cli/internal/trace/provider.go | 93 +++ app/cli/internal/trace/skill/skill.go | 552 ++++++++++++++++++ app/cli/internal/trace/skill/skill_test.go | 287 +++++++++ app/cli/internal/trace/skill/source.go | 90 +++ app/cli/internal/trace/skill/source_test.go | 54 ++ app/cli/internal/trace/state/skills.go | 103 ++++ app/cli/internal/trace/state/skills_test.go | 83 +++ app/cli/internal/trace/state/state.go | 36 +- app/cli/pkg/action/trace_agent_hook.go | 17 +- app/cli/pkg/action/trace_hook_handler.go | 17 +- app/cli/pkg/action/trace_skills.go | 278 +++++++++ app/cli/pkg/action/trace_skills_test.go | 324 ++++++++++ app/cli/pkg/action/trace_spec_materials.go | 18 +- .../ai-coding-session-0.1.schema.json | 40 +- .../aicodingsession/aicodingsession.go | 49 +- .../materials/aicodingsession/spec_test.go | 21 + 28 files changed, 3221 insertions(+), 45 deletions(-) create mode 100644 app/cli/internal/trace/claude/skills.go create mode 100644 app/cli/internal/trace/claude/skills_test.go create mode 100644 app/cli/internal/trace/opencode/skills.go create mode 100644 app/cli/internal/trace/opencode/skills_test.go create mode 100644 app/cli/internal/trace/skill/skill.go create mode 100644 app/cli/internal/trace/skill/skill_test.go create mode 100644 app/cli/internal/trace/skill/source.go create mode 100644 app/cli/internal/trace/skill/source_test.go create mode 100644 app/cli/internal/trace/state/skills.go create mode 100644 app/cli/internal/trace/state/skills_test.go create mode 100644 app/cli/pkg/action/trace_skills.go create mode 100644 app/cli/pkg/action/trace_skills_test.go diff --git a/app/cli/internal/trace/claude/hooks.go b/app/cli/internal/trace/claude/hooks.go index 2b975c01c..89560326b 100644 --- a/app/cli/internal/trace/claude/hooks.go +++ b/app/cli/internal/trace/claude/hooks.go @@ -57,6 +57,10 @@ var commandTools = []string{"Bash"} // kind fired. var hookToolMatcher = strings.Join(append(append([]string{}, fileWritingTools...), commandTools...), "|") +// postHookToolMatcher also covers the Skill tool. The skill is loaded only +// after the call, so the post hook is the first one that can copy its folder. +var postHookToolMatcher = hookToolMatcher + "|" + skillTool + type hookEvent struct { event string command string @@ -67,7 +71,7 @@ var hookEvents = []hookEvent{ {eventSessionStart, "chainloop trace hook claude session-start", ""}, {eventUserPromptSubmit, "chainloop trace hook claude user-prompt-submit", ""}, {eventPreToolUse, "chainloop trace hook claude pre-tool-use", hookToolMatcher}, - {eventPostToolUse, "chainloop trace hook claude post-tool-use", hookToolMatcher}, + {eventPostToolUse, "chainloop trace hook claude post-tool-use", postHookToolMatcher}, {eventPostToolUseFailure, "chainloop trace hook claude post-tool-use", hookToolMatcher}, {eventSessionEnd, "chainloop trace hook claude session-end", ""}, } diff --git a/app/cli/internal/trace/claude/hooks_test.go b/app/cli/internal/trace/claude/hooks_test.go index 5f666f451..ac7855648 100644 --- a/app/cli/internal/trace/claude/hooks_test.go +++ b/app/cli/internal/trace/claude/hooks_test.go @@ -64,12 +64,13 @@ func TestInstallHooks(t *testing.T) { preEntry := preEntries[0].(map[string]any) assert.Equal(t, "Edit|Write|MultiEdit|Bash", preEntry["matcher"]) - // PostToolUse should have matcher + // PostToolUse also matches Skill, to copy the folder of a loaded skill postEntries := hooks["PostToolUse"].([]any) postEntry := postEntries[0].(map[string]any) - assert.Equal(t, "Edit|Write|MultiEdit|Bash", postEntry["matcher"]) + assert.Equal(t, "Edit|Write|MultiEdit|Bash|Skill", postEntry["matcher"]) - // PostToolUseFailure should have the same matcher + // PostToolUseFailure keeps the edit matcher: a failed Skill call loads + // no skill failureEntries := hooks["PostToolUseFailure"].([]any) failureEntry := failureEntries[0].(map[string]any) assert.Equal(t, "Edit|Write|MultiEdit|Bash", failureEntry["matcher"]) diff --git a/app/cli/internal/trace/claude/skills.go b/app/cli/internal/trace/claude/skills.go new file mode 100644 index 000000000..1d8f3ebe9 --- /dev/null +++ b/app/cli/internal/trace/claude/skills.go @@ -0,0 +1,437 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package claude + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "time" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" +) + +// Compile-time check that Provider records skills. +var _ trace.SkillTracker = (*Provider)(nil) + +// skillTool is the tool that the model calls to load a skill. +const skillTool = "Skill" + +// mainTranscriptKey names the main transcript in the scan offsets of a +// session. Subagent transcripts use their file name. +const mainTranscriptKey = "main" + +// commandNameRe finds the slash command of a user message, as Claude Code +// writes it: /graphify. +var commandNameRe = regexp.MustCompile(`\s*/?([^<\s]+)\s*`) + +// skillRecord holds the fields of a transcript record that tell a skill use. +type skillRecord struct { + Type string `json:"type"` + UUID string `json:"uuid"` + ParentUUID string `json:"parentUuid"` + Timestamp string `json:"timestamp"` + IsMeta bool `json:"isMeta"` + // SourceToolUseID links the message that holds a loaded skill to the + // tool call that loaded it. A skill that the user loaded has none. + SourceToolUseID string `json:"sourceToolUseID"` + Message *struct { + Content json.RawMessage `json:"content"` + } `json:"message"` +} + +// skillBlock is a content block of a transcript message. +type skillBlock struct { + Type string `json:"type"` + Text string `json:"text"` + ID string `json:"id"` + Name string `json:"name"` + Input struct { + Skill string `json:"skill"` + } `json:"input"` +} + +// text returns the text of a message, whether the content is a string or a +// list of blocks. +func (r *skillRecord) text() string { + if r.Message == nil { + return "" + } + + var s string + if json.Unmarshal(r.Message.Content, &s) == nil { + return s + } + + var b strings.Builder + for _, block := range r.blocks() { + if block.Type == "text" { + b.WriteString(block.Text) + b.WriteString("\n") + } + } + + return b.String() +} + +func (r *skillRecord) blocks() []skillBlock { + if r.Message == nil { + return nil + } + + var blocks []skillBlock + if json.Unmarshal(r.Message.Content, &blocks) != nil { + return nil + } + + return blocks +} + +// mayHoldSkillUse is a cheap test that a scan runs before it decodes a line. +// Only the lines that start or load a skill pass it. The command marker is +// matched without its brackets, which a JSON encoder can escape. +func mayHoldSkillUse(line []byte) bool { + return skill.HasBaseDir(line) || + bytes.Contains(line, []byte(`"name":"`+skillTool+`"`)) || + bytes.Contains(line, []byte("command-name")) +} + +// start is the record that started a skill use: a tool call or a slash +// command. +type start struct { + name string + timestamp string +} + +// transcriptSkills counts the skill uses of one transcript. +// +// Claude Code writes a meta message that starts with "Base directory for this +// skill:" each time it loads a skill. A meta message that names a tool call +// is a model start, and one whose parent is a slash command is a user start. +// The meta message is what tells a skill command apart from a built-in or a +// custom command, and a loaded skill apart from a failed tool call, which +// gets no meta message. Each start counts once, also when the transcript +// holds its meta message more than once. +func transcriptSkills(path string, subagent bool) (map[string]*trace.SkillUse, error) { + records, err := readSkillRecords(path) + if err != nil { + return nil, err + } + + toolCalls, commands := skillStarts(records) + + uses := make(map[string]*trace.SkillUse) + counted := make(map[string]bool) + for i := range records { + r := &records[i] + if r.Type != recordTypeUser || !r.IsMeta { + continue + } + + dir, ok := skill.BaseDir(r.text()) + if !ok { + continue + } + + s, key, byModel, ok := startOf(r, dir, toolCalls, commands) + if !ok || counted[key] { + continue + } + counted[key] = true + + at := normalizeTimestamp(s.timestamp) + if at == "" { + at = normalizeTimestamp(r.Timestamp) + } + + use, ok := uses[s.name] + if !ok { + use = &trace.SkillUse{Name: s.name, Dir: dir} + uses[s.name] = use + } + use.Record(at, byModel, subagent) + } + + return uses, nil +} + +// readSkillRecords returns the records of a transcript that can start or +// load a skill. +func readSkillRecords(path string) ([]skillRecord, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer func() { _ = f.Close() }() + + var records []skillRecord + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 0, 64*1024), maxScannerBuffer) + for scanner.Scan() { + line := scanner.Bytes() + if !mayHoldSkillUse(line) { + continue + } + + var r skillRecord + if json.Unmarshal(line, &r) != nil { + continue + } + records = append(records, r) + } + + return records, scanner.Err() +} + +// skillStarts returns the skill tool calls by tool call ID, and the slash +// commands by message ID. +func skillStarts(records []skillRecord) (toolCalls, commands map[string]start) { + toolCalls = make(map[string]start) + commands = make(map[string]start) + for i := range records { + r := &records[i] + switch r.Type { + case recordTypeAssistant: + for _, b := range r.blocks() { + if b.Type == "tool_use" && b.Name == skillTool && b.ID != "" { + toolCalls[b.ID] = start{name: strings.TrimSpace(b.Input.Skill), timestamp: r.Timestamp} + } + } + case recordTypeUser: + if r.IsMeta || r.UUID == "" { + continue + } + if m := commandNameRe.FindStringSubmatch(r.text()); m != nil { + commands[r.UUID] = start{name: m[1], timestamp: r.Timestamp} + } + } + } + + return toolCalls, commands +} + +// startOf returns the start of the skill use that a meta message records, and +// a key that is the same for each meta message of that one start. It reports +// false for a meta message that no start explains. +func startOf(r *skillRecord, dir string, toolCalls, commands map[string]start) (s start, key string, byModel, ok bool) { + if r.SourceToolUseID != "" { + s, ok = toolCalls[r.SourceToolUseID] + if !ok { + // The tool call is not in this transcript. The meta message still + // tells that a skill loaded, and its folder names it. + s = start{name: filepath.Base(dir), timestamp: r.Timestamp} + } + + return s, "tool:" + r.SourceToolUseID, true, s.name != "" + } + + s, ok = commands[r.ParentUUID] + + return s, "command:" + r.ParentUUID, false, ok && s.name != "" +} + +// normalizeTimestamp returns a transcript timestamp as RFC3339 in UTC, or "" +// when it cannot be read. Claude Code writes milliseconds, and the other spec +// entries have seconds. +func normalizeTimestamp(ts string) string { + t, err := time.Parse(time.RFC3339Nano, ts) + if err != nil { + return "" + } + + return t.UTC().Format(time.RFC3339) +} + +// SessionSkills returns the skills that the session used, from the main +// transcript and the subagent transcripts copied under opts.SessionDir. +func (p *Provider) SessionSkills(opts *trace.ParseOpts) ([]trace.SkillUse, []string, error) { + jsonlPath, err := findJSONLPath(opts.SessionDir, opts.SessionID) + if err != nil { + return nil, nil, err + } + + merged, err := transcriptSkills(jsonlPath, false) + if err != nil { + return nil, nil, fmt.Errorf("read session file: %w", err) + } + + var warnings []string + subagents, _ := filepath.Glob(filepath.Join(state.RawSubagentDir(opts.SessionDir, opts.SessionID), "agent-*.jsonl")) + for _, path := range subagents { + uses, err := transcriptSkills(path, true) + if err != nil { + warnings = append(warnings, fmt.Sprintf("the skills of subagent %s were not recorded: it could not be read", filepath.Base(path))) + continue + } + + for name, use := range uses { + if cur, ok := merged[name]; ok { + cur.Merge(*use) + } else { + merged[name] = use + } + } + } + + return trace.SortedSkillUses(merged), warnings, nil +} + +// NewSkillLoads reads the transcripts of the session from where the last +// hook stopped, and returns the folder of each skill that they loaded since. +// It reads the live transcripts, not the copies in the state, because the +// hooks do not refresh the copies. +func (p *Provider) NewSkillLoads(store *state.Store, input *trace.HookInput) []string { + projectsDir := claudeProjectsDir() + if projectsDir == "" || !state.ValidSessionID(input.SessionID) { + return nil + } + + sourceDir, err := transcriptSourceDir(projectsDir, input.TranscriptPath, input.SessionID) + if err != nil { + if input.Cwd == "" { + return nil + } + sourceDir = filepath.Join(projectsDir, encodeCWDForClaudePath(input.Cwd)) + } + + offsets, err := store.SkillScanOffsets(input.SessionID) + if err != nil { + offsets = map[string]int64{} + } + + // The main transcript first, then the subagents, which Glob sorts. + subagents, _ := filepath.Glob(filepath.Join(sourceDir, input.SessionID, "subagents", "agent-*.jsonl")) + paths := append([]string{filepath.Join(sourceDir, input.SessionID+".jsonl")}, subagents...) + + var dirs []string + seen := make(map[string]bool) + changed := false + for i, path := range paths { + key := mainTranscriptKey + if i > 0 { + key = filepath.Base(path) + } + + found, next, err := scanSkillLoads(path, offsets[key]) + if err != nil { + continue + } + if next != offsets[key] { + offsets[key] = next + changed = true + } + for _, dir := range found { + if !seen[dir] { + seen[dir] = true + dirs = append(dirs, dir) + } + } + } + + if changed { + // A failure costs the next hook a scan of the same lines again. + _ = store.SetSkillScanOffsets(input.SessionID, offsets) + } + + return dirs +} + +// scanSkillLoads reads path from offset and returns the skill folders that +// the complete lines name, with the offset after the last complete line. A +// line that the agent is still writing is left for the next hook. +func scanSkillLoads(path string, offset int64) ([]string, int64, error) { + f, err := os.Open(path) + if err != nil { + return nil, offset, err + } + defer func() { _ = f.Close() }() + + info, err := f.Stat() + if err != nil { + return nil, offset, err + } + // A shorter file is a new file, so it is read from the start. + if info.Size() < offset { + offset = 0 + } + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return nil, offset, err + } + + var dirs []string + reader := bufio.NewReaderSize(f, 64*1024) + for { + line, err := reader.ReadBytes('\n') + if errors.Is(err, io.EOF) { + return dirs, offset, nil + } + if err != nil { + return dirs, offset, err + } + offset += int64(len(line)) + + if !skill.HasBaseDir(line) { + continue + } + + var r skillRecord + if json.Unmarshal(line, &r) != nil || r.Type != recordTypeUser || !r.IsMeta { + continue + } + if dir, ok := skill.BaseDir(r.text()); ok { + dirs = append(dirs, dir) + } + } +} + +// SkillRoots returns the Claude Code folders that tell where a skill came +// from. +func (p *Provider) SkillRoots(repoRoot string) skill.Roots { + roots := skill.Roots{ + Organization: []string{managedSettingsDir()}, + Project: []string{repoRoot}, + } + + if home, err := os.UserHomeDir(); err == nil { + roots.Plugin = []string{filepath.Join(home, ".claude", "plugins")} + roots.User = []string{filepath.Join(home, ".claude", "skills")} + } + + return roots +} + +// managedSettingsDir is the folder where an administrator installs the +// Claude Code configuration for all users of a machine. +func managedSettingsDir() string { + switch runtime.GOOS { + case "darwin": + return "/Library/Application Support/ClaudeCode" + case "windows": + return `C:\Program Files\ClaudeCode` + default: + return "/etc/claude-code" + } +} diff --git a/app/cli/internal/trace/claude/skills_test.go b/app/cli/internal/trace/claude/skills_test.go new file mode 100644 index 000000000..1666cb2a1 --- /dev/null +++ b/app/cli/internal/trace/claude/skills_test.go @@ -0,0 +1,317 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package claude + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const skillsSessionID = "5c584dae-abf9-4a94-8598-4dc1ce6579d0" + +const ( + steSkill = "asd-ste100" + graphSkill = "graphify" + tenOClock = "2026-10-07T10:00:00Z" + tenOClockM = "2026-10-07T10:00:00.000Z" +) + +// fixtureRecord, fixtureMessage and fixtureBlock have the shape that Claude +// Code 2.1 writes for a skill. +type fixtureRecord struct { + Type string `json:"type"` + UUID string `json:"uuid,omitempty"` + ParentUUID string `json:"parentUuid,omitempty"` + Timestamp string `json:"timestamp,omitempty"` + IsMeta bool `json:"isMeta,omitempty"` + SourceToolUseID string `json:"sourceToolUseID,omitempty"` + Message fixtureMessage `json:"message"` +} + +type fixtureMessage struct { + Role string `json:"role"` + // Content is a string or a list of blocks. + Content any `json:"content"` +} + +type fixtureBlock struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Input map[string]string `json:"input,omitempty"` + ToolUseID string `json:"tool_use_id,omitempty"` + Content string `json:"content,omitempty"` + IsError bool `json:"is_error,omitempty"` +} + +func toolCallLine(t *testing.T, uuid, toolID, skillName, ts string) string { + t.Helper() + + return mustLine(t, fixtureRecord{ + Type: recordTypeAssistant, UUID: uuid, Timestamp: ts, + Message: fixtureMessage{Role: recordTypeAssistant, Content: []fixtureBlock{{ + Type: "tool_use", ID: toolID, Name: skillTool, + Input: map[string]string{"skill": skillName, "args": "private arguments"}, + }}}, + }) +} + +func toolResultLine(t *testing.T, uuid, parent, toolID, result string, isError bool, ts string) string { + t.Helper() + + return mustLine(t, fixtureRecord{ + Type: recordTypeUser, UUID: uuid, ParentUUID: parent, Timestamp: ts, + Message: fixtureMessage{Role: recordTypeUser, Content: []fixtureBlock{{ + Type: "tool_result", ToolUseID: toolID, Content: result, IsError: isError, + }}}, + }) +} + +func commandLine(t *testing.T, uuid, command, ts string) string { + t.Helper() + + return mustLine(t, fixtureRecord{ + Type: recordTypeUser, UUID: uuid, Timestamp: ts, + Message: fixtureMessage{ + Role: recordTypeUser, + Content: "" + command + "\n/" + command + "", + }, + }) +} + +// metaLine is the message that holds a loaded skill. toolID is empty for a +// skill that the user loaded. +func metaLine(t *testing.T, uuid, parent, toolID, dir, ts string) string { + t.Helper() + + return mustLine(t, fixtureRecord{ + Type: recordTypeUser, UUID: uuid, ParentUUID: parent, Timestamp: ts, IsMeta: true, SourceToolUseID: toolID, + Message: fixtureMessage{Role: recordTypeUser, Content: []fixtureBlock{{ + Type: "text", Text: "Base directory for this skill: " + dir + "\n\n# The skill\n\nBody.", + }}}, + }) +} + +// mustLine encodes a record as Claude Code does, without escaping < and >. +func mustLine(t *testing.T, record fixtureRecord) string { + t.Helper() + + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + require.NoError(t, enc.Encode(record)) + + return strings.TrimSuffix(b.String(), "\n") +} + +func writeLines(t *testing.T, path string, lines ...string) { + t.Helper() + + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600)) +} + +func TestSessionSkills(t *testing.T) { + const ( + steDir = "/home/me/.claude/skills/asd-ste100" + graphDir = "/home/me/.claude/skills/graphify" + planDir = "/home/me/.claude/plugins/cache/x/br/1.0.0/skills/plan" + ) + + modelStart := func(t *testing.T, n, skillName, dir, ts string) []string { + return []string{ + toolCallLine(t, "a"+n, "toolu_"+n, skillName, ts), + toolResultLine(t, "r"+n, "a"+n, "toolu_"+n, "Launching skill: "+skillName, false, ts), + metaLine(t, "m"+n, "r"+n, "toolu_"+n, dir, ts), + } + } + userStart := func(t *testing.T, n, command, dir, ts string) []string { + return []string{ + commandLine(t, "c"+n, command, ts), + metaLine(t, "m"+n, "c"+n, "", dir, ts), + } + } + join := func(groups ...[]string) []string { + var out []string + for _, g := range groups { + out = append(out, g...) + } + return out + } + + testCases := []struct { + name string + main []string + subagents map[string][]string + want []trace.SkillUse + }{ + { + name: "no skill", + main: []string{commandLine(t, "c1", "clear", tenOClockM)}, + want: []trace.SkillUse{}, + }, + { + name: "a model start", + main: modelStart(t, "1", steSkill, steDir, "2026-10-07T10:14:51.622Z"), + want: []trace.SkillUse{{Name: steSkill, Dir: steDir, FirstUsedAt: "2026-10-07T10:14:51Z", ByModel: 1}}, + }, + { + name: "a user start", + main: userStart(t, "1", graphSkill, graphDir, tenOClockM), + want: []trace.SkillUse{{Name: graphSkill, Dir: graphDir, FirstUsedAt: tenOClock, ByUser: 1}}, + }, + { + name: "both starts for one skill, in the order of first use", + main: join( + userStart(t, "1", graphSkill, graphDir, tenOClockM), + modelStart(t, "2", steSkill, steDir, "2026-10-07T10:05:00.000Z"), + modelStart(t, "3", graphSkill, graphDir, "2026-10-07T10:10:00.000Z"), + ), + want: []trace.SkillUse{ + {Name: graphSkill, Dir: graphDir, FirstUsedAt: tenOClock, ByModel: 1, ByUser: 1}, + {Name: steSkill, Dir: steDir, FirstUsedAt: "2026-10-07T10:05:00Z", ByModel: 1}, + }, + }, + { + name: "a failed call does not count", + main: []string{ + toolCallLine(t, "a1", "toolu_1", "missing", tenOClockM), + toolResultLine(t, "r1", "a1", "toolu_1", "Unknown skill: missing", true, tenOClockM), + }, + want: []trace.SkillUse{}, + }, + { + name: "built-in and custom commands do not count", + main: []string{ + commandLine(t, "c1", "clear", tenOClockM), + commandLine(t, "c2", "model", "2026-10-07T10:00:01.000Z"), + commandLine(t, "c3", "my-custom-command", "2026-10-07T10:00:02.000Z"), + }, + want: []trace.SkillUse{}, + }, + { + name: "one use recorded twice counts once", + main: join( + modelStart(t, "1", steSkill, steDir, tenOClockM), + []string{metaLine(t, "m1-again", "r1", "toolu_1", steDir, tenOClockM)}, + userStart(t, "2", graphSkill, graphDir, "2026-10-07T10:01:00.000Z"), + []string{metaLine(t, "m2-again", "c2", "", graphDir, "2026-10-07T10:01:00.000Z")}, + ), + want: []trace.SkillUse{ + {Name: steSkill, Dir: steDir, FirstUsedAt: tenOClock, ByModel: 1}, + {Name: graphSkill, Dir: graphDir, FirstUsedAt: "2026-10-07T10:01:00Z", ByUser: 1}, + }, + }, + { + name: "a plugin skill keeps its prefixed name", + main: modelStart(t, "1", "br:plan", planDir, tenOClockM), + want: []trace.SkillUse{{Name: "br:plan", Dir: planDir, FirstUsedAt: tenOClock, ByModel: 1}}, + }, + { + name: "subagent uses merge into the session entry", + main: modelStart(t, "1", steSkill, steDir, "2026-10-07T10:05:00.000Z"), + subagents: map[string][]string{ + "agent-a1.jsonl": modelStart(t, "s1", steSkill, steDir, "2026-10-07T10:01:00.000Z"), + "agent-a2.jsonl": modelStart(t, "s2", graphSkill, graphDir, "2026-10-07T10:09:00.000Z"), + }, + want: []trace.SkillUse{ + {Name: steSkill, Dir: steDir, FirstUsedAt: "2026-10-07T10:01:00Z", ByModel: 2, InSubagents: 1}, + {Name: graphSkill, Dir: graphDir, FirstUsedAt: "2026-10-07T10:09:00Z", ByModel: 1, InSubagents: 1}, + }, + }, + { + // A format that this version does not know gives no entry, never + // a failed push. + name: "an unknown format gives no entry", + main: []string{mustLine(t, fixtureRecord{ + Type: recordTypeUser, UUID: "x", Message: fixtureMessage{Role: recordTypeUser, Content: "Launching skill: " + steSkill}, + })}, + want: []trace.SkillUse{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + rawDir := t.TempDir() + writeLines(t, state.RawSessionPath(rawDir, skillsSessionID), tc.main...) + for name, lines := range tc.subagents { + writeLines(t, filepath.Join(state.RawSubagentDir(rawDir, skillsSessionID), name), lines...) + } + + opts := &trace.ParseOpts{SessionDir: rawDir, SessionID: skillsSessionID} + got, warnings, err := New().SessionSkills(opts) + require.NoError(t, err) + assert.Empty(t, warnings) + assert.Equal(t, tc.want, got) + + again, _, err := New().SessionSkills(opts) + require.NoError(t, err) + assert.Equal(t, got, again, "the same transcript gives the same output") + }) + } +} + +func TestNewSkillLoads(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + + projectDir := filepath.Join(home, ".claude", "projects", "-repo") + transcript := filepath.Join(projectDir, skillsSessionID+".jsonl") + subagent := filepath.Join(projectDir, skillsSessionID, "subagents", "agent-a1.jsonl") + + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + input := &trace.HookInput{SessionID: skillsSessionID, TranscriptPath: transcript} + + writeLines(t, transcript, + commandLine(t, "c1", graphSkill, tenOClockM), + metaLine(t, "m1", "c1", "", "/skills/graphify", tenOClockM), + ) + + assert.Equal(t, []string{"/skills/graphify"}, New().NewSkillLoads(store, input)) + assert.Empty(t, New().NewSkillLoads(store, input), "a line is read once") + + // The agent appends a use, and writes half of the next line. + f, err := os.OpenFile(transcript, os.O_APPEND|os.O_WRONLY, 0o600) + require.NoError(t, err) + _, err = f.WriteString(metaLine(t, "m2", "r2", "toolu_2", "/skills/asd-ste100", "2026-10-07T10:01:00.000Z") + "\n") + require.NoError(t, err) + half := metaLine(t, "m3", "r3", "toolu_3", "/skills/later", "2026-10-07T10:02:00.000Z") + _, err = f.WriteString(half[:len(half)/2]) + require.NoError(t, err) + require.NoError(t, f.Close()) + + writeLines(t, subagent, metaLine(t, "s1", "rs1", "toolu_s1", "/skills/in-subagent", "2026-10-07T10:01:30.000Z")) + + assert.ElementsMatch(t, []string{"/skills/asd-ste100", "/skills/in-subagent"}, New().NewSkillLoads(store, input), + "a line that the agent is still writing waits for the next hook") + + f, err = os.OpenFile(transcript, os.O_APPEND|os.O_WRONLY, 0o600) + require.NoError(t, err) + _, err = f.WriteString(half[len(half)/2:] + "\n") + require.NoError(t, err) + require.NoError(t, f.Close()) + + assert.Equal(t, []string{"/skills/later"}, New().NewSkillLoads(store, input)) +} diff --git a/app/cli/internal/trace/opencode/hooks.go b/app/cli/internal/trace/opencode/hooks.go index 2ac5f7247..baedebdc6 100644 --- a/app/cli/internal/trace/opencode/hooks.go +++ b/app/cli/internal/trace/opencode/hooks.go @@ -25,9 +25,11 @@ import ( "path/filepath" "regexp" "slices" + "strconv" "strings" "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" ) const ( @@ -63,6 +65,22 @@ const pluginTemplate = `import { spawn } from "node:child_process" const fileWritingTools = {{FileWritingToolsArray}} const commandTools = {{CommandToolsArray}} +const skillTool = {{SkillTool}} + +// skillDirFrom returns the folder of the skill that a skill tool call loaded: +// the dir field of the result metadata, or else the base directory line of +// the result text. Each OpenCode major gives the result in its own shape, so +// each candidate is tried in turn. +function skillDirFrom(...results: any[]): string { + for (const r of results) { + const dir = r?.metadata?.dir + if (typeof dir === "string" && dir) return dir + const text = typeof r === "string" ? r : typeof r?.output === "string" ? r.output : "" + const m = /^Base directory for this skill:\s*(.+)$/m.exec(text) + if (m) return m[1].trim() + } + return "" +} function filePathsFromArgs(args: any): string[] { if (args?.filePath) return [args.filePath] @@ -175,8 +193,14 @@ async function sessionEvent(directory: string, type: string, sessionID: string, {{SessionEndBlock}} } -async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any) { +async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any, skillDir = "") { const payload = { session_id: sessionID, hook_event_name: hookEventName, tool_name: tool } + if (tool === skillTool) { + // The skill is loaded after the call, and the result names its folder. + // The hook copies the folder for the evidence. + if (skillDir) await fire(directory, hook, { ...payload, skill_dir: skillDir }) + return + } if (commandTools.includes(tool)) { // The call ID pairs this hook with the other hook of the same call, so // overlapping commands keep their own snapshots. @@ -230,8 +254,8 @@ async function server({ directory, client }: any) { "tool.execute.before": async (input: any, output: any) => { await toolEvent(directory, "pre-tool-use", "tool.execute.before", input.sessionID, input.tool, input.callID, output.args) }, - "tool.execute.after": async (input: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args) + "tool.execute.after": async (input: any, output: any) => { + await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args, input.tool === skillTool ? skillDirFrom(output) : "") }, } } @@ -245,7 +269,7 @@ async function setup(ctx: any) { await toolEvent(directory, "pre-tool-use", "tool.execute.before", event.sessionID, event.tool, event.id, event.input) }) await ctx.tool.hook("execute.after", async (event: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input) + await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input, event.tool === skillTool ? skillDirFrom(event.output, event.result, event) : "") }) // resume: false stores the message without asking the model for an answer. @@ -342,6 +366,7 @@ func (p *Provider) writePluginFile(repoRoot string, includeSessionEnd bool) erro content := pluginTemplate content = strings.Replace(content, "{{FileWritingToolsArray}}", toolsArrayLiteral(fileWritingTools), 1) content = strings.Replace(content, "{{CommandToolsArray}}", toolsArrayLiteral(commandTools), 1) + content = strings.Replace(content, "{{SkillTool}}", strconv.Quote(skillTool), 1) if includeSessionEnd { content = strings.Replace(content, "{{SessionEndBlock}}", sessionDeletedHandler, 1) } else { @@ -400,6 +425,7 @@ func (p *Provider) ReadHookInput(r io.Reader) (*trace.HookInput, error) { ToolName string `json:"tool_name"` FilePath string `json:"file_path"` ToolUseID string `json:"tool_use_id"` + SkillDir string `json:"skill_dir"` } if err := json.Unmarshal(data, &raw); err != nil { return nil, err @@ -417,12 +443,18 @@ func (p *Provider) ReadHookInput(r io.Reader) (*trace.HookInput, error) { filePath = abs } + // The plugin takes the skill folder from the result text when the + // metadata has none, and there it is a file URL. A folder that is not + // absolute is dropped. + skillDir, _ := skill.ParseDir(raw.SkillDir) + return &trace.HookInput{ SessionID: raw.SessionID, HookEventName: raw.HookEventName, ToolName: raw.ToolName, FilePath: filePath, ToolUseID: raw.ToolUseID, + SkillDir: skillDir, }, nil } diff --git a/app/cli/internal/trace/opencode/hooks_test.go b/app/cli/internal/trace/opencode/hooks_test.go index e109ebc5d..7def22013 100644 --- a/app/cli/internal/trace/opencode/hooks_test.go +++ b/app/cli/internal/trace/opencode/hooks_test.go @@ -228,10 +228,56 @@ func TestPluginSendsShellCallID(t *testing.T) { assert.Contains(t, content, "{ ...payload, tool_use_id: callID }") // OpenCode 1.x. assert.Contains(t, content, `toolEvent(directory, "pre-tool-use", "tool.execute.before", input.sessionID, input.tool, input.callID, output.args)`) - assert.Contains(t, content, `toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args)`) + assert.Contains(t, content, `toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args, input.tool === skillTool ? skillDirFrom(output) : "")`) // OpenCode 2. assert.Contains(t, content, `toolEvent(directory, "pre-tool-use", "tool.execute.before", event.sessionID, event.tool, event.id, event.input)`) - assert.Contains(t, content, `toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input)`) + assert.Contains(t, content, `toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input, event.tool === skillTool ? skillDirFrom(event.output, event.result, event) : "")`) +} + +// TestPluginSendsSkillDir pins that the plugin sends the folder of a loaded +// skill to the post hook of the skill tool, and fires no hook for the skill +// tool otherwise. +func TestPluginSendsSkillDir(t *testing.T) { + repoRoot := t.TempDir() + require.NoError(t, New().InstallHooks(repoRoot)) + + data, err := os.ReadFile(filepath.Join(repoRoot, settingsFile)) + require.NoError(t, err) + content := string(data) + + assert.Contains(t, content, `const skillTool = "skill"`) + assert.Contains(t, content, "if (skillDir) await fire(directory, hook, { ...payload, skill_dir: skillDir })") +} + +func TestReadHookInputParsesSkillDir(t *testing.T) { + testCases := []struct { + name string + payload string + want string + }{ + { + name: "a path from the tool metadata", + payload: `{"session_id":"ses_1","hook_event_name":"tool.execute.after","tool_name":"skill","skill_dir":"/home/me/.config/opencode/skills/git-release"}`, + want: "/home/me/.config/opencode/skills/git-release", + }, + { + name: "a file URL from the tool output", + payload: `{"session_id":"ses_1","hook_event_name":"tool.execute.after","tool_name":"skill","skill_dir":"file:///home/me/repo/.opencode/skills/git-release"}`, + want: "/home/me/repo/.opencode/skills/git-release", + }, + { + name: "a relative folder is dropped", + payload: `{"session_id":"ses_1","hook_event_name":"tool.execute.after","tool_name":"skill","skill_dir":"skills/git-release"}`, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + input, err := New().ReadHookInput(bytes.NewBufferString(tc.payload)) + require.NoError(t, err) + assert.Equal(t, tc.want, input.SkillDir) + }) + } } // The plugin sends opencode's callID as tool_use_id on shell hooks, so that diff --git a/app/cli/internal/trace/opencode/parse.go b/app/cli/internal/trace/opencode/parse.go index 7a2c582da..fe6cce86a 100644 --- a/app/cli/internal/trace/opencode/parse.go +++ b/app/cli/internal/trace/opencode/parse.go @@ -36,10 +36,24 @@ const ( partTool = "tool" ) +// toolStatusCompleted is the status of a tool call that ran to the end. +const toolStatusCompleted = "completed" + // parseExport reads and parses the opencode export JSON at path into // structured evidence. It reads both the OpenCode 1.x and the OpenCode 2 // export format. Unknown fields are ignored. func parseExport(path string) (*aicodingsession.Evidence, error) { + export, err := readExport(path) + if err != nil { + return nil, err + } + + return buildEvidence(export), nil +} + +// readExport reads an export file in either format, in the OpenCode 1.x +// shape. +func readExport(path string) (*exportData, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("read export file: %w", err) @@ -51,7 +65,7 @@ func parseExport(path string) (*aicodingsession.Evidence, error) { return nil, fmt.Errorf("parse export JSON: %w", err) } - return buildEvidence(export.toExportData()), nil + return export.toExportData(), nil } var export exportData @@ -59,7 +73,7 @@ func parseExport(path string) (*aicodingsession.Evidence, error) { return nil, fmt.Errorf("parse export JSON: %w", err) } - return buildEvidence(&export), nil + return &export, nil } // isExportV2 reports whether data is an OpenCode 2 export. Only the OpenCode 2 @@ -166,7 +180,7 @@ func buildEvidence(export *exportData) *aicodingsession.Evidence { // Count completed tool invocations from tool parts. for _, part := range msg.Parts { - if part.Type == partTool && part.State != nil && part.State.Status == "completed" { + if part.Type == partTool && part.State != nil && part.State.Status == toolStatusCompleted { name := part.Tool if name == "" { name = unknownValue @@ -309,7 +323,7 @@ func buildRawSessionEntries(messages []messageEntry) []json.RawMessage { blocks = append(blocks, trace.RawSessionTextBlock{Type: partText, Text: p.Text}) } case partTool: - if p.State == nil || p.State.Status != "completed" { + if p.State == nil || p.State.Status != toolStatusCompleted { continue } toolName := p.Tool diff --git a/app/cli/internal/trace/opencode/skills.go b/app/cli/internal/trace/opencode/skills.go new file mode 100644 index 000000000..9d3c4d670 --- /dev/null +++ b/app/cli/internal/trace/opencode/skills.go @@ -0,0 +1,143 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package opencode + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" +) + +// Compile-time check that Provider records skills. +var _ trace.SkillTracker = (*Provider)(nil) + +// skillTool is the native OpenCode tool that loads a skill. Its one input is +// the name of the skill. +const skillTool = "skill" + +// NewSkillLoads returns the skill folder that the plugin put in the payload +// of the hook after a skill tool call. OpenCode tells the folder in the tool +// result, so no transcript is read. +func (p *Provider) NewSkillLoads(_ *state.Store, input *trace.HookInput) []string { + if input.ToolName != skillTool || input.SkillDir == "" { + return nil + } + + return []string{input.SkillDir} +} + +// SessionSkills returns the skills that the session used, from the copied +// export. A model start is a completed call to the skill tool. OpenCode has +// no slash command that starts a skill, so the user count stays at zero. A +// subagent runs in a child session, which has its own export, so the count of +// subagent uses stays at zero too. +func (p *Provider) SessionSkills(opts *trace.ParseOpts) ([]trace.SkillUse, []string, error) { + path, err := state.FindRawSessionFile(opts.SessionDir, opts.SessionID) + if err != nil { + return nil, nil, fmt.Errorf("locate opencode export: %w", err) + } + + export, err := readExport(path) + if err != nil { + return nil, nil, err + } + + return exportSkills(export), nil, nil +} + +// exportSkills counts the completed skill tool calls of an export. +func exportSkills(export *exportData) []trace.SkillUse { + uses := make(map[string]*trace.SkillUse) + for _, msg := range export.Messages { + at := msgTimeRFC3339(msg.Info.Time) + + for _, part := range msg.Parts { + if part.Type != partTool || part.Tool != skillTool || part.State == nil || part.State.Status != toolStatusCompleted { + continue + } + + var input struct { + Name string `json:"name"` + } + if json.Unmarshal(part.State.Input, &input) != nil || strings.TrimSpace(input.Name) == "" { + continue + } + name := strings.TrimSpace(input.Name) + + use, ok := uses[name] + if !ok { + use = &trace.SkillUse{Name: name} + uses[name] = use + } + if use.Dir == "" { + use.Dir = toolSkillDir(part.State) + } + use.Record(at, true, false) + } + } + + return trace.SortedSkillUses(uses) +} + +// toolSkillDir returns the skill folder of a skill tool call: from the tool +// metadata, or else from the base directory line of the tool output. +func toolSkillDir(st *toolState) string { + if st.Metadata != nil && filepath.IsAbs(st.Metadata.Dir) { + return filepath.Clean(st.Metadata.Dir) + } + + var output string + if json.Unmarshal(st.Output, &output) != nil { + return "" + } + + dir, _ := skill.BaseDir(output) + + return dir +} + +// SkillRoots returns the OpenCode folders that tell where a skill came from. +// OpenCode reads project skills from .opencode/skills, .claude/skills and +// .agents/skills in the repository, and user skills from the same folders in +// the user's configuration. +func (p *Provider) SkillRoots(repoRoot string) skill.Roots { + roots := skill.Roots{Project: []string{repoRoot}} + + home, err := os.UserHomeDir() + if err != nil { + return roots + } + + configHome := os.Getenv("XDG_CONFIG_HOME") + if configHome == "" { + configHome = filepath.Join(home, ".config") + } + + roots.User = []string{ + filepath.Join(configHome, "opencode", "skills"), + filepath.Join(home, ".opencode", "skills"), + filepath.Join(home, ".claude", "skills"), + filepath.Join(home, ".agents", "skills"), + } + + return roots +} diff --git a/app/cli/internal/trace/opencode/skills_test.go b/app/cli/internal/trace/opencode/skills_test.go new file mode 100644 index 000000000..430c1c73c --- /dev/null +++ b/app/cli/internal/trace/opencode/skills_test.go @@ -0,0 +1,114 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package opencode + +import ( + "os" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSessionSkills(t *testing.T) { + const sessionID = "ses_skills" + + // skillPart is a skill tool call as OpenCode 1.x exports it. The tool + // output holds the folder as a file URL, and the metadata as a path. + skillPart := func(name, status, metadata, output string) string { + return `{"type":"tool","tool":"skill","state":{"status":"` + status + `","input":{"name":"` + name + `"}` + + metadata + `,"output":` + output + `}}` + } + message := func(created string, parts ...string) string { + joined := "" + for i, p := range parts { + if i > 0 { + joined += "," + } + joined += p + } + return `{"info":{"role":"assistant","id":"msg","time":{"created":` + created + `}},"parts":[` + joined + `]}` + } + export := func(messages ...string) string { + joined := "" + for i, m := range messages { + if i > 0 { + joined += "," + } + joined += m + } + return `{"info":{"id":"` + sessionID + `","directory":"/repo","time":{"created":1},"version":"1.18.0"},"messages":[` + joined + `]}` + } + + testCases := []struct { + name string + export string + want []trace.SkillUse + }{ + { + name: "no skill", + export: export(message("1791367200000", `{"type":"text","text":"hello"}`)), + want: []trace.SkillUse{}, + }, + { + name: "completed calls count, a failed call does not", + export: export( + message("1791367200000", skillPart("git-release", "completed", `,"metadata":{"dir":"/home/me/.config/opencode/skills/git-release"}`, `""`)), + message("1791367260000", skillPart("missing", "error", "", `"Skill not found"`)), + message("1791367320000", skillPart("git-release", "completed", "", `"Base directory for this skill: file:///home/me/.config/opencode/skills/git-release\n"`)), + message("1791367380000", skillPart("pdf", "completed", "", `"Base directory for this skill: file:///repo/.opencode/skills/pdf\n"`)), + ), + want: []trace.SkillUse{ + {Name: "git-release", Dir: "/home/me/.config/opencode/skills/git-release", FirstUsedAt: "2026-10-07T10:00:00Z", ByModel: 2}, + {Name: "pdf", Dir: "/repo/.opencode/skills/pdf", FirstUsedAt: "2026-10-07T10:03:00Z", ByModel: 1}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + rawDir := t.TempDir() + require.NoError(t, os.WriteFile(state.RawSessionPath(rawDir, sessionID), []byte(tc.export), 0o600)) + + got, warnings, err := New().SessionSkills(&trace.ParseOpts{SessionDir: rawDir, SessionID: sessionID}) + require.NoError(t, err) + assert.Empty(t, warnings) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestNewSkillLoads(t *testing.T) { + const pdfDir = "/skills/pdf" + + testCases := []struct { + name string + input trace.HookInput + want []string + }{ + {name: "a skill call with a folder", input: trace.HookInput{ToolName: "skill", SkillDir: pdfDir}, want: []string{pdfDir}}, + {name: "a skill call without a folder", input: trace.HookInput{ToolName: "skill"}}, + {name: "another tool", input: trace.HookInput{ToolName: "bash", SkillDir: pdfDir}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, New().NewSkillLoads(nil, &tc.input)) + }) + } +} diff --git a/app/cli/internal/trace/opencode/testdata/plugin_full.ts b/app/cli/internal/trace/opencode/testdata/plugin_full.ts index f23a9d010..2eb4e23d2 100644 --- a/app/cli/internal/trace/opencode/testdata/plugin_full.ts +++ b/app/cli/internal/trace/opencode/testdata/plugin_full.ts @@ -2,6 +2,22 @@ import { spawn } from "node:child_process" const fileWritingTools = ["edit","write","apply_patch","patch"] const commandTools = ["bash","shell"] +const skillTool = "skill" + +// skillDirFrom returns the folder of the skill that a skill tool call loaded: +// the dir field of the result metadata, or else the base directory line of +// the result text. Each OpenCode major gives the result in its own shape, so +// each candidate is tried in turn. +function skillDirFrom(...results: any[]): string { + for (const r of results) { + const dir = r?.metadata?.dir + if (typeof dir === "string" && dir) return dir + const text = typeof r === "string" ? r : typeof r?.output === "string" ? r.output : "" + const m = /^Base directory for this skill:\s*(.+)$/m.exec(text) + if (m) return m[1].trim() + } + return "" +} function filePathsFromArgs(args: any): string[] { if (args?.filePath) return [args.filePath] @@ -117,8 +133,14 @@ async function sessionEvent(directory: string, type: string, sessionID: string, } } -async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any) { +async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any, skillDir = "") { const payload = { session_id: sessionID, hook_event_name: hookEventName, tool_name: tool } + if (tool === skillTool) { + // The skill is loaded after the call, and the result names its folder. + // The hook copies the folder for the evidence. + if (skillDir) await fire(directory, hook, { ...payload, skill_dir: skillDir }) + return + } if (commandTools.includes(tool)) { // The call ID pairs this hook with the other hook of the same call, so // overlapping commands keep their own snapshots. @@ -172,8 +194,8 @@ async function server({ directory, client }: any) { "tool.execute.before": async (input: any, output: any) => { await toolEvent(directory, "pre-tool-use", "tool.execute.before", input.sessionID, input.tool, input.callID, output.args) }, - "tool.execute.after": async (input: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args) + "tool.execute.after": async (input: any, output: any) => { + await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args, input.tool === skillTool ? skillDirFrom(output) : "") }, } } @@ -187,7 +209,7 @@ async function setup(ctx: any) { await toolEvent(directory, "pre-tool-use", "tool.execute.before", event.sessionID, event.tool, event.id, event.input) }) await ctx.tool.hook("execute.after", async (event: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input) + await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input, event.tool === skillTool ? skillDirFrom(event.output, event.result, event) : "") }) // resume: false stores the message without asking the model for an answer. diff --git a/app/cli/internal/trace/opencode/testdata/plugin_tracerun.ts b/app/cli/internal/trace/opencode/testdata/plugin_tracerun.ts index 1a20fd518..43e717716 100644 --- a/app/cli/internal/trace/opencode/testdata/plugin_tracerun.ts +++ b/app/cli/internal/trace/opencode/testdata/plugin_tracerun.ts @@ -2,6 +2,22 @@ import { spawn } from "node:child_process" const fileWritingTools = ["edit","write","apply_patch","patch"] const commandTools = ["bash","shell"] +const skillTool = "skill" + +// skillDirFrom returns the folder of the skill that a skill tool call loaded: +// the dir field of the result metadata, or else the base directory line of +// the result text. Each OpenCode major gives the result in its own shape, so +// each candidate is tried in turn. +function skillDirFrom(...results: any[]): string { + for (const r of results) { + const dir = r?.metadata?.dir + if (typeof dir === "string" && dir) return dir + const text = typeof r === "string" ? r : typeof r?.output === "string" ? r.output : "" + const m = /^Base directory for this skill:\s*(.+)$/m.exec(text) + if (m) return m[1].trim() + } + return "" +} function filePathsFromArgs(args: any): string[] { if (args?.filePath) return [args.filePath] @@ -113,8 +129,14 @@ async function sessionEvent(directory: string, type: string, sessionID: string, } } -async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any) { +async function toolEvent(directory: string, hook: string, hookEventName: string, sessionID: string, tool: string, callID: string, args: any, skillDir = "") { const payload = { session_id: sessionID, hook_event_name: hookEventName, tool_name: tool } + if (tool === skillTool) { + // The skill is loaded after the call, and the result names its folder. + // The hook copies the folder for the evidence. + if (skillDir) await fire(directory, hook, { ...payload, skill_dir: skillDir }) + return + } if (commandTools.includes(tool)) { // The call ID pairs this hook with the other hook of the same call, so // overlapping commands keep their own snapshots. @@ -168,8 +190,8 @@ async function server({ directory, client }: any) { "tool.execute.before": async (input: any, output: any) => { await toolEvent(directory, "pre-tool-use", "tool.execute.before", input.sessionID, input.tool, input.callID, output.args) }, - "tool.execute.after": async (input: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args) + "tool.execute.after": async (input: any, output: any) => { + await toolEvent(directory, "post-tool-use", "tool.execute.after", input.sessionID, input.tool, input.callID, input.args, input.tool === skillTool ? skillDirFrom(output) : "") }, } } @@ -183,7 +205,7 @@ async function setup(ctx: any) { await toolEvent(directory, "pre-tool-use", "tool.execute.before", event.sessionID, event.tool, event.id, event.input) }) await ctx.tool.hook("execute.after", async (event: any) => { - await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input) + await toolEvent(directory, "post-tool-use", "tool.execute.after", event.sessionID, event.tool, event.id, event.input, event.tool === skillTool ? skillDirFrom(event.output, event.result, event) : "") }) // resume: false stores the message without asking the model for an answer. diff --git a/app/cli/internal/trace/opencode/types.go b/app/cli/internal/trace/opencode/types.go index b73daaa42..638e66850 100644 --- a/app/cli/internal/trace/opencode/types.go +++ b/app/cli/internal/trace/opencode/types.go @@ -115,6 +115,14 @@ type toolState struct { Status string `json:"status"` Input json.RawMessage `json:"input,omitempty"` Output json.RawMessage `json:"output,omitempty"` + // Metadata is read for the skill tool only, which puts the skill folder + // in it. + Metadata *toolMetadata `json:"metadata,omitempty"` +} + +// toolMetadata holds the fields of the tool metadata that the provider reads. +type toolMetadata struct { + Dir string `json:"dir,omitempty"` } // exportDataV2 is the top-level shape of `opencode session export`, the diff --git a/app/cli/internal/trace/provider.go b/app/cli/internal/trace/provider.go index 114cd9b56..53452d9ef 100644 --- a/app/cli/internal/trace/provider.go +++ b/app/cli/internal/trace/provider.go @@ -19,7 +19,9 @@ import ( "context" "errors" "io" + "sort" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" ) @@ -158,6 +160,94 @@ type Provider interface { AnnounceToUser(msg string) error } +// SkillTracker is implemented by a provider that records the skills that a +// session used. A provider without it records no skills, and for its sessions +// no skill entry means "not recorded", not "no skill used". +type SkillTracker interface { + // NewSkillLoads returns the folders of the skills that the session loaded + // since the last call for the session, as far as the hook can tell. The + // caller copies each folder. It never fails: a hook that cannot tell + // returns nothing, and the push copies the folder later. + NewSkillLoads(store *state.Store, input *HookInput) []string + + // SessionSkills returns the skills that the session used, from the + // session data under opts.SessionDir, in the order of their first use. + // The warnings are for uses that could not be read. + SessionSkills(opts *ParseOpts) ([]SkillUse, []string, error) + + // SkillRoots returns the folders that tell where a skill came from, for a + // session in repoRoot. + SkillRoots(repoRoot string) skill.Roots +} + +// SkillUse is one skill that a session used, with the counts of its uses. +type SkillUse struct { + // Name is the name that the agent used for the skill. + Name string + // Dir is the folder that the agent loaded the skill from, or "" when the + // session data does not tell. + Dir string + // FirstUsedAt is the time of the first use, RFC3339. + FirstUsedAt string + // ByModel counts the uses that the model started with a tool call. + ByModel int + // ByUser counts the uses that the user started with a slash command. + ByUser int + // InSubagents counts the uses, of either start, that came from subagents. + InSubagents int +} + +// Record adds one use at time at, RFC3339, to the counts. The first use is +// the earliest time recorded, whatever the order of the calls. +func (u *SkillUse) Record(at string, byModel, subagent bool) { + u.noteTime(at) + if byModel { + u.ByModel++ + } else { + u.ByUser++ + } + if subagent { + u.InSubagents++ + } +} + +// Merge adds the uses of other, of the same skill, to u. +func (u *SkillUse) Merge(other SkillUse) { + u.noteTime(other.FirstUsedAt) + u.ByModel += other.ByModel + u.ByUser += other.ByUser + u.InSubagents += other.InSubagents + if u.Dir == "" { + u.Dir = other.Dir + } +} + +func (u *SkillUse) noteTime(at string) { + if at != "" && (u.FirstUsedAt == "" || at < u.FirstUsedAt) { + u.FirstUsedAt = at + } +} + +// SortedSkillUses returns the uses in the order of their first use. The name +// breaks ties, so that the same session data always gives the same order: +// the limit on skill entries keeps the first ones. +func SortedSkillUses(uses map[string]*SkillUse) []SkillUse { + out := make([]SkillUse, 0, len(uses)) + for _, use := range uses { + out = append(out, *use) + } + + sort.Slice(out, func(i, j int) bool { + if out[i].FirstUsedAt != out[j].FirstUsedAt { + return out[i].FirstUsedAt < out[j].FirstUsedAt + } + + return out[i].Name < out[j].Name + }) + + return out +} + // HookInput represents parsed hook invocation data from an AI agent. type HookInput struct { // SessionID is the agent-assigned identifier for the session. @@ -199,6 +289,9 @@ type HookInput struct { // only emit post-edit events (e.g., Cursor's afterFileEdit) populate it so // consumers can reconstruct the "before" content via reverse application. Edits []HookEdit `json:"-"` + // SkillDir is the folder of the skill that the tool call loaded, when the + // hook payload gives it. Empty otherwise. + SkillDir string `json:"-"` // ToolFailed reports that the hook fires after a tool call that failed // (Claude's PostToolUseFailure). The tool can still have changed files, // so its changes are recorded. But the agent expects a different hook diff --git a/app/cli/internal/trace/skill/skill.go b/app/cli/internal/trace/skill/skill.go new file mode 100644 index 000000000..b63957fcb --- /dev/null +++ b/app/cli/internal/trace/skill/skill.go @@ -0,0 +1,552 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package skill keeps a copy of each skill folder that a coding session used, +// and turns the copy into the materials that the evidence holds. +// +// The copy is made when the session uses the skill, not when the session is +// pushed, so the evidence holds the skill that ran also when its files change +// before the push. A copy is never replaced: a later use that finds different +// content only marks the copy as changed. +// +// A copy holds the files of the folder as they are. Secrets are removed only +// when a package is made from the copy, by the same redaction as the other +// spec sources. +package skill + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "time" + "unicode/utf8" +) + +const ( + // DefinitionFile is the file that defines a skill. A folder without it is + // not a skill, and is never read. + DefinitionFile = "SKILL.md" + + // MaxPackageSize is the largest archive that is uploaded. A larger skill + // keeps its definition and its counts, and loses its package only. + MaxPackageSize = 5 << 20 + + // maxCopySize bounds the bytes that a hook copies. The archive is + // compressed, so a folder can be larger than MaxPackageSize and still + // give a package under the limit. A folder over this bound cannot. + maxCopySize = 50 << 20 + + // filesDir holds the copied files inside a copy. + filesDir = "files" + // infoFile records what the copy was made from. + infoFile = "info.json" + // changedFile marks a copy whose folder changed after the first use. + changedFile = "changed" + // packageDir holds the redacted definition and archive, made at the first + // push that needs them. + packageDir = "package" + packageArchive = "package.tar.gz" + // oversizeFile marks a copy whose archive is over MaxPackageSize. + oversizeFile = "oversize" + + // baseDirMarker opens the line that tells where an agent loaded a skill + // from. Claude Code writes it with a path, and OpenCode with a file URL. + baseDirMarker = "Base directory for this skill:" +) + +// vcsDirs are the version-control folders that a package leaves out. +var vcsDirs = []string{".git", ".hg", ".svn", ".bzr", ".jj"} + +// ErrNotSkill is returned for a folder that holds no SKILL.md file. +var ErrNotSkill = errors.New("the folder holds no " + DefinitionFile) + +// Info describes a copy. +type Info struct { + // Digest identifies the content of the folder when it was copied. The + // same content gives the same digest in every session. + Digest string `json:"digest"` + // Truncated reports a folder over the copy bound. Only its definition was + // copied, and it gets no package. + Truncated bool `json:"truncated,omitempty"` + // SkippedLinks counts the links that point out of the folder, or to a + // folder, and so were left out. + SkippedLinks int `json:"skipped_links,omitempty"` + // Changed reports that a later use found different content in the folder. + // The copy keeps the content of the first use. + Changed bool `json:"-"` +} + +// file is one file of a skill folder that goes into a copy. +type file struct { + // rel is the path inside the folder, with forward slashes. + rel string + // path is where the content is read from. For a link inside the folder, + // it is the target of the link. + path string + // executable keeps the one mode bit a script needs. + executable bool + size int64 +} + +// Copy copies the skill folder src into dst, which must not exist yet, and +// returns what it copied. When another hook renamed its copy into dst first, +// Copy returns that copy unchanged. +func Copy(src, dst string) (Info, error) { + files, skipped, err := listFiles(src) + if err != nil { + return Info{}, err + } + + info := Info{SkippedLinks: skipped} + + var total int64 + for _, f := range files { + total += f.size + } + // A folder over the bound keeps only its definition. Its digest is still + // that of the whole folder, for the comparison at a later use. + info.Truncated = total > maxCopySize + + if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { + return Info{}, fmt.Errorf("create skill copy directory: %w", err) + } + + // The copy is made in a sibling folder and renamed into place, so that a + // reader never sees half of it and two hooks cannot mix their files. + tmp, err := os.MkdirTemp(filepath.Dir(dst), "."+filepath.Base(dst)+"-*") + if err != nil { + return Info{}, fmt.Errorf("create skill copy directory: %w", err) + } + defer func() { _ = os.RemoveAll(tmp) }() + + h := sha256.New() + for _, f := range files { + content, err := os.ReadFile(f.path) + if err != nil { + return Info{}, fmt.Errorf("read skill file %q: %w", f.rel, err) + } + + writeDigestEntry(h, f.rel, f.executable, content) + if info.Truncated && f.rel != DefinitionFile { + continue + } + + mode := os.FileMode(0o600) + if f.executable { + mode = 0o700 + } + + // The path comes from a walk of the folder, so it cannot leave the + // copy. The check keeps that true if the walk ever changes. + target := filepath.Join(tmp, filesDir, filepath.FromSlash(f.rel)) + if !Within(target, filepath.Join(tmp, filesDir)) { + return Info{}, fmt.Errorf("skill file %q is outside the skill folder", f.rel) + } + if err := os.MkdirAll(filepath.Dir(target), 0o700); err != nil { + return Info{}, fmt.Errorf("copy skill file %q: %w", f.rel, err) + } + //nolint:gosec // target is checked above to stay inside the copy + if err := os.WriteFile(target, content, mode); err != nil { + return Info{}, fmt.Errorf("copy skill file %q: %w", f.rel, err) + } + } + + info.Digest = "sha256:" + hex.EncodeToString(h.Sum(nil)) + + data, err := json.Marshal(info) + if err != nil { + return Info{}, fmt.Errorf("encode skill copy info: %w", err) + } + if err := os.WriteFile(filepath.Join(tmp, infoFile), data, 0o600); err != nil { + return Info{}, fmt.Errorf("write skill copy info: %w", err) + } + + if err := os.Rename(tmp, dst); err != nil { + // Another hook renamed its copy into place first. Its copy is of the + // same use, so it stands. + if existing, loadErr := Load(dst); loadErr == nil { + return existing, nil + } + + return Info{}, fmt.Errorf("store skill copy: %w", err) + } + + return info, nil +} + +// Load returns the description of an existing copy. +func Load(dst string) (Info, error) { + data, err := os.ReadFile(filepath.Join(dst, infoFile)) + if err != nil { + return Info{}, err + } + + var info Info + if err := json.Unmarshal(data, &info); err != nil { + return Info{}, fmt.Errorf("parse skill copy info: %w", err) + } + + if _, err := os.Stat(filepath.Join(dst, changedFile)); err == nil { + info.Changed = true + } + + return info, nil +} + +// MarkChanged records that a later use found content in the folder that is +// not the content of the copy. +func MarkChanged(dst string) error { + return os.WriteFile(filepath.Join(dst, changedFile), nil, 0o600) +} + +// ContentDigest returns the digest of the content of a skill folder, as Copy +// computes it, without copying anything. +func ContentDigest(src string) (string, error) { + files, _, err := listFiles(src) + if err != nil { + return "", err + } + + return contentDigest(files) +} + +func contentDigest(files []file) (string, error) { + h := sha256.New() + for _, f := range files { + content, err := os.ReadFile(f.path) + if err != nil { + return "", fmt.Errorf("read skill file %q: %w", f.rel, err) + } + writeDigestEntry(h, f.rel, f.executable, content) + } + + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + +// writeDigestEntry adds one file to a content digest. Each field has its +// length first, so that no two lists of files give the same input. +func writeDigestEntry(w io.Writer, rel string, executable bool, content []byte) { + sum := sha256.Sum256(content) + _, _ = fmt.Fprintf(w, "%d:%s:%t:%x\n", len(rel), rel, executable, sum) +} + +// listFiles returns the files of a skill folder that go into a copy, sorted by +// path, and the number of links that it left out. +// +// The folder itself can be a link: an agent often installs a skill as a link +// to a checkout. The links inside it are followed only to a file inside the +// folder. A link out of the folder, or to a folder, is left out, so a skill +// cannot pull other files of the user into the evidence, and a loop of links +// cannot stop the walk. +func listFiles(src string) (files []file, skipped int, err error) { + root, err := filepath.EvalSymlinks(src) + if err != nil { + return nil, 0, fmt.Errorf("resolve skill folder: %w", err) + } + + if info, err := os.Stat(filepath.Join(root, DefinitionFile)); err != nil || !info.Mode().IsRegular() { + return nil, 0, ErrNotSkill + } + + err = filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + if path != root && slices.Contains(vcsDirs, d.Name()) { + return filepath.SkipDir + } + + return nil + } + + // A git worktree or submodule keeps a .git file, not a folder. + if slices.Contains(vcsDirs, d.Name()) { + return nil + } + + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + + source := path + if d.Type()&fs.ModeSymlink != 0 { + target, ok := linkInside(root, path) + if !ok { + skipped++ + return nil + } + source = target + } + + info, err := os.Stat(source) + if err != nil { + return err + } + if !info.Mode().IsRegular() { + if info.IsDir() { + skipped++ + } + + return nil + } + + files = append(files, file{ + rel: filepath.ToSlash(rel), + path: source, + executable: info.Mode().Perm()&0o111 != 0, + size: info.Size(), + }) + + return nil + }) + if err != nil { + return nil, 0, fmt.Errorf("read skill folder: %w", err) + } + + sort.Slice(files, func(i, j int) bool { return files[i].rel < files[j].rel }) + + return files, skipped, nil +} + +// linkInside resolves a link and reports whether its target is a file or a +// folder inside root. +func linkInside(root, link string) (string, bool) { + target, err := filepath.EvalSymlinks(link) + if err != nil { + return "", false + } + + return target, Within(target, root) +} + +// Within reports whether path is root or a path below it. +func Within(path, root string) bool { + rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(path)) + if err != nil { + return false + } + + return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))) +} + +// Package is what the evidence holds for one skill. +type Package struct { + // Definition is the redacted SKILL.md file. + Definition []byte + // Archive is a reproducible gzip-compressed tar archive of the redacted + // folder, or nil when it is over MaxPackageSize or the copy was + // truncated. + Archive []byte + // Oversize reports that the archive was left out for its size. + Oversize bool +} + +// Redactor removes secrets from the text of one file. +type Redactor func(doc []byte) ([]byte, error) + +// MakePackage returns the package of a copy. The first call redacts each text +// file and keeps the result in the copy, so a later push of the session does +// not redact an unchanged skill again. Binary files are stored as they are. +// +// Redaction fails closed: a file that could not be scanned fails the package, +// and nothing of the skill is uploaded. +func MakePackage(dst string, redact Redactor) (Package, error) { + info, err := Load(dst) + if err != nil { + return Package{}, fmt.Errorf("read skill copy: %w", err) + } + + cache := filepath.Join(dst, packageDir) + if pkg, ok := loadPackage(cache); ok { + return pkg, nil + } + + files, _, err := listFiles(filepath.Join(dst, filesDir)) + if err != nil { + return Package{}, err + } + + var pkg Package + var buf bytes.Buffer + + gz, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) + if err != nil { + return Package{}, err + } + tw := tar.NewWriter(gz) + + for _, f := range files { + content, err := os.ReadFile(f.path) + if err != nil { + return Package{}, fmt.Errorf("read skill file %q: %w", f.rel, err) + } + + if utf8.Valid(content) { + if content, err = redact(content); err != nil { + return Package{}, fmt.Errorf("scanning %q for secrets: %w", f.rel, err) + } + } + + if f.rel == DefinitionFile { + pkg.Definition = content + } + + if err := writeTarEntry(tw, f, content); err != nil { + return Package{}, err + } + } + + if err := tw.Close(); err != nil { + return Package{}, fmt.Errorf("close skill archive: %w", err) + } + if err := gz.Close(); err != nil { + return Package{}, fmt.Errorf("close skill archive: %w", err) + } + + if info.Truncated || buf.Len() > MaxPackageSize { + pkg.Oversize = true + } else { + pkg.Archive = buf.Bytes() + } + + // A cache we cannot write costs the next push a redaction, never this + // push its package. + _ = storePackage(cache, pkg) + + return pkg, nil +} + +// writeTarEntry adds one file with fixed metadata, so the same content gives +// the same archive on every machine: no time, no owner, and a mode that keeps +// only the executable bit. +func writeTarEntry(tw *tar.Writer, f file, content []byte) error { + mode := int64(0o644) + if f.executable { + mode = 0o755 + } + + hdr := &tar.Header{ + Typeflag: tar.TypeReg, + Name: f.rel, + Mode: mode, + Size: int64(len(content)), + ModTime: time.Unix(0, 0).UTC(), + Format: tar.FormatPAX, + } + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("write skill archive: %w", err) + } + if _, err := tw.Write(content); err != nil { + return fmt.Errorf("write skill archive: %w", err) + } + + return nil +} + +func loadPackage(cache string) (Package, bool) { + definition, err := os.ReadFile(filepath.Join(cache, DefinitionFile)) + if err != nil { + return Package{}, false + } + + pkg := Package{Definition: definition} + if _, err := os.Stat(filepath.Join(cache, oversizeFile)); err == nil { + pkg.Oversize = true + return pkg, true + } + + archive, err := os.ReadFile(filepath.Join(cache, packageArchive)) + if err != nil { + return Package{}, false + } + pkg.Archive = archive + + return pkg, true +} + +// storePackage writes the package next to the copy. The definition goes last, +// because loadPackage takes its presence to mean that the rest is there. +func storePackage(cache string, pkg Package) error { + if err := os.MkdirAll(cache, 0o700); err != nil { + return err + } + + if pkg.Oversize { + if err := os.WriteFile(filepath.Join(cache, oversizeFile), nil, 0o600); err != nil { + return err + } + } else if err := os.WriteFile(filepath.Join(cache, packageArchive), pkg.Archive, 0o600); err != nil { + return err + } + + return os.WriteFile(filepath.Join(cache, DefinitionFile), pkg.Definition, 0o600) +} + +// BaseDir returns the skill folder that a "Base directory for this skill:" +// line in text names. Claude Code writes a path, and OpenCode a file URL. Only +// an absolute path is accepted. +func BaseDir(text string) (string, bool) { + for line := range strings.SplitSeq(text, "\n") { + rest, ok := strings.CutPrefix(strings.TrimSpace(line), baseDirMarker) + if !ok { + continue + } + + return ParseDir(rest) + } + + return "", false +} + +// ParseDir reads a skill folder as an agent gives it: a path or a file URL. +// Only an absolute path is accepted. +func ParseDir(s string) (string, bool) { + dir := strings.TrimSpace(s) + if strings.HasPrefix(dir, "file://") { + u, err := url.Parse(dir) + if err != nil { + return "", false + } + dir = u.Path + } + + if dir == "" || !filepath.IsAbs(dir) { + return "", false + } + + return filepath.Clean(dir), true +} + +// HasBaseDir reports whether text can hold a base directory line. It is a +// cheap test that a scan of a large transcript runs before it decodes a line. +func HasBaseDir(text []byte) bool { + return bytes.Contains(text, []byte(baseDirMarker)) +} diff --git a/app/cli/internal/trace/skill/skill_test.go b/app/cli/internal/trace/skill/skill_test.go new file mode 100644 index 000000000..31a1f25d4 --- /dev/null +++ b/app/cli/internal/trace/skill/skill_test.go @@ -0,0 +1,287 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skill + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/rand" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeSkill makes a skill folder from a map of relative paths to contents. +func writeSkill(t *testing.T, dir string, files map[string]string) string { + t.Helper() + + for rel, content := range files { + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + } + + return dir +} + +// skillDoc is the definition of most test skills, and insideDoc the content +// of a file inside a test skill. +const ( + skillDoc = "# Skill" + insideDoc = "inside" +) + +// noRedaction keeps every file as it is. +func noRedaction(doc []byte) ([]byte, error) { return doc, nil } + +// archiveFiles reads an archive back as a map of paths to contents. +func archiveFiles(t *testing.T, archive []byte) map[string]string { + t.Helper() + + gz, err := gzip.NewReader(bytes.NewReader(archive)) + require.NoError(t, err) + + out := make(map[string]string) + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + + assert.Zero(t, hdr.ModTime.Unix(), "an entry has no time") + assert.Zero(t, hdr.Uid) + assert.Empty(t, hdr.Uname) + + content, err := io.ReadAll(tr) + require.NoError(t, err) + out[hdr.Name] = string(content) + } + + return out +} + +func TestCopyAndPackage(t *testing.T) { + t.Run("a package holds the full folder, redacted", func(t *testing.T) { + src := writeSkill(t, t.TempDir(), map[string]string{ + DefinitionFile: "# Skill\nuse token SECRET\n", + "references/rules.md": "the rules", + "scripts/lint.py": "print('SECRET')", + ".git/HEAD": "ref: refs/heads/main", + "nested/.git": "gitdir: ../../.git/modules/nested", + "examples/before-after.txt": "before and after", + }) + + dst := filepath.Join(t.TempDir(), "copy") + info, err := Copy(src, dst) + require.NoError(t, err) + assert.False(t, info.Truncated) + assert.True(t, strings.HasPrefix(info.Digest, "sha256:")) + + redact := func(doc []byte) ([]byte, error) { + return bytes.ReplaceAll(doc, []byte("SECRET"), []byte("[REDACTED]")), nil + } + + pkg, err := MakePackage(dst, redact) + require.NoError(t, err) + + assert.Equal(t, "# Skill\nuse token [REDACTED]\n", string(pkg.Definition)) + assert.Equal(t, map[string]string{ + DefinitionFile: "# Skill\nuse token [REDACTED]\n", + "references/rules.md": "the rules", + "scripts/lint.py": "print('[REDACTED]')", + "examples/before-after.txt": "before and after", + }, archiveFiles(t, pkg.Archive), "version-control folders and files are left out") + }) + + t.Run("the same content gives the same digests in every session", func(t *testing.T) { + files := map[string]string{DefinitionFile: skillDoc, "a/b.md": "b", "c.md": "c"} + first := writeSkill(t, t.TempDir(), files) + second := writeSkill(t, t.TempDir(), files) + + dst1 := filepath.Join(t.TempDir(), "copy") + dst2 := filepath.Join(t.TempDir(), "copy") + + info1, err := Copy(first, dst1) + require.NoError(t, err) + info2, err := Copy(second, dst2) + require.NoError(t, err) + assert.Equal(t, info1.Digest, info2.Digest) + + pkg1, err := MakePackage(dst1, noRedaction) + require.NoError(t, err) + pkg2, err := MakePackage(dst2, noRedaction) + require.NoError(t, err) + assert.Equal(t, pkg1.Archive, pkg2.Archive, "the archive is reproducible") + }) + + t.Run("a copy does not change when the folder changes", func(t *testing.T) { + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: "first"}) + dst := filepath.Join(t.TempDir(), "copy") + + first, err := Copy(src, dst) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(filepath.Join(src, DefinitionFile), []byte("second"), 0o600)) + + again, err := Copy(src, dst) + require.NoError(t, err) + assert.Equal(t, first.Digest, again.Digest, "the first copy stands") + + live, err := ContentDigest(src) + require.NoError(t, err) + assert.NotEqual(t, first.Digest, live) + + require.NoError(t, MarkChanged(dst)) + loaded, err := Load(dst) + require.NoError(t, err) + assert.True(t, loaded.Changed) + + pkg, err := MakePackage(dst, noRedaction) + require.NoError(t, err) + assert.Equal(t, "first", string(pkg.Definition)) + }) + + t.Run("a folder without SKILL.md is not read", func(t *testing.T) { + src := writeSkill(t, t.TempDir(), map[string]string{"README.md": "not a skill"}) + + _, err := Copy(src, filepath.Join(t.TempDir(), "copy")) + assert.ErrorIs(t, err, ErrNotSkill) + }) + + t.Run("a link out of the folder is not followed", func(t *testing.T) { + outside := writeSkill(t, t.TempDir(), map[string]string{"secret.txt": "private"}) + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc, "docs/inside.md": insideDoc}) + require.NoError(t, os.Symlink(filepath.Join(outside, "secret.txt"), filepath.Join(src, "secret.txt"))) + require.NoError(t, os.Symlink(outside, filepath.Join(src, "outside-dir"))) + require.NoError(t, os.Symlink(filepath.Join(src, "docs", "inside.md"), filepath.Join(src, "alias.md"))) + + dst := filepath.Join(t.TempDir(), "copy") + info, err := Copy(src, dst) + require.NoError(t, err) + assert.Equal(t, 2, info.SkippedLinks) + + pkg, err := MakePackage(dst, noRedaction) + require.NoError(t, err) + assert.Equal(t, map[string]string{ + DefinitionFile: skillDoc, + "docs/inside.md": insideDoc, + "alias.md": insideDoc, + }, archiveFiles(t, pkg.Archive)) + }) + + t.Run("a skill folder that is itself a link is read", func(t *testing.T) { + target := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc}) + link := filepath.Join(t.TempDir(), "skill") + require.NoError(t, os.Symlink(target, link)) + + dst := filepath.Join(t.TempDir(), "copy") + _, err := Copy(link, dst) + require.NoError(t, err) + + pkg, err := MakePackage(dst, noRedaction) + require.NoError(t, err) + assert.Equal(t, skillDoc, string(pkg.Definition)) + }) + + t.Run("a package over the limit keeps its definition only", func(t *testing.T) { + big := make([]byte, MaxPackageSize+1024) + _, err := rand.Read(big) + require.NoError(t, err) + + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc}) + require.NoError(t, os.WriteFile(filepath.Join(src, "model.bin"), big, 0o600)) + + dst := filepath.Join(t.TempDir(), "copy") + _, err = Copy(src, dst) + require.NoError(t, err) + + pkg, err := MakePackage(dst, noRedaction) + require.NoError(t, err) + assert.True(t, pkg.Oversize) + assert.Nil(t, pkg.Archive) + assert.Equal(t, skillDoc, string(pkg.Definition)) + + // The second push reads the stored result. + again, err := MakePackage(dst, func([]byte) ([]byte, error) { return nil, errors.New("not called") }) + require.NoError(t, err) + assert.Equal(t, pkg, again) + }) + + t.Run("a file that cannot be scanned fails the package", func(t *testing.T) { + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc}) + dst := filepath.Join(t.TempDir(), "copy") + _, err := Copy(src, dst) + require.NoError(t, err) + + _, err = MakePackage(dst, func([]byte) ([]byte, error) { return nil, errors.New("scanner down") }) + assert.Error(t, err) + }) + + t.Run("binary files are stored as they are", func(t *testing.T) { + binary := string([]byte{0xff, 0xfe, 0x00, 'S', 'E', 'C', 'R', 'E', 'T'}) + src := writeSkill(t, t.TempDir(), map[string]string{DefinitionFile: skillDoc, "icon.bin": binary}) + dst := filepath.Join(t.TempDir(), "copy") + _, err := Copy(src, dst) + require.NoError(t, err) + + pkg, err := MakePackage(dst, func(doc []byte) ([]byte, error) { + return bytes.ReplaceAll(doc, []byte("SECRET"), []byte("x")), nil + }) + require.NoError(t, err) + assert.Equal(t, binary, archiveFiles(t, pkg.Archive)["icon.bin"]) + }) +} + +func TestBaseDir(t *testing.T) { + testCases := []struct { + name string + text string + want string + wantOK bool + }{ + { + name: "a Claude Code path", + text: "Base directory for this skill: /Users/me/.claude/skills/asd-ste100\n\n# Simplified Technical English", + want: "/Users/me/.claude/skills/asd-ste100", + wantOK: true, + }, + { + name: "an OpenCode file URL inside the tool output", + text: "\n# Skill: git-release\n\nBase directory for this skill: file:///home/me/.config/opencode/skills/git%20release\n", + want: "/home/me/.config/opencode/skills/git release", + wantOK: true, + }, + {name: "a relative path", text: "Base directory for this skill: skills/x", wantOK: false}, + {name: "no marker", text: "Launching skill: x", wantOK: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, ok := BaseDir(tc.text) + assert.Equal(t, tc.wantOK, ok) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/app/cli/internal/trace/skill/source.go b/app/cli/internal/trace/skill/source.go new file mode 100644 index 000000000..a5dd0b45e --- /dev/null +++ b/app/cli/internal/trace/skill/source.go @@ -0,0 +1,90 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skill + +import ( + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +// Roots are the folders that tell where a skill came from. Each agent has its +// own layout, so each provider gives its own. +type Roots struct { + // Plugin holds the folders where the agent installs plugins. + Plugin []string + // Organization holds the folders of the managed configuration, which an + // administrator installs for all users. + Organization []string + // User holds the skill folders of the user's agent configuration. + User []string + // Project holds the repository and the session folder. + Project []string +} + +// Source returns where a skill came from, as one of the +// aicodingsession.SkillSource* constants. A plugin skill has a name with a +// plugin prefix (plugin:skill) or a folder in a plugin folder. Otherwise the +// first root list that holds the folder gives the source, in the order of +// the Roots fields. A folder that no root holds, or no folder, gives unknown. +func Source(name, dir string, roots Roots) string { + if strings.Contains(name, ":") { + return aicodingsession.SkillSourcePlugin + } + + if dir == "" { + return aicodingsession.SkillSourceUnknown + } + + for _, r := range []struct { + roots []string + source string + }{ + {roots.Plugin, aicodingsession.SkillSourcePlugin}, + {roots.Organization, aicodingsession.SkillSourceOrganization}, + {roots.User, aicodingsession.SkillSourceUser}, + {roots.Project, aicodingsession.SkillSourceProject}, + } { + for _, root := range r.roots { + if root != "" && withinResolved(dir, root) { + return r.source + } + } + } + + return aicodingsession.SkillSourceUnknown +} + +// withinResolved compares the paths as given, and again with their links +// resolved: on macOS /tmp is a link to /private/tmp, and an agent can report +// either. +func withinResolved(path, root string) bool { + if Within(path, root) { + return true + } + + resolvedPath, err := filepath.EvalSymlinks(path) + if err != nil { + return false + } + resolvedRoot, err := filepath.EvalSymlinks(root) + if err != nil { + return false + } + + return Within(resolvedPath, resolvedRoot) +} diff --git a/app/cli/internal/trace/skill/source_test.go b/app/cli/internal/trace/skill/source_test.go new file mode 100644 index 000000000..c9e79812e --- /dev/null +++ b/app/cli/internal/trace/skill/source_test.go @@ -0,0 +1,54 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package skill + +import ( + "testing" + + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/stretchr/testify/assert" +) + +func TestSource(t *testing.T) { + roots := Roots{ + Plugin: []string{"/home/me/.claude/plugins"}, + Organization: []string{"/etc/claude-code/.claude/skills"}, + User: []string{"/home/me/.claude/skills"}, + Project: []string{"/home/me/src/repo"}, + } + + testCases := []struct { + name string + skillName string + dir string + want string + }{ + {name: "a plugin prefix", skillName: "superpowers:brainstorming", dir: "/somewhere/else", want: aicodingsession.SkillSourcePlugin}, + {name: "a plugin folder", skillName: "brainstorming", dir: "/home/me/.claude/plugins/cache/x/1.0.0/skills/brainstorming", want: aicodingsession.SkillSourcePlugin}, + {name: "a managed folder", skillName: "policy", dir: "/etc/claude-code/.claude/skills/policy", want: aicodingsession.SkillSourceOrganization}, + {name: "a user folder", skillName: "asd-ste100", dir: "/home/me/.claude/skills/asd-ste100", want: aicodingsession.SkillSourceUser}, + {name: "a repository folder", skillName: "release", dir: "/home/me/src/repo/.claude/skills/release", want: aicodingsession.SkillSourceProject}, + {name: "a folder next to the repository", skillName: "release", dir: "/home/me/src/repo-other/.claude/skills/release", want: aicodingsession.SkillSourceUnknown}, + {name: "a bundled skill", skillName: "claude-api", dir: "/tmp/bundled-skills/2.1.280/claude-api", want: aicodingsession.SkillSourceUnknown}, + {name: "no folder", skillName: "graphify", want: aicodingsession.SkillSourceUnknown}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, Source(tc.skillName, tc.dir, roots)) + }) + } +} diff --git a/app/cli/internal/trace/state/skills.go b/app/cli/internal/trace/state/skills.go new file mode 100644 index 000000000..1cf380aa5 --- /dev/null +++ b/app/cli/internal/trace/state/skills.go @@ -0,0 +1,103 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +const ( + // traceDirSkills holds, for each session, a copy of each skill folder + // that the session used. A copy lives for the session, not for one push. + traceDirSkills = "skills" + // skillScanName records, inside a session's skills directory, how far + // the hooks have read each transcript of the session. + skillScanName = "scan.json" +) + +// SkillCopyDir returns the directory that holds the copy of the skill folder +// dir for a session. The name is a digest of the folder path, so the path +// itself is not in the name, and two folders never share a copy. +func (s *Store) SkillCopyDir(sessionID, dir string) string { + sum := sha256.Sum256([]byte(filepath.Clean(dir))) + + return filepath.Join(s.skillsDir(sessionID), hex.EncodeToString(sum[:16])) +} + +func (s *Store) skillsDir(sessionID string) string { + return filepath.Join(s.traceDirPath(), traceDirSkills, SanitizeID(sessionID)) +} + +// SkillScanOffsets returns how many bytes of each transcript of a session the +// hooks have read for skill uses, by a key that the provider chooses. A +// session that no hook has scanned has none. +func (s *Store) SkillScanOffsets(sessionID string) (map[string]int64, error) { + data, err := os.ReadFile(filepath.Join(s.skillsDir(sessionID), skillScanName)) + if errors.Is(err, os.ErrNotExist) { + return map[string]int64{}, nil + } + if err != nil { + return nil, fmt.Errorf("read skill scan offsets: %w", err) + } + + offsets := map[string]int64{} + if err := json.Unmarshal(data, &offsets); err != nil { + return nil, fmt.Errorf("parse skill scan offsets: %w", err) + } + + return offsets, nil +} + +// SetSkillScanOffsets replaces the offsets of a session. +func (s *Store) SetSkillScanOffsets(sessionID string, offsets map[string]int64) error { + if err := writeJSONAtomic(filepath.Join(s.skillsDir(sessionID), skillScanName), offsets); err != nil { + return fmt.Errorf("write skill scan offsets: %w", err) + } + + return nil +} + +// gcSkillCopies drops the skill copies of each session that has no record +// any more. Nothing else drops them: a push after the session end still needs +// them, and the session end never runs when the agent is killed. +func (s *Store) gcSkillCopies() error { + dir := filepath.Join(s.traceDirPath(), traceDirSkills) + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("read %s dir: %w", traceDirSkills, err) + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + record := filepath.Join(s.traceDirPath(), traceDirSessions, entry.Name()+sessionRecordExt) + if _, err := os.Stat(record); errors.Is(err, os.ErrNotExist) { + _ = os.RemoveAll(filepath.Join(dir, entry.Name())) + } + } + + return nil +} diff --git a/app/cli/internal/trace/state/skills_test.go b/app/cli/internal/trace/state/skills_test.go new file mode 100644 index 000000000..bee4ecb95 --- /dev/null +++ b/app/cli/internal/trace/state/skills_test.go @@ -0,0 +1,83 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package state + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSkillCopyDir(t *testing.T) { + store := NewGitStore(filepath.Join(t.TempDir(), ".git")) + + a := store.SkillCopyDir("sess-a", "/home/me/.claude/skills/asd-ste100") + assert.Equal(t, a, store.SkillCopyDir("sess-a", "/home/me/.claude/skills/asd-ste100/"), "the path is cleaned first") + assert.NotEqual(t, a, store.SkillCopyDir("sess-a", "/home/me/.claude/skills/graphify")) + assert.NotEqual(t, a, store.SkillCopyDir("sess-b", "/home/me/.claude/skills/asd-ste100")) + assert.False(t, strings.Contains(a, "asd-ste100"), "the folder path is not in the state") +} + +func TestSkillScanOffsets(t *testing.T) { + store := NewGitStore(filepath.Join(t.TempDir(), ".git")) + + got, err := store.SkillScanOffsets("sess-a") + require.NoError(t, err) + assert.Empty(t, got, "a session that no hook scanned has no offsets") + + require.NoError(t, store.SetSkillScanOffsets("sess-a", map[string]int64{"main": 120, "agent-a1.jsonl": 40})) + + got, err = store.SkillScanOffsets("sess-a") + require.NoError(t, err) + assert.Equal(t, map[string]int64{"main": 120, "agent-a1.jsonl": 40}, got) + + other, err := store.SkillScanOffsets("sess-b") + require.NoError(t, err) + assert.Empty(t, other) +} + +// TestGCOrphansDropsSkillCopies covers the skill copies of a session. They +// stay after the session end, for a push that comes later, and go with the +// session records. +func TestGCOrphansDropsSkillCopies(t *testing.T) { + store := NewGitStore(filepath.Join(t.TempDir(), ".git")) + require.NoError(t, store.InitTraceDir()) + + const ( + live = "skills-live" + orphan = "skills-orphan" + committed = "2026-10-07T00:00:00Z" + ) + + require.NoError(t, store.SaveCommitRecord(&CommitRecord{SHA: "sha-skills-live", SessionIDs: []string{live}, Timestamp: committed})) + require.NoError(t, store.SaveCommitRecord(&CommitRecord{SHA: "sha-skills-orphan", SessionIDs: []string{orphan}, Timestamp: committed})) + for _, id := range []string{live, orphan} { + require.NoError(t, store.SaveSessionRecord(&SessionRecord{SessionID: id, Active: true, StartedAt: time.Now().UTC().Format(time.RFC3339)})) + copyDir := store.SkillCopyDir(id, "/skills/x") + require.NoError(t, os.MkdirAll(copyDir, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(copyDir, "info.json"), []byte("{}"), 0o600)) + } + + require.NoError(t, store.GCOrphans(map[string]bool{"sha-skills-live": true})) + + assert.FileExists(t, filepath.Join(store.SkillCopyDir(live, "/skills/x"), "info.json")) + assert.NoDirExists(t, store.SkillCopyDir(orphan, "/skills/x")) +} diff --git a/app/cli/internal/trace/state/state.go b/app/cli/internal/trace/state/state.go index 7689b888a..f73564214 100644 --- a/app/cli/internal/trace/state/state.go +++ b/app/cli/internal/trace/state/state.go @@ -136,35 +136,42 @@ func (s *Store) SetRecordedSpecFiles(sessionID string, names []string) error { return removeIfExists(path) } - data, err := json.Marshal(names) + if err := writeJSONAtomic(path, names); err != nil { + return fmt.Errorf("write recorded spec files: %w", err) + } + + return nil +} + +// writeJSONAtomic writes v as JSON to path, creating its directory. The data +// goes to a temporary file that is then renamed over path, so a reader never +// sees half of it. +func writeJSONAtomic(path string, v any) error { + data, err := json.Marshal(v) if err != nil { - return fmt.Errorf("encode recorded spec files: %w", err) + return err } - dir := s.SpecRedactionDir(sessionID) + dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0o700); err != nil { - return fmt.Errorf("create spec redaction directory: %w", err) + return err } - tmp, err := os.CreateTemp(dir, "."+recordedSpecFilesName+"-*") + tmp, err := os.CreateTemp(dir, "."+filepath.Base(path)+"-*") if err != nil { - return fmt.Errorf("write recorded spec files: %w", err) + return err } defer func() { _ = os.Remove(tmp.Name()) }() if _, err := tmp.Write(data); err != nil { _ = tmp.Close() - return fmt.Errorf("write recorded spec files: %w", err) + return err } if err := tmp.Close(); err != nil { - return fmt.Errorf("write recorded spec files: %w", err) - } - - if err := os.Rename(tmp.Name(), path); err != nil { - return fmt.Errorf("write recorded spec files: %w", err) + return err } - return nil + return os.Rename(tmp.Name(), path) } func (s *Store) recordedSpecFilesPath(sessionID string) string { @@ -283,7 +290,8 @@ func (s *Store) GCOrphans(liveSHAs map[string]bool) error { _ = os.RemoveAll(filepath.Join(redactionsDir, entry.Name())) } } - return nil + + return s.gcSkillCopies() } // staleSessionIDs returns the sanitized IDs of session records older than diff --git a/app/cli/pkg/action/trace_agent_hook.go b/app/cli/pkg/action/trace_agent_hook.go index 188916518..ace543ffc 100644 --- a/app/cli/pkg/action/trace_agent_hook.go +++ b/app/cli/pkg/action/trace_agent_hook.go @@ -65,6 +65,10 @@ func HandleAgentSessionEnd(provider trace.Provider, log zerolog.Logger) error { log.Debug().Err(err).Msg("copy session data failed") } + // The last turn can have used a skill that no hook saw yet. The copies + // stay after the session end, for a push that comes later. + captureSkillLoads(provider, store, input, log) + setSessionActive(store, sessionID, false, log) // The spec stays on disk for the whole session, so that every push of it @@ -218,12 +222,16 @@ func HandleAgentPromptSubmit(provider trace.Provider, log zerolog.Logger) error log.Debug().Str("session_id", input.SessionID).Msg("prompt-submit hook invoked") - _, repoRoot, err := state.Locate() + store, repoRoot, err := state.Locate() if err != nil { log.Debug().Err(err).Msg("prompt-submit: no trace state located") return nil } + // A skill that the user started with a slash command fires no tool hook. + // This is the first hook after it when the skill calls no tool. + captureSkillLoads(provider, store, input, log) + reminder := sessionSpecReminder(repoRoot, input.SessionID, log) if err := provider.AnnouncePromptSubmit(reminder); err != nil { log.Debug().Err(err).Msg("prompt-submit: failed to send the spec reminder") @@ -270,6 +278,7 @@ func HandleAgentPreToolUse(provider trace.Provider, log zerolog.Logger) error { } ensureSessionTracked(provider, store, repoRoot, input, log) + captureSkillLoads(provider, store, input, log) switch { case provider.IsCommandTool(input.ToolName): @@ -573,6 +582,12 @@ func HandleAgentPostToolUse(provider trace.Provider, log zerolog.Logger) error { isCommand := provider.IsCommandTool(input.ToolName) isFileWriting := provider.IsFileWritingTool(input.ToolName) if !isCommand && !isFileWriting { + // The hook fires for the skill tool too, whose skill is loaded only + // after the call. The skill is the one thing to record for it. + if store, _, err := state.Locate(); err == nil { + captureSkillLoads(provider, store, input, log) + } + log.Debug().Str("tool", input.ToolName).Msg("post-tool-use: not a tracked tool, skipping") return nil } diff --git a/app/cli/pkg/action/trace_hook_handler.go b/app/cli/pkg/action/trace_hook_handler.go index fee48bf25..b45742e30 100644 --- a/app/cli/pkg/action/trace_hook_handler.go +++ b/app/cli/pkg/action/trace_hook_handler.go @@ -656,6 +656,7 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts sessionID string evidence *aicodingsession.Evidence specs []spec.Capture + skills []sessionSkill } var sessions []sessionEvidence @@ -708,6 +709,11 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts captures, specWarnings := readSessionSpecs(store, repoRoot, sessionID, log) result.Data.Warnings = append(result.Data.Warnings, specWarnings...) + // The skills that the session used, each with the copy of its folder + // that a hook made when the session used it. + skills, skillWarnings := readSessionSkills(provider, store, parseOpts, repoRoot, log) + result.Data.Warnings = append(result.Data.Warnings, skillWarnings...) + // Apply repo-wide context with per-session commit overrides if gitCtxErr == nil { sessionCtx := *gitCtx @@ -746,6 +752,7 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts sessionID: sessionID, evidence: result, specs: captures, + skills: skills, }) } @@ -802,10 +809,16 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts // replace an earlier material. names := materials.NewNameAllocator(nil) for _, se := range sessions { - entries, warnings, stored := attachSpecs(ctx, executor, newSpecRedactor(store.SpecRedactionDir(se.sessionID)), names, se.sessionID, se.specs, log) - se.evidence.Data.Spec = entries + redactor := newSpecRedactor(store.SpecRedactionDir(se.sessionID)) + entries, warnings, stored := attachSpecs(ctx, executor, redactor, names, se.sessionID, se.specs, log) se.evidence.Data.Warnings = append(se.evidence.Data.Warnings, warnings...) + // The skill entries come after the sources that the agent wrote. + skillEntries, skillWarnings := attachSkills(ctx, executor, redactor, names, se.sessionID, se.skills, log) + entries = append(entries, skillEntries...) + se.evidence.Data.Spec = entries + se.evidence.Data.Warnings = append(se.evidence.Data.Warnings, skillWarnings...) + name := evidenceName(se.sessionID) if err := addSessionEvidence(ctx, executor, name, se.evidence); err != nil { // Warn, not debug: the session is left out of the attestation, diff --git a/app/cli/pkg/action/trace_skills.go b/app/cli/pkg/action/trace_skills.go new file mode 100644 index 000000000..d8ba416ec --- /dev/null +++ b/app/cli/pkg/action/trace_skills.go @@ -0,0 +1,278 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package action + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/rs/zerolog" +) + +// maxSkillEntries is the limit of skill entries for each session. It is apart +// from the limit of the spec sources that the agent writes, so that many +// skills cannot push out the task and its spec. +const maxSkillEntries = 25 + +// specAnnotationSkillContent tells the two materials of a skill apart. +const specAnnotationSkillContent = "chainloop.spec.skill.content" + +// The values of specAnnotationSkillContent. +const ( + skillContentDefinition = "definition" + skillContentPackage = "package" +) + +// captureSkillLoads copies the folder of each skill that the session loaded +// since the last hook, into the trace state. A hook runs it, so that the +// evidence holds the skill as it was when it ran, also when its files change +// before the push. +// +// A copy is never replaced. A later use that finds different content only +// marks the copy, and the push records a warning. +// +// Best effort throughout: a skill that cannot be copied here is copied at the +// push, and nothing here may slow down or fail the agent's tool call. +func captureSkillLoads(provider trace.Provider, store *state.Store, input *trace.HookInput, log zerolog.Logger) { + tracker, ok := provider.(trace.SkillTracker) + if !ok { + return + } + + for _, dir := range tracker.NewSkillLoads(store, input) { + copySkill(store, input.SessionID, dir, log) + } +} + +// copySkill makes the copy of a skill folder for a session, or checks an +// existing copy against the folder. +func copySkill(store *state.Store, sessionID, dir string, log zerolog.Logger) { + dst := store.SkillCopyDir(sessionID, dir) + + info, err := skill.Load(dst) + if err != nil { + if _, err := skill.Copy(dir, dst); err != nil { + log.Debug().Err(err).Str("session_id", sessionID).Msg("could not copy a skill folder; the push copies it") + } + + return + } + + if info.Changed { + return + } + + live, err := skill.ContentDigest(dir) + if err != nil || live == info.Digest { + return + } + + if err := skill.MarkChanged(dst); err != nil { + log.Debug().Err(err).Str("session_id", sessionID).Msg("could not mark a changed skill folder") + } +} + +// sessionSkill is one used skill, ready to be stored: its counts, where it +// came from, and the copy of its folder. +type sessionSkill struct { + use trace.SkillUse + source string + copyDir string +} + +// readSessionSkills returns the skills that a session used, each with a copy +// of its folder, and a warning for each skill that is left out or not as it +// was when it ran. +// +// A provider without skill support gives no skills and no warning. A failure +// to read the skills is never a reason to lose the session. +func readSessionSkills(provider trace.Provider, store *state.Store, opts *trace.ParseOpts, repoRoot string, log zerolog.Logger) ([]sessionSkill, []string) { + tracker, ok := provider.(trace.SkillTracker) + if !ok { + return nil, nil + } + + uses, warnings, err := tracker.SessionSkills(opts) + if err != nil { + // The error stays in the local log: its text carries local paths. + log.Warn().Err(err).Str("session", opts.SessionID).Msg("could not read the skills that the session used; the session is attested without them") + return nil, []string{"the skills that the session used were not recorded: the session data could not be read"} + } + + if len(uses) > maxSkillEntries { + // The skills that the session used first are kept. + warnings = append(warnings, fmt.Sprintf("%d skill entries beyond the first %d were dropped", len(uses)-maxSkillEntries, maxSkillEntries)) + uses = uses[:maxSkillEntries] + } + + roots := tracker.SkillRoots(repoRoot) + + skills := make([]sessionSkill, 0, len(uses)) + for _, use := range uses { + if use.Dir == "" { + warnings = append(warnings, fmt.Sprintf("skill %q was not recorded: the session data does not tell its folder", use.Name)) + continue + } + + dst := store.SkillCopyDir(opts.SessionID, use.Dir) + info, err := skill.Load(dst) + switch { + case err != nil: + // No hook ran after the use, so the folder is copied now. It can + // differ from the folder that the session used. + if info, err = skill.Copy(use.Dir, dst); err != nil { + log.Warn().Err(err).Str("session", opts.SessionID).Str("skill", use.Name).Msg("could not copy a skill folder") + warnings = append(warnings, fmt.Sprintf("skill %q was not recorded: its folder could not be read", use.Name)) + continue + } + warnings = append(warnings, fmt.Sprintf("skill %q was copied at push time, not when the session used it", use.Name)) + case info.Changed: + warnings = append(warnings, fmt.Sprintf("skill %q changed after its first use; the evidence holds the skill as it was at the first use", use.Name)) + } + + if info.SkippedLinks > 0 { + warnings = append(warnings, fmt.Sprintf("skill %q: %d links out of the skill folder, or to a folder, were left out", use.Name, info.SkippedLinks)) + } + + skills = append(skills, sessionSkill{ + use: use, + source: skill.Source(use.Name, use.Dir, roots), + copyDir: dst, + }) + } + + return skills, warnings +} + +// attachSkills stores the materials of each used skill and returns the spec +// entries that the session material records for them. +// +// Each skill gives two EVIDENCE materials: its redacted SKILL.md file, which +// the digest of the entry points to so that a viewer can show it, and a +// reproducible archive of the full redacted folder, which the metadata points +// to as the download. A skill that cannot be stored is left out with a +// warning, as a spec source is. +func attachSkills(ctx context.Context, adder specMaterialAdder, redactor *specRedactor, names *materials.NameAllocator, sessionID string, skills []sessionSkill, log zerolog.Logger) (entries []aicodingsession.SpecEntry, warnings []string) { + if len(skills) == 0 { + return nil, nil + } + + tmpDir, err := os.MkdirTemp("", "chainloop-trace-skill-*") + if err != nil { + return nil, []string{fmt.Sprintf("%d skill entries were not recorded: %v", len(skills), err)} + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + for i, s := range skills { + entry, warning, err := storeSkill(ctx, adder, redactor, names, filepath.Join(tmpDir, strconv.Itoa(i)), sessionID, s) + if err != nil { + // The error stays in the local log. The warning goes into the + // uploaded evidence, and error text can carry local paths. + log.Warn().Err(err).Str("session", sessionID).Str("skill", s.use.Name).Msg("could not record a skill entry") + warnings = append(warnings, fmt.Sprintf("skill %q was not recorded", s.use.Name)) + + continue + } + if warning != "" { + warnings = append(warnings, warning) + } + + entries = append(entries, entry) + } + + return entries, warnings +} + +// storeSkill adds the materials of one skill and returns its spec entry. A +// package that cannot be stored costs the package only, with a warning: the +// definition and the counts still go into the evidence. +func storeSkill(ctx context.Context, adder specMaterialAdder, redactor *specRedactor, names *materials.NameAllocator, dir, sessionID string, s sessionSkill) (aicodingsession.SpecEntry, string, error) { + pkg, err := skill.MakePackage(s.copyDir, func(doc []byte) ([]byte, error) { + return redactor.Redact(ctx, doc) + }) + if err != nil { + return aicodingsession.SpecEntry{}, "", err + } + + if err := os.MkdirAll(dir, 0o700); err != nil { + return aicodingsession.SpecEntry{}, "", fmt.Errorf("staging the skill: %w", err) + } + + // spec--skill-. The + // allocator makes each name unique within the attestation. + stem := materialStem(s.use.Name, "skill") + base := specMaterialPrefix(sessionID) + "skill-" + stem + annotations := func(content string) map[string]string { + return map[string]string{ + specAnnotationSession: sessionID, + specAnnotationKind: aicodingsession.SpecKindSkill, + specAnnotationTitle: s.use.Name, + specAnnotationSkillContent: content, + } + } + + definitionPath := filepath.Join(dir, skill.DefinitionFile) + if err := os.WriteFile(definitionPath, pkg.Definition, 0o600); err != nil { + return aicodingsession.SpecEntry{}, "", fmt.Errorf("staging the skill: %w", err) + } + + digest, err := adder.AddMaterial(ctx, names.AllocateNamed(base), definitionPath, specMaterialKind, annotations(skillContentDefinition)) + if err != nil { + return aicodingsession.SpecEntry{}, "", err + } + + var packageDigest, warning string + if pkg.Oversize { + warning = fmt.Sprintf("the package of skill %q is larger than %d MB and was not uploaded", s.use.Name, skill.MaxPackageSize>>20) + } else { + // The archive is stored under the name of the skill. + archivePath := filepath.Join(dir, stem+".tar.gz") + if err = os.WriteFile(archivePath, pkg.Archive, 0o600); err == nil { + packageDigest, err = adder.AddMaterial(ctx, names.AllocateNamed(base+"-pkg"), archivePath, specMaterialKind, annotations(skillContentPackage)) + } + if err != nil { + warning = fmt.Sprintf("the package of skill %q was not uploaded", s.use.Name) + } + } + + capturedAt := s.use.FirstUsedAt + if capturedAt == "" { + capturedAt = state.NowTimestamp() + } + + return aicodingsession.SpecEntry{ + Kind: aicodingsession.SpecKindSkill, + Title: s.use.Name, + Digest: digest, + CapturedAt: capturedAt, + Metadata: &aicodingsession.SpecMetadata{ + Source: s.source, + PackageDigest: packageDigest, + InvocationCount: s.use.ByModel + s.use.ByUser, + ByModel: s.use.ByModel, + ByUser: s.use.ByUser, + InSubagents: s.use.InSubagents, + }, + }, warning, nil +} diff --git a/app/cli/pkg/action/trace_skills_test.go b/app/cli/pkg/action/trace_skills_test.go new file mode 100644 index 000000000..a946f6f64 --- /dev/null +++ b/app/cli/pkg/action/trace_skills_test.go @@ -0,0 +1,324 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package action + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + skillSessionID = "7412a0c2-1a2b-4c3d-8e9f-0a1b2c3d4e5f" + skillDoc = "# Skill" + skillTenOClock = "2026-10-07T10:00:00Z" + steSkillName = "asd-ste100" +) + +// skillProvider is a provider that records skills, with uses and loads set by +// the test. +type skillProvider struct { + trace.Provider + + uses []trace.SkillUse + loads []string + err error + roots skill.Roots +} + +func (p *skillProvider) NewSkillLoads(_ *state.Store, _ *trace.HookInput) []string { + return p.loads +} + +func (p *skillProvider) SessionSkills(_ *trace.ParseOpts) ([]trace.SkillUse, []string, error) { + return p.uses, nil, p.err +} + +func (p *skillProvider) SkillRoots(_ string) skill.Roots { + return p.roots +} + +// noSkillProvider is a provider without skill support, such as Cursor. +type noSkillProvider struct { + trace.Provider +} + +func makeSkill(t *testing.T, files map[string]string) string { + t.Helper() + + dir := t.TempDir() + for rel, content := range files { + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + } + + return dir +} + +func untar(t *testing.T, archive string) map[string]string { + t.Helper() + + gz, err := gzip.NewReader(bytes.NewReader([]byte(archive))) + require.NoError(t, err) + + out := make(map[string]string) + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return out + } + require.NoError(t, err) + + content, err := io.ReadAll(tr) + require.NoError(t, err) + out[hdr.Name] = string(content) + } +} + +func TestCaptureSkillLoads(t *testing.T) { + t.Run("a skill file that changes after the use does not change the package", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: "as it ran"}) + provider := &skillProvider{ + loads: []string{dir}, + uses: []trace.SkillUse{{Name: steSkillName, Dir: dir, FirstUsedAt: skillTenOClock, ByModel: 2}}, + } + input := &trace.HookInput{SessionID: skillSessionID} + + captureSkillLoads(provider, store, input, zerolog.Nop()) + require.NoError(t, os.WriteFile(filepath.Join(dir, skill.DefinitionFile), []byte("changed later"), 0o600)) + // A later use finds the changed folder. + captureSkillLoads(provider, store, input, zerolog.Nop()) + + skills, warnings := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + require.Len(t, skills, 1) + assert.Equal(t, []string{`skill "asd-ste100" changed after its first use; the evidence holds the skill as it was at the first use`}, warnings) + + adder := &fakeMaterialAdder{} + _, _ = attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, skills, zerolog.Nop()) + require.Len(t, adder.added, 2) + assert.Equal(t, "as it ran", adder.added[0].content) + }) + + t.Run("a provider without skill support copies nothing", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + captureSkillLoads(&noSkillProvider{}, store, &trace.HookInput{SessionID: skillSessionID}, zerolog.Nop()) + + skills, warnings := readSessionSkills(&noSkillProvider{}, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + assert.Empty(t, skills) + assert.Empty(t, warnings, "no skill entry means not recorded, with no warning") + }) +} + +func TestReadSessionSkills(t *testing.T) { + t.Run("a skill that no hook copied is copied at push time, with a warning", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: skillDoc}) + provider := &skillProvider{uses: []trace.SkillUse{{Name: steSkillName, Dir: dir, ByUser: 1}}} + + skills, warnings := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + require.Len(t, skills, 1) + assert.Equal(t, []string{`skill "asd-ste100" was copied at push time, not when the session used it`}, warnings) + }) + + t.Run("a missing folder gives a warning and the other skills", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: skillDoc}) + provider := &skillProvider{uses: []trace.SkillUse{ + {Name: "gone", Dir: filepath.Join(t.TempDir(), "gone"), ByModel: 1}, + {Name: "unknown-dir", ByModel: 1}, + {Name: steSkillName, Dir: dir, ByModel: 1}, + }} + captureSkillLoads(&skillProvider{loads: []string{dir}}, store, &trace.HookInput{SessionID: skillSessionID}, zerolog.Nop()) + + skills, warnings := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + require.Len(t, skills, 1) + assert.Equal(t, steSkillName, skills[0].use.Name) + assert.Equal(t, []string{ + `skill "gone" was not recorded: its folder could not be read`, + `skill "unknown-dir" was not recorded: the session data does not tell its folder`, + }, warnings) + for _, w := range warnings { + assert.NotContains(t, w, string(filepath.Separator)+"gone", "a warning holds no local path") + } + }) + + t.Run("skills beyond the limit are dropped, the first ones kept", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: skillDoc}) + captureSkillLoads(&skillProvider{loads: []string{dir}}, store, &trace.HookInput{SessionID: skillSessionID}, zerolog.Nop()) + + uses := make([]trace.SkillUse, 0, maxSkillEntries+2) + for i := range maxSkillEntries + 2 { + uses = append(uses, trace.SkillUse{Name: "skill-" + string(rune('a'+i)), Dir: dir, ByModel: 1}) + } + + skills, warnings := readSessionSkills(&skillProvider{uses: uses}, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + require.Len(t, skills, maxSkillEntries) + assert.Equal(t, "skill-a", skills[0].use.Name) + assert.Contains(t, warnings, "2 skill entries beyond the first 25 were dropped") + }) + + t.Run("a session data failure costs the skills only", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + skills, warnings := readSessionSkills(&skillProvider{err: errors.New("/home/me/x: no such file")}, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + assert.Empty(t, skills) + assert.Equal(t, []string{"the skills that the session used were not recorded: the session data could not be read"}, warnings) + }) + + t.Run("the source comes from the provider roots", func(t *testing.T) { + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, map[string]string{skill.DefinitionFile: skillDoc}) + captureSkillLoads(&skillProvider{loads: []string{dir}}, store, &trace.HookInput{SessionID: skillSessionID}, zerolog.Nop()) + + provider := &skillProvider{ + uses: []trace.SkillUse{{Name: steSkillName, Dir: dir, ByModel: 1}}, + roots: skill.Roots{User: []string{filepath.Dir(dir)}}, + } + + skills, _ := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + require.Len(t, skills, 1) + assert.Equal(t, aicodingsession.SkillSourceUser, skills[0].source) + }) +} + +func TestAttachSkills(t *testing.T) { + // Assembled from fragments so that nothing credential-shaped is committed. + const pat = "ghp_erOZlZv0B1e3amrQ" + "ugdwZ8Ro2W4kDql9WPTf" + + stored := func(t *testing.T, files map[string]string, use trace.SkillUse) sessionSkill { + t.Helper() + + store := state.NewGitStore(filepath.Join(t.TempDir(), ".git")) + dir := makeSkill(t, files) + dst := store.SkillCopyDir(skillSessionID, dir) + _, err := skill.Copy(dir, dst) + require.NoError(t, err) + + use.Dir = dir + + return sessionSkill{use: use, source: aicodingsession.SkillSourceUser, copyDir: dst} + } + + t.Run("each skill gives a definition, a package and an entry", func(t *testing.T) { + s := stored(t, map[string]string{ + skill.DefinitionFile: "# STE\ntoken " + pat + "\n", + "scripts/ste-lint.py": "print('" + pat + "')", + }, trace.SkillUse{Name: steSkillName, FirstUsedAt: "2026-10-07T10:14:51Z", ByModel: 2, ByUser: 1, InSubagents: 1}) + + adder := &fakeMaterialAdder{} + entries, warnings := attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + assert.Empty(t, warnings) + require.Len(t, adder.added, 2) + + definition, pkg := adder.added[0], adder.added[1] + assert.Equal(t, "spec-7412a0-skill-asd-ste100", definition.name) + assert.Equal(t, skill.DefinitionFile, definition.fileName) + assert.Equal(t, "EVIDENCE", definition.kind) + assert.NotContains(t, definition.content, pat, "the definition is redacted") + assert.Equal(t, map[string]string{ + specAnnotationSession: skillSessionID, + specAnnotationKind: aicodingsession.SpecKindSkill, + specAnnotationTitle: steSkillName, + specAnnotationSkillContent: skillContentDefinition, + }, definition.annotations) + + assert.Equal(t, "spec-7412a0-skill-asd-ste100-pkg", pkg.name) + assert.Equal(t, "asd-ste100.tar.gz", pkg.fileName) + assert.Equal(t, "EVIDENCE", pkg.kind) + assert.Equal(t, "package", pkg.annotations["chainloop.spec.skill.content"]) + + files := untar(t, pkg.content) + assert.Equal(t, definition.content, files[skill.DefinitionFile], "the definition and the packaged SKILL.md are the same bytes") + assert.NotContains(t, files["scripts/ste-lint.py"], pat, "each text file is redacted") + + assert.Equal(t, []aicodingsession.SpecEntry{{ + Kind: aicodingsession.SpecKindSkill, + Title: steSkillName, + Digest: "sha256:spec-7412a0-skill-asd-ste100", + CapturedAt: "2026-10-07T10:14:51Z", + Metadata: &aicodingsession.SpecMetadata{ + Source: aicodingsession.SkillSourceUser, + PackageDigest: "sha256:spec-7412a0-skill-asd-ste100-pkg", + InvocationCount: 3, ByModel: 2, ByUser: 1, InSubagents: 1, + }, + }}, entries, "no role, description or source address") + }) + + t.Run("a package over the limit keeps the definition and the counts", func(t *testing.T) { + big := make([]byte, skill.MaxPackageSize+1024) + _, err := rand.Read(big) + require.NoError(t, err) + + s := stored(t, map[string]string{skill.DefinitionFile: "# Big", "model.bin": string(big)}, trace.SkillUse{Name: "big", FirstUsedAt: skillTenOClock, ByModel: 1}) + + adder := &fakeMaterialAdder{} + entries, warnings := attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + assert.Equal(t, []string{`the package of skill "big" is larger than 5 MB and was not uploaded`}, warnings) + require.Len(t, adder.added, 1) + require.Len(t, entries, 1) + assert.Empty(t, entries[0].Metadata.PackageDigest) + assert.Equal(t, 1, entries[0].Metadata.InvocationCount) + }) + + t.Run("a package that cannot be stored costs the package only", func(t *testing.T) { + s := stored(t, map[string]string{skill.DefinitionFile: skillDoc}, trace.SkillUse{Name: steSkillName, FirstUsedAt: skillTenOClock, ByModel: 1}) + + adder := &fakeMaterialAdder{failOn: map[string]bool{"spec-7412a0-skill-asd-ste100-pkg": true}} + entries, warnings := attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + assert.Equal(t, []string{`the package of skill "asd-ste100" was not uploaded`}, warnings) + require.Len(t, entries, 1) + assert.Empty(t, entries[0].Metadata.PackageDigest) + }) + + t.Run("a definition that cannot be stored leaves the skill out", func(t *testing.T) { + s := stored(t, map[string]string{skill.DefinitionFile: skillDoc}, trace.SkillUse{Name: steSkillName, FirstUsedAt: skillTenOClock, ByModel: 1}) + + adder := &fakeMaterialAdder{failOn: map[string]bool{"spec-7412a0-skill-asd-ste100": true}} + entries, warnings := attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + assert.Empty(t, entries) + assert.Equal(t, []string{`skill "asd-ste100" was not recorded`}, warnings) + }) + + t.Run("a plugin skill gets a valid material name", func(t *testing.T) { + s := stored(t, map[string]string{skill.DefinitionFile: skillDoc}, trace.SkillUse{Name: "superpowers:brainstorming", FirstUsedAt: skillTenOClock, ByModel: 1}) + + adder := &fakeMaterialAdder{} + _, _ = attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir()), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop()) + require.Len(t, adder.added, 2) + assert.Equal(t, "spec-7412a0-skill-superpowers-brainstorming", adder.added[0].name) + assert.Equal(t, "superpowers-brainstorming.tar.gz", adder.added[1].fileName) + assert.Equal(t, "superpowers:brainstorming", adder.added[0].annotations["chainloop.spec.title"]) + }) +} diff --git a/app/cli/pkg/action/trace_spec_materials.go b/app/cli/pkg/action/trace_spec_materials.go index ab443b391..b0b8ebbda 100644 --- a/app/cli/pkg/action/trace_spec_materials.go +++ b/app/cli/pkg/action/trace_spec_materials.go @@ -261,18 +261,30 @@ func (r *specRedactor) Redact(ctx context.Context, doc []byte) ([]byte, error) { // ID>-. The allocator sanitizes it into a valid material name and // makes it unique within the attestation. func specMaterialName(sessionID, fileName string) string { + return specMaterialPrefix(sessionID) + materialStem(strings.TrimSuffix(fileName, filepath.Ext(fileName)), "source") +} + +// specMaterialPrefix is the start of the name of each spec material of a +// session: spec--. +func specMaterialPrefix(sessionID string) string { short := sessionID if len(short) > 6 { short = short[:6] } - stem := materials.SanitizeMaterialName(strings.TrimSuffix(fileName, filepath.Ext(fileName))) + return "spec-" + short + "-" +} + +// materialStem sanitizes name into the part of a material name taken from it, +// cut to maxSpecSlugLen, or returns fallback when nothing is left. +func materialStem(name, fallback string) string { + stem := materials.SanitizeMaterialName(name) if len(stem) > maxSpecSlugLen { stem = strings.TrimRight(stem[:maxSpecSlugLen], "-") } if stem == "" { - stem = "source" + return fallback } - return "spec-" + short + "-" + stem + return stem } diff --git a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json index 70654051e..b3bf2d908 100644 --- a/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json +++ b/internal/schemavalidators/internal_schemas/aicodingsession/ai-coding-session-0.1.schema.json @@ -80,7 +80,7 @@ "properties": { "kind": { "type": "string", - "description": "What this entry was resolved from: 'ticket', 'document', 'image', or 'text' for a spec stated in the session itself. Left unconstrained so further kinds need no schema version bump; a producer normalises anything it does not recognise to 'text'." + "description": "What this entry was resolved from: 'ticket', 'document', 'image', 'text' for a spec stated in the session itself, or 'skill' for a skill that the session used. Left unconstrained so further kinds need no schema version bump; a producer normalises anything it does not recognise to 'text'." }, "role": { "type": "string", @@ -106,7 +106,43 @@ "captured_at": { "type": "string", "minLength": 1, - "description": "ISO 8601 timestamp of when the agent wrote this entry. Self-reported by the producer, not a trusted timestamp." + "description": "ISO 8601 timestamp of when the agent wrote this entry. For a 'skill' entry, the time of the first use in the transcript. Self-reported by the producer, not a trusted timestamp." + }, + "metadata": { + "type": "object", + "description": "Data that depends on the kind of the entry. Only 'skill' entries have it. For a skill, the digest of the entry points to the SKILL.md file.", + "properties": { + "source": { + "type": "string", + "description": "Where the skill came from: 'project', 'user', 'organization', 'plugin', or 'unknown'. Left unconstrained so further values need no schema version bump." + }, + "package_digest": { + "type": "string", + "pattern": "^sha256:[a-f0-9]{64}$", + "description": "Digest of the EVIDENCE material that holds a reproducible archive of the full skill folder. Absent when the archive was not uploaded." + }, + "invocation_count": { + "type": "integer", + "minimum": 0, + "description": "Number of times the session used the skill: by_model plus by_user" + }, + "by_model": { + "type": "integer", + "minimum": 0, + "description": "Number of uses that the model started with a tool call" + }, + "by_user": { + "type": "integer", + "minimum": 0, + "description": "Number of uses that the user started with a slash command" + }, + "in_subagents": { + "type": "integer", + "minimum": 0, + "description": "Number of uses that came from subagents" + } + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go index 58ca4ae9d..638921db7 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go +++ b/pkg/attestation/crafter/materials/aicodingsession/aicodingsession.go @@ -70,6 +70,11 @@ const ( SpecKindText = "text" ) +// SpecKindSkill is a skill that the agent used in the session. The CLI makes +// these entries from the transcript, never the agent, so ResolveSpecKind does +// not accept it: an agent that writes it in a spec file gets kind text. +const SpecKindSkill = "skill" + // ResolveSpecKind maps a captured kind onto the vocabulary above, so the // normalisation is written down once instead of at each producer and consumer. func ResolveSpecKind(kind string) string { @@ -137,8 +142,50 @@ type SpecEntry struct { Digest string `json:"digest"` // CapturedAt is when the agent wrote this entry, RFC3339. It comes from the // file's modification time, so it is the last write rather than the first, - // and it is trivially forgeable: not a trusted timestamp. + // and it is trivially forgeable: not a trusted timestamp. For a skill, it + // is the time of the first use in the transcript. CapturedAt string `json:"captured_at"` + // Metadata holds data that depends on the kind. Only skill entries have + // it. + Metadata *SpecMetadata `json:"metadata,omitempty"` +} + +// Where a skill came from. The evidence never holds the path of the skill +// folder, because the path exposes local user data. +const ( + // SkillSourceProject is a skill from the repository. + SkillSourceProject = "project" + // SkillSourceUser is a skill from the user's agent configuration. + SkillSourceUser = "user" + // SkillSourceOrganization is a skill that an administrator installs for + // all users. + SkillSourceOrganization = "organization" + // SkillSourcePlugin is a skill from an installed plugin. + SkillSourcePlugin = "plugin" + // SkillSourceUnknown is any other skill. + SkillSourceUnknown = "unknown" +) + +// SpecMetadata is the kind-specific data of a spec entry. The fields below are +// those of a skill entry. A later kind can add its own fields. +type SpecMetadata struct { + // Source is one of the SkillSource* constants. + Source string `json:"source,omitempty"` + // PackageDigest identifies the EVIDENCE material that holds an archive + // of the full skill folder, as "sha256:". Absent when the archive was + // not uploaded, for example because it was too large. + PackageDigest string `json:"package_digest,omitempty"` + // InvocationCount is the number of times the session used the skill. It + // is ByModel plus ByUser. + InvocationCount int `json:"invocation_count"` + // ByModel is the number of uses that the model started with a tool call. + ByModel int `json:"by_model"` + // ByUser is the number of uses that the user started with a slash + // command. + ByUser int `json:"by_user"` + // InSubagents is the number of uses, of either start, that came from + // subagents. + InSubagents int `json:"in_subagents"` } // Agent identifies the AI agent provider. diff --git a/pkg/attestation/crafter/materials/aicodingsession/spec_test.go b/pkg/attestation/crafter/materials/aicodingsession/spec_test.go index 376141a28..a5ae7282a 100644 --- a/pkg/attestation/crafter/materials/aicodingsession/spec_test.go +++ b/pkg/attestation/crafter/materials/aicodingsession/spec_test.go @@ -41,6 +41,8 @@ func TestResolveSpecKind(t *testing.T) { // newer Chainloop, so a value outside the vocabulary is a mistake to // normalise away rather than a future kind to pass through. {name: "a kind outside the vocabulary is text", kind: "jira-ticket", want: SpecKindText}, + // Only the CLI makes skill entries, so an agent cannot write one. + {name: "an agent-written skill is text", kind: SpecKindSkill, want: SpecKindText}, } for _, tc := range testCases { @@ -97,6 +99,25 @@ func TestSpecEntryMatchesSchema(t *testing.T) { name: "only what the schema requires", entry: SpecEntry{Kind: SpecKindText, Digest: testSpecDigest, CapturedAt: "2026-09-16T10:12:03Z"}, }, + { + name: "a skill with its metadata", + entry: SpecEntry{ + Kind: SpecKindSkill, Title: "asd-ste100", Digest: testSpecDigest, CapturedAt: "2026-10-07T10:14:51Z", + Metadata: &SpecMetadata{ + Source: SkillSourceUser, PackageDigest: testSpecDigest, + InvocationCount: 3, ByModel: 2, ByUser: 1, InSubagents: 1, + }, + }, + }, + { + // A package over the limit is not uploaded, and a session can + // use a skill only from the model: zero counts stay in the output. + name: "a skill without a package", + entry: SpecEntry{ + Kind: SpecKindSkill, Title: "graphify", Digest: testSpecDigest, CapturedAt: "2026-10-07T10:14:51Z", + Metadata: &SpecMetadata{Source: SkillSourceUnknown, InvocationCount: 1, ByModel: 1}, + }, + }, } for _, tc := range testCases { From 1f650f11310b11a61995b34a0c0d2e4173e3e544 Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Wed, 7 Oct 2026 18:40:28 +0200 Subject: [PATCH 2/4] test(trace): cover a skill from its use to the attestation Split RunTracePush into a step that builds the session evidence and a step that adds the materials, so that a test can run the push up to the upload. A new scenario test drives the Claude Code hooks for a skill that the model starts and one that the user starts, and checks the definition, the package and the session material that the push adds. Refs #3540 Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: ee05bc4f-cf05-4433-b614-415b89577db7 --- app/cli/pkg/action/trace_hook_handler.go | 276 +++++++++-------- .../pkg/action/trace_skills_scenario_test.go | 280 ++++++++++++++++++ app/cli/pkg/action/trace_skills_test.go | 6 +- 3 files changed, 431 insertions(+), 131 deletions(-) create mode 100644 app/cli/pkg/action/trace_skills_scenario_test.go diff --git a/app/cli/pkg/action/trace_hook_handler.go b/app/cli/pkg/action/trace_hook_handler.go index b45742e30..1a435449a 100644 --- a/app/cli/pkg/action/trace_hook_handler.go +++ b/app/cli/pkg/action/trace_hook_handler.go @@ -639,6 +639,139 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts log.Debug().Int("session_count", len(sessionCommits)).Msg("grouped commits by session") + sessions := buildSessionEvidence(ctx, store, repoRoot, gitClient, sessionCommits, sessionRecords, sessionMode, log) + + if len(sessions) == 0 { + log.Debug().Msg("no session evidence could be generated") + + return nil + } + + // Create attestation using local state file to avoid conflicts with other attestations + localStatePath := store.AttestationStatePath() + defer func() { _ = os.Remove(localStatePath) }() + + projectName, organization, workflowName := resolvePushIdentity(repoRoot, opts) + if projectName == "" { + if opts.IgnoreYAML { + return fmt.Errorf("no project name provided") + } + + return fmt.Errorf("no project name found (pass via options or .chainloop.yml)") + } + + log.Debug().Str("project", projectName).Msg("resolved project for trace push") + + execOpts := []ExecutorOption{WithLocalStatePath(localStatePath), WithLogger(log)} + if organization != "" { + log.Debug().Str("forced_org", organization).Msg("forcing organization for trace push") + execOpts = append(execOpts, WithForcedOrganization(organization)) + } + executor, err := NewAttestationExecutor(opts.ActionOpts, opts.CLIVersion, execOpts...) + if err != nil { + // Warn (not debug): a misconfiguration here means no attestation is + // sent at all, and the pre-push hook swallows the returned error + // unless require-trace is on. + log.Warn().Err(err).Msg("skipping trace attestation") + + return err + } + defer func() { _ = executor.Close() }() + + attestationID, err := executor.Init(ctx, workflowName, projectName, opts.ProjectVersion) + if err != nil { + return fmt.Errorf("attestation init: %w", err) + } + + log.Debug().Str("attestation_id", attestationID).Msg("attestation initialized") + + // Add evidence for each session: its spec materials first, so that the + // session material can record their digests, then the session itself. + attestedSessions, attestedSpecs := attachSessionEvidence(ctx, executor, store, sessions, log) + + if len(attestedSessions) == 0 { + log.Debug().Msg("no evidence successfully added, resetting attestation") + _ = executor.Reset(ctx, AttestationResetTriggerCancelled, "no CHAINLOOP_AI_CODING_SESSION evidence added") + + return nil + } + + // Push attestation + log.Debug().Msg("pushing attestation") + res, err := executor.Push(ctx) + if err != nil { + return fmt.Errorf("attestation push: %w", err) + } + + // Tell the user where each session landed. The organization comes from the + // control plane rather than opts.Organization, which is empty whenever the + // CLI's current org is used. + links := logAttestedSessions(log, res.UIDashboardURL, res.GetOrganization(), attestedSessions) + + // Hand the links to the agent hook that runs after this push. When the + // push was driven by a coding agent's shell tool, the log line above is + // captured into that tool's output rather than shown to the user, so the + // hook is what actually puts the link in front of them. A failure here + // costs a notification, never the attestation that already succeeded. + // + // The caller tells us whether to skip, rather than us inferring it from + // on-disk state: the trace-run sentinel outlives a killed run, and + // reading it here would silently suppress every later notification in + // that repository. + if opts.SkipAgentNotification { + log.Debug().Msg("caller already showed the links; not recording them for an agent hook") + } else if err := store.SavePendingLinks(links); err != nil { + log.Debug().Err(err).Msg("could not record session links for the agent hook") + } + + log.Debug().Msg("attestation pushed, wiping single-use trace state") + + // A later push of each session keeps the spec files recorded here, so + // that the limit never drops a spec that is already in the evidence. + for sessionID, fileNames := range attestedSpecs { + recordPushedSpecs(store, sessionID, fileNames, log) + } + + // Mark every AI commit included in this attestation as tracked so that a + // subsequent `git push` with no new commits short-circuits at the skip + // check above. Save errors are non-fatal: the attestation already went + // out, and at worst we re-attest the same commits on the next push. + for _, c := range aiCommits { + if c.Tracked { + continue + } + c.Tracked = true + if err := store.SaveCommitRecord(c); err != nil { + log.Debug().Err(err).Str("sha", c.SHA).Msg("could not mark commit record as tracked") + } + } + + _ = store.WipeTraceDir() + if liveSHAs, err := gitClient.LocalReachableSHAs(repoRoot); err == nil { + if err := store.GCOrphans(liveSHAs); err != nil { + log.Debug().Err(err).Msg("orphan GC failed; trace state is intact but not pruned") + } + } else { + log.Debug().Err(err).Msg("could not enumerate local branch SHAs; skipping orphan GC") + } + + return nil +} + +// sessionEvidence is the evidence of one session before it is added to the +// attestation. It is written out only once the attestation exists, because it +// records the digests of the spec and skill materials added to it first. +type sessionEvidence struct { + sessionID string + evidence *aicodingsession.Evidence + specs []spec.Capture + skills []sessionSkill +} + +// buildSessionEvidence parses each session of the push into its evidence, +// with the specs and the skills that go into the attestation next to it. A +// session that cannot be read is left out, with a warning in the log. +func buildSessionEvidence(ctx context.Context, store *state.Store, repoRoot string, gitClient tracegit.Client, sessionCommits map[string][]*state.CommitRecord, sessionRecords map[string]*state.SessionRecord, sessionMode string, log zerolog.Logger) []sessionEvidence { // Collect repo-level context once (same for all sessions) rawDir := store.RawSessionDir() @@ -649,15 +782,6 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts isGenerated := gitClient.GeneratedMatcher(repoRoot) - // Build evidence for each session. It is written out only once the - // attestation exists, because it records the digests of the spec materials - // added to it first. - type sessionEvidence struct { - sessionID string - evidence *aicodingsession.Evidence - specs []spec.Capture - skills []sessionSkill - } var sessions []sessionEvidence for sessionID, commits := range sessionCommits { @@ -756,71 +880,33 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts }) } - if len(sessions) == 0 { - log.Debug().Msg("no session evidence could be generated") - - return nil - } - - // Create attestation using local state file to avoid conflicts with other attestations - localStatePath := store.AttestationStatePath() - defer func() { _ = os.Remove(localStatePath) }() - - projectName, organization, workflowName := resolvePushIdentity(repoRoot, opts) - if projectName == "" { - if opts.IgnoreYAML { - return fmt.Errorf("no project name provided") - } - - return fmt.Errorf("no project name found (pass via options or .chainloop.yml)") - } - - log.Debug().Str("project", projectName).Msg("resolved project for trace push") - - execOpts := []ExecutorOption{WithLocalStatePath(localStatePath), WithLogger(log)} - if organization != "" { - log.Debug().Str("forced_org", organization).Msg("forcing organization for trace push") - execOpts = append(execOpts, WithForcedOrganization(organization)) - } - executor, err := NewAttestationExecutor(opts.ActionOpts, opts.CLIVersion, execOpts...) - if err != nil { - // Warn (not debug): a misconfiguration here means no attestation is - // sent at all, and the pre-push hook swallows the returned error - // unless require-trace is on. - log.Warn().Err(err).Msg("skipping trace attestation") - - return err - } - defer func() { _ = executor.Close() }() - - attestationID, err := executor.Init(ctx, workflowName, projectName, opts.ProjectVersion) - if err != nil { - return fmt.Errorf("attestation init: %w", err) - } - - log.Debug().Str("attestation_id", attestationID).Msg("attestation initialized") + return sessions +} - // Add evidence for each session: its spec materials first, so that the - // session material can record their digests, then the session itself. - attestedSessions := make([]string, 0, len(sessions)) - attestedSpecs := make(map[string][]string, len(sessions)) +// attachSessionEvidence adds the materials of each session to the attestation: +// its spec and skill materials first, so that the session material can record +// their digests, then the session itself. It returns the sessions that were +// added, and the spec files that each of them stored. +func attachSessionEvidence(ctx context.Context, adder specMaterialAdder, store *state.Store, sessions []sessionEvidence, log zerolog.Logger) (attestedSessions []string, attestedSpecs map[string][]string) { + attestedSessions = make([]string, 0, len(sessions)) + attestedSpecs = make(map[string][]string, len(sessions)) // One allocator for the whole attestation: names taken from the start of // a session ID can repeat across sessions, and a repeated name would // replace an earlier material. names := materials.NewNameAllocator(nil) for _, se := range sessions { redactor := newSpecRedactor(store.SpecRedactionDir(se.sessionID)) - entries, warnings, stored := attachSpecs(ctx, executor, redactor, names, se.sessionID, se.specs, log) + entries, warnings, stored := attachSpecs(ctx, adder, redactor, names, se.sessionID, se.specs, log) se.evidence.Data.Warnings = append(se.evidence.Data.Warnings, warnings...) // The skill entries come after the sources that the agent wrote. - skillEntries, skillWarnings := attachSkills(ctx, executor, redactor, names, se.sessionID, se.skills, log) + skillEntries, skillWarnings := attachSkills(ctx, adder, redactor, names, se.sessionID, se.skills, log) entries = append(entries, skillEntries...) se.evidence.Data.Spec = entries se.evidence.Data.Warnings = append(se.evidence.Data.Warnings, skillWarnings...) name := evidenceName(se.sessionID) - if err := addSessionEvidence(ctx, executor, name, se.evidence); err != nil { + if err := addSessionEvidence(ctx, adder, name, se.evidence); err != nil { // Warn, not debug: the session is left out of the attestation, // and this is the only place that says why. log.Warn().Err(err).Str("session", se.sessionID).Msg("could not add the evidence of an AI session; it is left out of the attestation") @@ -831,73 +917,7 @@ func RunTracePush(ctx context.Context, log zerolog.Logger, opts RunTracePushOpts log.Debug().Str("session", se.sessionID).Str("name", name).Int("spec_entries", len(entries)).Msg("evidence added") } - if len(attestedSessions) == 0 { - log.Debug().Msg("no evidence successfully added, resetting attestation") - _ = executor.Reset(ctx, AttestationResetTriggerCancelled, "no CHAINLOOP_AI_CODING_SESSION evidence added") - - return nil - } - - // Push attestation - log.Debug().Msg("pushing attestation") - res, err := executor.Push(ctx) - if err != nil { - return fmt.Errorf("attestation push: %w", err) - } - - // Tell the user where each session landed. The organization comes from the - // control plane rather than opts.Organization, which is empty whenever the - // CLI's current org is used. - links := logAttestedSessions(log, res.UIDashboardURL, res.GetOrganization(), attestedSessions) - - // Hand the links to the agent hook that runs after this push. When the - // push was driven by a coding agent's shell tool, the log line above is - // captured into that tool's output rather than shown to the user, so the - // hook is what actually puts the link in front of them. A failure here - // costs a notification, never the attestation that already succeeded. - // - // The caller tells us whether to skip, rather than us inferring it from - // on-disk state: the trace-run sentinel outlives a killed run, and - // reading it here would silently suppress every later notification in - // that repository. - if opts.SkipAgentNotification { - log.Debug().Msg("caller already showed the links; not recording them for an agent hook") - } else if err := store.SavePendingLinks(links); err != nil { - log.Debug().Err(err).Msg("could not record session links for the agent hook") - } - - log.Debug().Msg("attestation pushed, wiping single-use trace state") - - // A later push of each session keeps the spec files recorded here, so - // that the limit never drops a spec that is already in the evidence. - for sessionID, fileNames := range attestedSpecs { - recordPushedSpecs(store, sessionID, fileNames, log) - } - - // Mark every AI commit included in this attestation as tracked so that a - // subsequent `git push` with no new commits short-circuits at the skip - // check above. Save errors are non-fatal: the attestation already went - // out, and at worst we re-attest the same commits on the next push. - for _, c := range aiCommits { - if c.Tracked { - continue - } - c.Tracked = true - if err := store.SaveCommitRecord(c); err != nil { - log.Debug().Err(err).Str("sha", c.SHA).Msg("could not mark commit record as tracked") - } - } - - _ = store.WipeTraceDir() - if liveSHAs, err := gitClient.LocalReachableSHAs(repoRoot); err == nil { - if err := store.GCOrphans(liveSHAs); err != nil { - log.Debug().Err(err).Msg("orphan GC failed; trace state is intact but not pruned") - } - } else { - log.Debug().Err(err).Msg("could not enumerate local branch SHAs; skipping orphan GC") - } - - return nil + return attestedSessions, attestedSpecs } // logAttestedSessions reports one line per attested session. When the @@ -931,7 +951,7 @@ func sessionLinkMessage(url string) string { // addSessionEvidence writes one session's evidence to a temporary file and adds // it to the attestation. -func addSessionEvidence(ctx context.Context, executor *AttestationExecutor, name string, evidence *aicodingsession.Evidence) error { +func addSessionEvidence(ctx context.Context, adder specMaterialAdder, name string, evidence *aicodingsession.Evidence) error { tmpFile, err := os.CreateTemp("", "chainloop-trace-*.json") if err != nil { return fmt.Errorf("create temp file: %w", err) @@ -949,7 +969,7 @@ func addSessionEvidence(ctx context.Context, executor *AttestationExecutor, name return fmt.Errorf("write evidence: %w", err) } - _, err = executor.AddMaterial(ctx, name, tmpFile.Name(), "CHAINLOOP_AI_CODING_SESSION", nil) + _, err = adder.AddMaterial(ctx, name, tmpFile.Name(), "CHAINLOOP_AI_CODING_SESSION", nil) return err } diff --git a/app/cli/pkg/action/trace_skills_scenario_test.go b/app/cli/pkg/action/trace_skills_scenario_test.go new file mode 100644 index 000000000..d82afab4b --- /dev/null +++ b/app/cli/pkg/action/trace_skills_scenario_test.go @@ -0,0 +1,280 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package action + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" + tracegit "github.com/chainloop-dev/chainloop/app/cli/internal/trace/git" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/internal/schemavalidators" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// steLintScript is the script of the STE test skill. +const steLintScript = "scripts/ste-lint.py" + +// digestAdder records each material that the push adds, under its name, and +// returns the real digest of its content, as the attestation would. +type digestAdder struct { + added map[string]addedMaterial + order []string +} + +func (d *digestAdder) AddMaterial(_ context.Context, name, path, kind string, annotations map[string]string) (string, error) { + content, err := os.ReadFile(path) + if err != nil { + return "", err + } + + if d.added == nil { + d.added = make(map[string]addedMaterial) + } + d.added[name] = addedMaterial{name: name, fileName: filepath.Base(path), kind: kind, content: string(content), annotations: annotations} + d.order = append(d.order, name) + + sum := sha256.Sum256(content) + + return "sha256:" + hex.EncodeToString(sum[:]), nil +} + +// transcript appends Claude Code transcript records to the session file, in +// the shape that Claude Code 2.1 writes them. +type transcript struct { + t *testing.T + path string +} + +func (tr transcript) append(records ...string) { + tr.t.Helper() + + f, err := os.OpenFile(tr.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + require.NoError(tr.t, err) + defer func() { _ = f.Close() }() + + for _, r := range records { + _, err := f.WriteString(r + "\n") + require.NoError(tr.t, err) + } +} + +// jsonString quotes s as a JSON string, without escaping < and >, as Claude +// Code writes it. +func jsonString(t *testing.T, s string) string { + t.Helper() + + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + require.NoError(t, enc.Encode(s)) + + return strings.TrimSuffix(b.String(), "\n") +} + +// TestSkillScenario follows two skills from their use in a Claude Code session +// to the materials that the pre-push hook adds to the attestation. The model +// starts one skill with the Skill tool, and the user starts the other with a +// slash command. The hooks run as Claude Code runs them, with the payload on +// stdin, and the push runs every step up to the upload. +func TestSkillScenario(t *testing.T) { + const ( + sessionID = "0f3a6c1e-2b4d-4e8f-9a1b-2c3d4e5f6a7b" + // Assembled from fragments so that nothing credential-shaped is + // committed. + pat = "ghp_erOZlZv0B1e3amrQ" + "ugdwZ8Ro2W4kDql9WPTf" + ) + + home := t.TempDir() + t.Setenv("HOME", home) + + repoDir := initTempGitRepo(t) + require.NoError(t, state.NewGitStore(filepath.Join(repoDir, ".git")).InitTraceDir()) + t.Chdir(repoDir) + + // A user skill with a script and a version-control folder, and a project + // skill in the repository. + userSkill := makeSkillAt(t, filepath.Join(home, ".claude", "skills", "asd-ste100"), map[string]string{ + skill.DefinitionFile: "# STE\nAs it ran. Token " + pat + "\n", + steLintScript: "TOKEN = '" + pat + "'\n", + ".git/HEAD": "ref: refs/heads/main\n", + }) + projectSkill := makeSkillAt(t, filepath.Join(repoDir, ".claude", "skills", "release"), map[string]string{ + skill.DefinitionFile: "# Release\nCut a release.\n", + }) + + projectDir := filepath.Join(home, ".claude", "projects", strings.NewReplacer("/", "-", ".", "-").Replace(repoDir)) + require.NoError(t, os.MkdirAll(projectDir, 0o700)) + tr := transcript{t: t, path: filepath.Join(projectDir, sessionID+".jsonl")} + + hookPayload := func(event, tool string) string { + payload := `{"session_id":"` + sessionID + `","hook_event_name":"` + event + `","transcript_path":` + + jsonString(t, tr.path) + `,"cwd":` + jsonString(t, repoDir) + if tool != "" { + payload += `,"tool_name":"` + tool + `","tool_input":{"skill":"asd-ste100"}` + } + + return payload + "}" + } + + provider := claude.New() + + // The session starts. + tr.append(`{"type":"user","uuid":"u1","timestamp":"2026-10-07T10:00:00.000Z","sessionId":"` + sessionID + `","message":{"role":"user","content":"Rewrite the PR description in STE."}}`) + withStdin(t, hookPayload("SessionStart", "")) + captureStdout(t, func() { require.NoError(t, HandleAgentSessionStart(provider, zerolog.Nop())) }) + + // The model calls the Skill tool, Claude Code loads the skill, and the + // post-tool-use hook fires. + tr.append( + `{"type":"assistant","uuid":"a1","parentUuid":"u1","timestamp":"2026-10-07T10:01:00.000Z","message":{"model":"claude-opus-5-5","role":"assistant","content":[{"type":"tool_use","id":"toolu_1","name":"Skill","input":{"skill":"asd-ste100","args":"the PR description"}}],"usage":{"input_tokens":10,"output_tokens":5}}}`, + `{"type":"user","uuid":"r1","parentUuid":"a1","timestamp":"2026-10-07T10:01:01.000Z","toolUseResult":{"success":true,"commandName":"asd-ste100"},"message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"Launching skill: asd-ste100"}]}}`, + `{"type":"user","uuid":"m1","parentUuid":"r1","isMeta":true,"sourceToolUseID":"toolu_1","timestamp":"2026-10-07T10:01:01.000Z","message":{"role":"user","content":[{"type":"text","text":`+jsonString(t, "Base directory for this skill: "+userSkill+"\n\n# STE")+`}]}}`, + ) + withStdin(t, hookPayload("PostToolUse", "Skill")) + require.NoError(t, HandleAgentPostToolUse(provider, zerolog.Nop())) + + // The skill changes on disk after it ran. The evidence must hold the + // skill as it ran. + require.NoError(t, os.WriteFile(filepath.Join(userSkill, skill.DefinitionFile), []byte("# STE\nEdited after the use.\n"), 0o600)) + + // The user starts the project skill with a slash command, and the next + // prompt fires the prompt-submit hook. + tr.append( + `{"type":"user","uuid":"c1","timestamp":"2026-10-07T10:05:00.000Z","message":{"role":"user","content":"release\n/release"}}`, + `{"type":"user","uuid":"m2","parentUuid":"c1","isMeta":true,"timestamp":"2026-10-07T10:05:00.000Z","message":{"role":"user","content":[{"type":"text","text":`+jsonString(t, "Base directory for this skill: "+projectSkill+"\n\n# Release")+`}]}}`, + `{"type":"user","uuid":"u2","timestamp":"2026-10-07T10:06:00.000Z","message":{"role":"user","content":"thanks"}}`, + ) + withStdin(t, hookPayload("UserPromptSubmit", "")) + captureStdout(t, func() { require.NoError(t, HandleAgentPromptSubmit(provider, zerolog.Nop())) }) + + // The pre-push hook: the push builds the session evidence and adds the + // materials, as RunTracePush does once the attestation exists. + ctx := context.Background() + store, repoRoot, err := state.Locate() + require.NoError(t, err) + records, err := store.LoadAllSessionRecords() + require.NoError(t, err) + + sessions := buildSessionEvidence(ctx, store, repoRoot, tracegit.NewGoGitClient(), + map[string][]*state.CommitRecord{sessionID: nil}, records, aicodingsession.ModeCoding, zerolog.Nop()) + require.Len(t, sessions, 1) + + adder := &digestAdder{} + attested, _ := attachSessionEvidence(ctx, adder, store, sessions, zerolog.Nop()) + require.Equal(t, []string{sessionID}, attested) + + // Each skill gives its definition and its package, then the session + // material comes last. + assert.Equal(t, []string{ + "spec-0f3a6c-skill-asd-ste100", + "spec-0f3a6c-skill-asd-ste100-pkg", + "spec-0f3a6c-skill-release", + "spec-0f3a6c-skill-release-pkg", + "ai-coding-session-0f3a6c", + }, adder.order) + + definition := adder.added["spec-0f3a6c-skill-asd-ste100"] + assert.Equal(t, "EVIDENCE", definition.kind) + assert.Equal(t, skill.DefinitionFile, definition.fileName) + assert.Contains(t, definition.content, "As it ran.", "the evidence holds the skill as it ran") + assert.NotContains(t, definition.content, pat, "the definition is redacted") + assert.Equal(t, skillContentDefinition, definition.annotations[specAnnotationSkillContent]) + + pkg := adder.added["spec-0f3a6c-skill-asd-ste100-pkg"] + assert.Equal(t, "EVIDENCE", pkg.kind) + assert.Equal(t, "asd-ste100.tar.gz", pkg.fileName) + assert.Equal(t, skillContentPackage, pkg.annotations[specAnnotationSkillContent]) + files := untar(t, pkg.content) + assert.ElementsMatch(t, []string{skill.DefinitionFile, steLintScript}, mapKeys(files), "the package holds the full folder, without .git") + assert.Equal(t, definition.content, files[skill.DefinitionFile]) + assert.NotContains(t, files[steLintScript], pat, "each file of the package is redacted") + + // The session material holds the skill entries, which point to the + // materials above, and passes the schema. + session := adder.added["ai-coding-session-0f3a6c"] + assert.Equal(t, "CHAINLOOP_AI_CODING_SESSION", session.kind) + + var evidence aicodingsession.Evidence + require.NoError(t, json.Unmarshal([]byte(session.content), &evidence)) + require.Len(t, evidence.Data.Spec, 2) + + digestOf := func(name string) string { + sum := sha256.Sum256([]byte(adder.added[name].content)) + return "sha256:" + hex.EncodeToString(sum[:]) + } + + assert.Equal(t, aicodingsession.SpecEntry{ + Kind: aicodingsession.SpecKindSkill, Title: "asd-ste100", + Digest: digestOf("spec-0f3a6c-skill-asd-ste100"), CapturedAt: "2026-10-07T10:01:00Z", + Metadata: &aicodingsession.SpecMetadata{ + Source: aicodingsession.SkillSourceUser, PackageDigest: digestOf("spec-0f3a6c-skill-asd-ste100-pkg"), + InvocationCount: 1, ByModel: 1, + }, + }, evidence.Data.Spec[0]) + assert.Equal(t, aicodingsession.SpecEntry{ + Kind: aicodingsession.SpecKindSkill, Title: "release", + Digest: digestOf("spec-0f3a6c-skill-release"), CapturedAt: "2026-10-07T10:05:00Z", + Metadata: &aicodingsession.SpecMetadata{ + Source: aicodingsession.SkillSourceProject, PackageDigest: digestOf("spec-0f3a6c-skill-release-pkg"), + InvocationCount: 1, ByUser: 1, + }, + }, evidence.Data.Spec[1]) + + for _, w := range evidence.Data.Warnings { + assert.NotContains(t, w, "skill", "the hooks copied both skills at their use") + } + + // The material crafter validates the data of the envelope, decoded as + // generic JSON so that an unknown field fails the schema. + var envelope struct { + Data any `json:"data"` + } + require.NoError(t, json.Unmarshal([]byte(session.content), &envelope)) + require.NoError(t, schemavalidators.ValidateAICodingSession(envelope.Data, schemavalidators.AICodingSessionVersion0_1)) +} + +func makeSkillAt(t *testing.T, dir string, files map[string]string) string { + t.Helper() + + for rel, content := range files { + path := filepath.Join(dir, filepath.FromSlash(rel)) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + } + + return dir +} + +func mapKeys(m map[string]string) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + + return keys +} diff --git a/app/cli/pkg/action/trace_skills_test.go b/app/cli/pkg/action/trace_skills_test.go index a946f6f64..322ca0b2f 100644 --- a/app/cli/pkg/action/trace_skills_test.go +++ b/app/cli/pkg/action/trace_skills_test.go @@ -233,8 +233,8 @@ func TestAttachSkills(t *testing.T) { t.Run("each skill gives a definition, a package and an entry", func(t *testing.T) { s := stored(t, map[string]string{ - skill.DefinitionFile: "# STE\ntoken " + pat + "\n", - "scripts/ste-lint.py": "print('" + pat + "')", + skill.DefinitionFile: "# STE\ntoken " + pat + "\n", + steLintScript: "print('" + pat + "')", }, trace.SkillUse{Name: steSkillName, FirstUsedAt: "2026-10-07T10:14:51Z", ByModel: 2, ByUser: 1, InSubagents: 1}) adder := &fakeMaterialAdder{} @@ -261,7 +261,7 @@ func TestAttachSkills(t *testing.T) { files := untar(t, pkg.content) assert.Equal(t, definition.content, files[skill.DefinitionFile], "the definition and the packaged SKILL.md are the same bytes") - assert.NotContains(t, files["scripts/ste-lint.py"], pat, "each text file is redacted") + assert.NotContains(t, files[steLintScript], pat, "each text file is redacted") assert.Equal(t, []aicodingsession.SpecEntry{{ Kind: aicodingsession.SpecKindSkill, From 963c129282188eed2078195ea333185cd32bc067 Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Wed, 7 Oct 2026 18:50:15 +0200 Subject: [PATCH 3/4] test(trace): reuse the existing test helpers in the skill scenario Use one fake material adder, with an option for real digests, and the existing helpers for the Claude Code project folder and skill folders. Refs #3540 Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: ee05bc4f-cf05-4433-b614-415b89577db7 --- .../pkg/action/trace_skills_scenario_test.go | 109 +++++------------- app/cli/pkg/action/trace_skills_test.go | 9 +- .../pkg/action/trace_spec_materials_test.go | 38 +++++- 3 files changed, 70 insertions(+), 86 deletions(-) diff --git a/app/cli/pkg/action/trace_skills_scenario_test.go b/app/cli/pkg/action/trace_skills_scenario_test.go index d82afab4b..8ddc867c6 100644 --- a/app/cli/pkg/action/trace_skills_scenario_test.go +++ b/app/cli/pkg/action/trace_skills_scenario_test.go @@ -17,12 +17,11 @@ package action import ( "context" - "crypto/sha256" - "encoding/hex" "encoding/json" + "maps" "os" "path/filepath" - "strings" + "slices" "testing" "github.com/chainloop-dev/chainloop/app/cli/internal/trace/claude" @@ -39,30 +38,6 @@ import ( // steLintScript is the script of the STE test skill. const steLintScript = "scripts/ste-lint.py" -// digestAdder records each material that the push adds, under its name, and -// returns the real digest of its content, as the attestation would. -type digestAdder struct { - added map[string]addedMaterial - order []string -} - -func (d *digestAdder) AddMaterial(_ context.Context, name, path, kind string, annotations map[string]string) (string, error) { - content, err := os.ReadFile(path) - if err != nil { - return "", err - } - - if d.added == nil { - d.added = make(map[string]addedMaterial) - } - d.added[name] = addedMaterial{name: name, fileName: filepath.Base(path), kind: kind, content: string(content), annotations: annotations} - d.order = append(d.order, name) - - sum := sha256.Sum256(content) - - return "sha256:" + hex.EncodeToString(sum[:]), nil -} - // transcript appends Claude Code transcript records to the session file, in // the shape that Claude Code 2.1 writes them. type transcript struct { @@ -83,19 +58,6 @@ func (tr transcript) append(records ...string) { } } -// jsonString quotes s as a JSON string, without escaping < and >, as Claude -// Code writes it. -func jsonString(t *testing.T, s string) string { - t.Helper() - - var b strings.Builder - enc := json.NewEncoder(&b) - enc.SetEscapeHTML(false) - require.NoError(t, enc.Encode(s)) - - return strings.TrimSuffix(b.String(), "\n") -} - // TestSkillScenario follows two skills from their use in a Claude Code session // to the materials that the pre-push hook adds to the attestation. The model // starts one skill with the Skill tool, and the user starts the other with a @@ -127,18 +89,27 @@ func TestSkillScenario(t *testing.T) { skill.DefinitionFile: "# Release\nCut a release.\n", }) - projectDir := filepath.Join(home, ".claude", "projects", strings.NewReplacer("/", "-", ".", "-").Replace(repoDir)) + projectDir := claudeProjectDir(home, repoDir) require.NoError(t, os.MkdirAll(projectDir, 0o700)) tr := transcript{t: t, path: filepath.Join(projectDir, sessionID+".jsonl")} - hookPayload := func(event, tool string) string { - payload := `{"session_id":"` + sessionID + `","hook_event_name":"` + event + `","transcript_path":` + - jsonString(t, tr.path) + `,"cwd":` + jsonString(t, repoDir) - if tool != "" { - payload += `,"tool_name":"` + tool + `","tool_input":{"skill":"asd-ste100"}` - } + quote := func(s string) string { + b, err := json.Marshal(s) + require.NoError(t, err) + return string(b) + } - return payload + "}" + // hookPayload is what Claude Code writes to the stdin of a hook. + hookPayload := func(event, tool string) string { + b, err := json.Marshal(struct { + SessionID string `json:"session_id"` + HookEventName string `json:"hook_event_name"` + TranscriptPath string `json:"transcript_path"` + Cwd string `json:"cwd"` + ToolName string `json:"tool_name,omitempty"` + }{sessionID, event, tr.path, repoDir, tool}) + require.NoError(t, err) + return string(b) } provider := claude.New() @@ -153,7 +124,7 @@ func TestSkillScenario(t *testing.T) { tr.append( `{"type":"assistant","uuid":"a1","parentUuid":"u1","timestamp":"2026-10-07T10:01:00.000Z","message":{"model":"claude-opus-5-5","role":"assistant","content":[{"type":"tool_use","id":"toolu_1","name":"Skill","input":{"skill":"asd-ste100","args":"the PR description"}}],"usage":{"input_tokens":10,"output_tokens":5}}}`, `{"type":"user","uuid":"r1","parentUuid":"a1","timestamp":"2026-10-07T10:01:01.000Z","toolUseResult":{"success":true,"commandName":"asd-ste100"},"message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"Launching skill: asd-ste100"}]}}`, - `{"type":"user","uuid":"m1","parentUuid":"r1","isMeta":true,"sourceToolUseID":"toolu_1","timestamp":"2026-10-07T10:01:01.000Z","message":{"role":"user","content":[{"type":"text","text":`+jsonString(t, "Base directory for this skill: "+userSkill+"\n\n# STE")+`}]}}`, + `{"type":"user","uuid":"m1","parentUuid":"r1","isMeta":true,"sourceToolUseID":"toolu_1","timestamp":"2026-10-07T10:01:01.000Z","message":{"role":"user","content":[{"type":"text","text":`+quote("Base directory for this skill: "+userSkill+"\n\n# STE")+`}]}}`, ) withStdin(t, hookPayload("PostToolUse", "Skill")) require.NoError(t, HandleAgentPostToolUse(provider, zerolog.Nop())) @@ -166,7 +137,7 @@ func TestSkillScenario(t *testing.T) { // prompt fires the prompt-submit hook. tr.append( `{"type":"user","uuid":"c1","timestamp":"2026-10-07T10:05:00.000Z","message":{"role":"user","content":"release\n/release"}}`, - `{"type":"user","uuid":"m2","parentUuid":"c1","isMeta":true,"timestamp":"2026-10-07T10:05:00.000Z","message":{"role":"user","content":[{"type":"text","text":`+jsonString(t, "Base directory for this skill: "+projectSkill+"\n\n# Release")+`}]}}`, + `{"type":"user","uuid":"m2","parentUuid":"c1","isMeta":true,"timestamp":"2026-10-07T10:05:00.000Z","message":{"role":"user","content":[{"type":"text","text":`+quote("Base directory for this skill: "+projectSkill+"\n\n# Release")+`}]}}`, `{"type":"user","uuid":"u2","timestamp":"2026-10-07T10:06:00.000Z","message":{"role":"user","content":"thanks"}}`, ) withStdin(t, hookPayload("UserPromptSubmit", "")) @@ -184,7 +155,7 @@ func TestSkillScenario(t *testing.T) { map[string][]*state.CommitRecord{sessionID: nil}, records, aicodingsession.ModeCoding, zerolog.Nop()) require.Len(t, sessions, 1) - adder := &digestAdder{} + adder := &fakeMaterialAdder{realDigests: true} attested, _ := attachSessionEvidence(ctx, adder, store, sessions, zerolog.Nop()) require.Equal(t, []string{sessionID}, attested) @@ -196,37 +167,34 @@ func TestSkillScenario(t *testing.T) { "spec-0f3a6c-skill-release", "spec-0f3a6c-skill-release-pkg", "ai-coding-session-0f3a6c", - }, adder.order) + }, adder.names()) - definition := adder.added["spec-0f3a6c-skill-asd-ste100"] + definition := adder.byName("spec-0f3a6c-skill-asd-ste100") assert.Equal(t, "EVIDENCE", definition.kind) assert.Equal(t, skill.DefinitionFile, definition.fileName) assert.Contains(t, definition.content, "As it ran.", "the evidence holds the skill as it ran") assert.NotContains(t, definition.content, pat, "the definition is redacted") assert.Equal(t, skillContentDefinition, definition.annotations[specAnnotationSkillContent]) - pkg := adder.added["spec-0f3a6c-skill-asd-ste100-pkg"] + pkg := adder.byName("spec-0f3a6c-skill-asd-ste100-pkg") assert.Equal(t, "EVIDENCE", pkg.kind) assert.Equal(t, "asd-ste100.tar.gz", pkg.fileName) assert.Equal(t, skillContentPackage, pkg.annotations[specAnnotationSkillContent]) files := untar(t, pkg.content) - assert.ElementsMatch(t, []string{skill.DefinitionFile, steLintScript}, mapKeys(files), "the package holds the full folder, without .git") + assert.ElementsMatch(t, []string{skill.DefinitionFile, steLintScript}, slices.Collect(maps.Keys(files)), "the package holds the full folder, without .git") assert.Equal(t, definition.content, files[skill.DefinitionFile]) assert.NotContains(t, files[steLintScript], pat, "each file of the package is redacted") // The session material holds the skill entries, which point to the // materials above, and passes the schema. - session := adder.added["ai-coding-session-0f3a6c"] + session := adder.byName("ai-coding-session-0f3a6c") assert.Equal(t, "CHAINLOOP_AI_CODING_SESSION", session.kind) var evidence aicodingsession.Evidence require.NoError(t, json.Unmarshal([]byte(session.content), &evidence)) require.Len(t, evidence.Data.Spec, 2) - digestOf := func(name string) string { - sum := sha256.Sum256([]byte(adder.added[name].content)) - return "sha256:" + hex.EncodeToString(sum[:]) - } + digestOf := func(name string) string { return adder.byName(name).digest } assert.Equal(t, aicodingsession.SpecEntry{ Kind: aicodingsession.SpecKindSkill, Title: "asd-ste100", @@ -257,24 +225,3 @@ func TestSkillScenario(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(session.content), &envelope)) require.NoError(t, schemavalidators.ValidateAICodingSession(envelope.Data, schemavalidators.AICodingSessionVersion0_1)) } - -func makeSkillAt(t *testing.T, dir string, files map[string]string) string { - t.Helper() - - for rel, content := range files { - path := filepath.Join(dir, filepath.FromSlash(rel)) - require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) - } - - return dir -} - -func mapKeys(m map[string]string) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - - return keys -} diff --git a/app/cli/pkg/action/trace_skills_test.go b/app/cli/pkg/action/trace_skills_test.go index 322ca0b2f..555692f1a 100644 --- a/app/cli/pkg/action/trace_skills_test.go +++ b/app/cli/pkg/action/trace_skills_test.go @@ -75,7 +75,14 @@ type noSkillProvider struct { func makeSkill(t *testing.T, files map[string]string) string { t.Helper() - dir := t.TempDir() + return makeSkillAt(t, t.TempDir(), files) +} + +// makeSkillAt writes a skill folder at dir from a map of relative paths to +// contents. +func makeSkillAt(t *testing.T, dir string, files map[string]string) string { + t.Helper() + for rel, content := range files { path := filepath.Join(dir, filepath.FromSlash(rel)) require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) diff --git a/app/cli/pkg/action/trace_spec_materials_test.go b/app/cli/pkg/action/trace_spec_materials_test.go index cc6030ec0..255da9cbb 100644 --- a/app/cli/pkg/action/trace_spec_materials_test.go +++ b/app/cli/pkg/action/trace_spec_materials_test.go @@ -36,14 +36,38 @@ import ( // addedMaterial is one call recorded by fakeMaterialAdder, with the file read // back at call time: the caller is free to delete it once the add returns. type addedMaterial struct { - name, fileName, kind, content string - annotations map[string]string + name, fileName, kind, content, digest string + annotations map[string]string } type fakeMaterialAdder struct { added []addedMaterial // failOn names the materials whose add fails. failOn map[string]bool + // realDigests returns the SHA-256 digest of the content, as the + // attestation does, instead of a digest made from the name. + realDigests bool +} + +// byName returns the material added under name. +func (f *fakeMaterialAdder) byName(name string) addedMaterial { + for _, m := range f.added { + if m.name == name { + return m + } + } + + return addedMaterial{} +} + +// names returns the names of the added materials, in the order of the adds. +func (f *fakeMaterialAdder) names() []string { + out := make([]string, 0, len(f.added)) + for _, m := range f.added { + out = append(out, m.name) + } + + return out } func (f *fakeMaterialAdder) AddMaterial(_ context.Context, name, path, kind string, annotations map[string]string) (string, error) { @@ -56,11 +80,17 @@ func (f *fakeMaterialAdder) AddMaterial(_ context.Context, name, path, kind stri return "", err } + digest := "sha256:" + name + if f.realDigests { + sum := sha256.Sum256(content) + digest = "sha256:" + hex.EncodeToString(sum[:]) + } + f.added = append(f.added, addedMaterial{ - name: name, fileName: filepath.Base(path), kind: kind, content: string(content), annotations: annotations, + name: name, fileName: filepath.Base(path), kind: kind, content: string(content), digest: digest, annotations: annotations, }) - return "sha256:" + name, nil + return digest, nil } // specCapture builds a capture the way the spec folder reader does: parsed from From e976c76a2a1fb3831d735a3eb2cfae0a2f4839bb Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Wed, 7 Oct 2026 18:58:39 +0200 Subject: [PATCH 4/4] feat(trace): log a warning for a skill with no folder A skill that is part of the agent, such as /simplify in Claude Code, has no folder on disk. The CLI does not record it, and writes a warning to the local log so that the gap is visible. Refs #3540 Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: ee05bc4f-cf05-4433-b614-415b89577db7 --- app/cli/internal/trace/claude/skills.go | 37 +++++++++++--------- app/cli/internal/trace/claude/skills_test.go | 27 +++++++++++--- app/cli/internal/trace/provider.go | 3 +- app/cli/pkg/action/trace_skills.go | 18 +++++++--- app/cli/pkg/action/trace_skills_test.go | 7 ++-- 5 files changed, 63 insertions(+), 29 deletions(-) diff --git a/app/cli/internal/trace/claude/skills.go b/app/cli/internal/trace/claude/skills.go index 1d8f3ebe9..48cab1805 100644 --- a/app/cli/internal/trace/claude/skills.go +++ b/app/cli/internal/trace/claude/skills.go @@ -111,10 +111,11 @@ func (r *skillRecord) blocks() []skillBlock { } // mayHoldSkillUse is a cheap test that a scan runs before it decodes a line. -// Only the lines that start or load a skill pass it. The command marker is -// matched without its brackets, which a JSON encoder can escape. +// Only the lines that start or load a skill pass it: the meta messages, the +// skill tool calls and the slash commands. The command marker is matched +// without its brackets, which a JSON encoder can escape. func mayHoldSkillUse(line []byte) bool { - return skill.HasBaseDir(line) || + return bytes.Contains(line, []byte(`"isMeta":true`)) || bytes.Contains(line, []byte(`"name":"`+skillTool+`"`)) || bytes.Contains(line, []byte("command-name")) } @@ -128,13 +129,17 @@ type start struct { // transcriptSkills counts the skill uses of one transcript. // -// Claude Code writes a meta message that starts with "Base directory for this -// skill:" each time it loads a skill. A meta message that names a tool call -// is a model start, and one whose parent is a slash command is a user start. -// The meta message is what tells a skill command apart from a built-in or a -// custom command, and a loaded skill apart from a failed tool call, which -// gets no meta message. Each start counts once, also when the transcript -// holds its meta message more than once. +// Claude Code writes a meta message each time it loads a skill. A meta message +// that names a skill tool call is a model start, and one whose parent is a +// slash command is a user start. A built-in command such as /clear gets no +// meta message, and a failed tool call loads nothing, so neither counts. Each +// start counts once, also when the transcript holds its meta message more than +// once. +// +// The meta message of a skill with a folder starts with "Base directory for +// this skill:". A skill built into Claude Code, such as /simplify, and a +// custom command have no folder: their use is returned with no Dir, and is +// not recorded. func transcriptSkills(path string, subagent bool) (map[string]*trace.SkillUse, error) { records, err := readSkillRecords(path) if err != nil { @@ -151,10 +156,7 @@ func transcriptSkills(path string, subagent bool) (map[string]*trace.SkillUse, e continue } - dir, ok := skill.BaseDir(r.text()) - if !ok { - continue - } + dir, _ := skill.BaseDir(r.text()) s, key, byModel, ok := startOf(r, dir, toolCalls, commands) if !ok || counted[key] { @@ -169,9 +171,12 @@ func transcriptSkills(path string, subagent bool) (map[string]*trace.SkillUse, e use, ok := uses[s.name] if !ok { - use = &trace.SkillUse{Name: s.name, Dir: dir} + use = &trace.SkillUse{Name: s.name} uses[s.name] = use } + if use.Dir == "" { + use.Dir = dir + } use.Record(at, byModel, subagent) } @@ -239,7 +244,7 @@ func skillStarts(records []skillRecord) (toolCalls, commands map[string]start) { func startOf(r *skillRecord, dir string, toolCalls, commands map[string]start) (s start, key string, byModel, ok bool) { if r.SourceToolUseID != "" { s, ok = toolCalls[r.SourceToolUseID] - if !ok { + if !ok && dir != "" { // The tool call is not in this transcript. The meta message still // tells that a skill loaded, and its folder names it. s = start{name: filepath.Base(dir), timestamp: r.Timestamp} diff --git a/app/cli/internal/trace/claude/skills_test.go b/app/cli/internal/trace/claude/skills_test.go index 1666cb2a1..6103cc11f 100644 --- a/app/cli/internal/trace/claude/skills_test.go +++ b/app/cli/internal/trace/claude/skills_test.go @@ -35,6 +35,7 @@ const ( graphSkill = "graphify" tenOClock = "2026-10-07T10:00:00Z" tenOClockM = "2026-10-07T10:00:00.000Z" + tenOhOne = "2026-10-07T10:01:00Z" ) // fixtureRecord, fixtureMessage and fixtureBlock have the shape that Claude @@ -102,14 +103,19 @@ func commandLine(t *testing.T, uuid, command, ts string) string { } // metaLine is the message that holds a loaded skill. toolID is empty for a -// skill that the user loaded. +// skill that the user loaded, and dir for a skill with no folder. func metaLine(t *testing.T, uuid, parent, toolID, dir, ts string) string { t.Helper() + text := "Base directory for this skill: " + dir + "\n\n# The skill\n\nBody." + if dir == "" { + text = "`/simplify` Review the changed code.\n\nBody." + } + return mustLine(t, fixtureRecord{ Type: recordTypeUser, UUID: uuid, ParentUUID: parent, Timestamp: ts, IsMeta: true, SourceToolUseID: toolID, Message: fixtureMessage{Role: recordTypeUser, Content: []fixtureBlock{{ - Type: "text", Text: "Base directory for this skill: " + dir + "\n\n# The skill\n\nBody.", + Type: "text", Text: text, }}}, }) } @@ -221,7 +227,20 @@ func TestSessionSkills(t *testing.T) { ), want: []trace.SkillUse{ {Name: steSkill, Dir: steDir, FirstUsedAt: tenOClock, ByModel: 1}, - {Name: graphSkill, Dir: graphDir, FirstUsedAt: "2026-10-07T10:01:00Z", ByUser: 1}, + {Name: graphSkill, Dir: graphDir, FirstUsedAt: tenOhOne, ByUser: 1}, + }, + }, + { + // A skill built into Claude Code, or a custom command, has no + // folder. Its use is returned without one, and is not recorded. + name: "a skill with no folder", + main: join( + userStart(t, "1", "simplify", "", tenOClockM), + modelStart(t, "2", "code-review", "", "2026-10-07T10:01:00.000Z"), + ), + want: []trace.SkillUse{ + {Name: "simplify", FirstUsedAt: tenOClock, ByUser: 1}, + {Name: "code-review", FirstUsedAt: tenOhOne, ByModel: 1}, }, }, { @@ -237,7 +256,7 @@ func TestSessionSkills(t *testing.T) { "agent-a2.jsonl": modelStart(t, "s2", graphSkill, graphDir, "2026-10-07T10:09:00.000Z"), }, want: []trace.SkillUse{ - {Name: steSkill, Dir: steDir, FirstUsedAt: "2026-10-07T10:01:00Z", ByModel: 2, InSubagents: 1}, + {Name: steSkill, Dir: steDir, FirstUsedAt: tenOhOne, ByModel: 2, InSubagents: 1}, {Name: graphSkill, Dir: graphDir, FirstUsedAt: "2026-10-07T10:09:00Z", ByModel: 1, InSubagents: 1}, }, }, diff --git a/app/cli/internal/trace/provider.go b/app/cli/internal/trace/provider.go index 53452d9ef..d3c579fc7 100644 --- a/app/cli/internal/trace/provider.go +++ b/app/cli/internal/trace/provider.go @@ -185,7 +185,8 @@ type SkillUse struct { // Name is the name that the agent used for the skill. Name string // Dir is the folder that the agent loaded the skill from, or "" when the - // session data does not tell. + // skill has no folder (a skill built into the agent) or the session data + // does not tell. A use with no folder is not recorded. Dir string // FirstUsedAt is the time of the first use, RFC3339. FirstUsedAt string diff --git a/app/cli/pkg/action/trace_skills.go b/app/cli/pkg/action/trace_skills.go index d8ba416ec..828c2f4f3 100644 --- a/app/cli/pkg/action/trace_skills.go +++ b/app/cli/pkg/action/trace_skills.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strconv" "github.com/chainloop-dev/chainloop/app/cli/internal/trace" @@ -120,6 +121,18 @@ func readSessionSkills(provider trace.Provider, store *state.Store, opts *trace. return nil, []string{"the skills that the session used were not recorded: the session data could not be read"} } + // A skill with no folder, such as a skill built into the agent, has + // nothing to store. It is left out with a warning in the local log only. + uses = slices.DeleteFunc(uses, func(use trace.SkillUse) bool { + if use.Dir != "" { + return false + } + + log.Warn().Str("session", opts.SessionID).Str("skill", use.Name).Msg("a skill that the session used has no skill folder, so it is not recorded") + + return true + }) + if len(uses) > maxSkillEntries { // The skills that the session used first are kept. warnings = append(warnings, fmt.Sprintf("%d skill entries beyond the first %d were dropped", len(uses)-maxSkillEntries, maxSkillEntries)) @@ -130,11 +143,6 @@ func readSessionSkills(provider trace.Provider, store *state.Store, opts *trace. skills := make([]sessionSkill, 0, len(uses)) for _, use := range uses { - if use.Dir == "" { - warnings = append(warnings, fmt.Sprintf("skill %q was not recorded: the session data does not tell its folder", use.Name)) - continue - } - dst := store.SkillCopyDir(opts.SessionID, use.Dir) info, err := skill.Load(dst) switch { diff --git a/app/cli/pkg/action/trace_skills_test.go b/app/cli/pkg/action/trace_skills_test.go index 555692f1a..b644306ee 100644 --- a/app/cli/pkg/action/trace_skills_test.go +++ b/app/cli/pkg/action/trace_skills_test.go @@ -169,13 +169,14 @@ func TestReadSessionSkills(t *testing.T) { }} captureSkillLoads(&skillProvider{loads: []string{dir}}, store, &trace.HookInput{SessionID: skillSessionID}, zerolog.Nop()) - skills, warnings := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.Nop()) + var logs bytes.Buffer + skills, warnings := readSessionSkills(provider, store, &trace.ParseOpts{SessionID: skillSessionID}, t.TempDir(), zerolog.New(&logs)) require.Len(t, skills, 1) assert.Equal(t, steSkillName, skills[0].use.Name) assert.Equal(t, []string{ `skill "gone" was not recorded: its folder could not be read`, - `skill "unknown-dir" was not recorded: the session data does not tell its folder`, - }, warnings) + }, warnings, "a skill with no folder is only in the local log") + assert.Contains(t, logs.String(), `"level":"warn","session":"`+skillSessionID+`","skill":"unknown-dir","message":"a skill that the session used has no skill folder, so it is not recorded"`) for _, w := range warnings { assert.NotContains(t, w, string(filepath.Separator)+"gone", "a warning holds no local path") }