diff --git a/app/cli/Makefile b/app/cli/Makefile index be3e15ce2..db2eee42f 100644 --- a/app/cli/Makefile +++ b/app/cli/Makefile @@ -12,6 +12,11 @@ build: test: go test ./... +.PHONY: test-trace-e2e +# run the Chainloop Trace end-to-end suite with real agents and models (uses model tokens) +test-trace-e2e: + go test -tags e2e -run TestTraceMatrix -timeout 60m -v ./internal/trace/e2e/ + .PHONY: lint # lint lint: diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index fa8d84a1d..e7729c171 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -128,3 +128,21 @@ Why not Yes: | OpenCode | `.opencode/plugins/chainloop-trace.ts` | Session start and end, user prompt, and before and after each tool. One plugin covers both versions: `server()` for 1.x (1.3.4 or later) and `setup()` for 2. The provider picks the export command from `opencode --version`. | All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`. + +## End-to-end tests + +The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code column only. + +A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. The suite tests a cell marked **Yes**, **Partial**, or **Unknown** only when the row has a registered check. It reports the other cells as follows: + +- A cell marked **No** or **Not applicable** is `skipped`. +- A feature that needs a control plane is `not covered`, because the suite uses local storage mode only. +- A row with no registered check is `no check`. + +`e2e/config.yaml` sets the version of each agent that the suite expects, and the models. The suite does not install the agent. It runs the installed version, and it logs a warning when that version is not the one in the configuration. The suite uses model tokens. To run it: + +```bash +make -C app/cli test-trace-e2e +``` + +`CHAINLOOP_E2E_HARNESS`, `CHAINLOOP_E2E_MODEL`, and `CHAINLOOP_E2E_FEATURE` limit a run to some columns, models, or features (comma-separated). Set `CHAINLOOP_E2E_WORK_DIR` to keep the workspaces and the agent output. The results go to `e2e/results/`, or to `CHAINLOOP_E2E_RESULTS_DIR`. Only a run with no filter becomes the previous result of the next comparison. diff --git a/app/cli/internal/trace/e2e/.gitignore b/app/cli/internal/trace/e2e/.gitignore new file mode 100644 index 000000000..fbca22537 --- /dev/null +++ b/app/cli/internal/trace/e2e/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go new file mode 100644 index 000000000..97ed067a4 --- /dev/null +++ b/app/cli/internal/trace/e2e/checks.go @@ -0,0 +1,309 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +// errNotCovered marks a check that cannot see the feature in this run, for +// example because the harness does not report hook messages in headless mode +// (R-004). +var errNotCovered = errors.New("not covered") + +// Session is the result of one scenario session. +type Session struct { + Scenario string + Model string + Workspace *Workspace + Output *RunOutput + // Stored is the session that the push stored in local storage mode. + Stored *StoredSession + // HeadMessage is the message of the commit that the agent made. + HeadMessage string +} + +// Check verifies one feature on the result of one scenario session. +type Check struct { + Scenario string + Verify func(s *Session) error +} + +// Feature is the test definition of one feature row of the table. The key of +// the features map is the feature name in the table (R-002). +type Feature struct { + // ControlPlane is true for a feature that needs a control plane. The suite + // runs in local storage mode only, so its cells are "not covered" (D-007). + ControlPlane bool + // ModelDriven is true for a feature that depends on the model following an + // instruction. Its cells run more than once (R-006). + ModelDriven bool + // Checks must all pass for a run of the feature to pass. + Checks []Check +} + +// features are the test definitions of the feature rows. A row of the table +// with no entry here is reported as "no check". +var features = map[string]Feature{ + // Session collection + "Transcript": {Checks: []Check{{scenarioEditAndCommit, checkTranscript}}}, + "Token usage and cost": {Checks: []Check{{scenarioEditAndCommit, checkTokenUsage}}}, + "Model detection": {Checks: []Check{{scenarioEditAndCommit, checkModel}}}, + + // Code attribution + "Line attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("greeting.txt")}}}, + "Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}}, + + // User experience + "Welcome message": {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}}, + "Session link after push": {ControlPlane: true}, + "Credentials warning": {ControlPlane: true}, + + // Specs and skills + "Spec capture instruction": {ModelDriven: true, Checks: []Check{{scenarioTicket, checkTicketCaptured}}}, + "Spec capture reminder": {Checks: []Check{{scenarioEditAndCommit, checkPromptReminder}}}, + "Local spec sources read at push": {ModelDriven: true, Checks: []Check{{scenarioTicket, checkTicketContentStored}}}, + "Skills tracking": {ModelDriven: true, Checks: []Check{{scenarioSkill, checkSkillTracked}}}, + + // Security + "Secret redaction": {Checks: []Check{{scenarioEditAndCommit, checkSecretRedacted}}}, + "Allow markers in redaction": {Checks: []Check{{scenarioEditAndCommit, checkAllowedSecretKept}}}, + + // Git integration + "Commit trailer": {Checks: []Check{{scenarioEditAndCommit, checkCommitTrailer}}}, + "Attestation at push": {ControlPlane: true}, + "Local storage at push": {Checks: []Check{{scenarioEditAndCommit, checkLocalStorage}}}, +} + +func evidence(s *Session) *aicodingsession.Data { + return &s.Stored.Evidence.Data +} + +func checkTranscript(s *Session) error { + main := evidence(s).RawSession["main"] + if len(main) == 0 { + return errors.New("the evidence has no main transcript") + } + for _, msg := range main { + if bytes.Contains(msg, []byte("greeting.txt")) { + return nil + } + } + return errors.New("the transcript does not show the work of the session") +} + +func checkTokenUsage(s *Session) error { + u := evidence(s).Usage + if u == nil || u.TotalTokens == 0 { + return errors.New("the evidence has no token usage") + } + if u.EstimatedCostUSD <= 0 { + return errors.New("the evidence has no cost") + } + return nil +} + +func checkModel(s *Session) error { + m := evidence(s).Model + if m == nil || m.Primary == "" { + return errors.New("the evidence has no primary model") + } + if !strings.Contains(strings.ToLower(m.Primary), strings.ToLower(s.Model)) { + return fmt.Errorf("primary model %q is not the model of the session %q", m.Primary, s.Model) + } + return nil +} + +func checkFileAttributedToAI(path string) func(s *Session) error { + return func(s *Session) error { + cc := evidence(s).CodeChanges + if cc == nil { + return errors.New("the evidence has no code changes") + } + for _, f := range cc.Files { + if f.Path != path { + continue + } + if f.Attribution != trace.AttributionAI || f.LinesAdded == 0 { + return fmt.Errorf("%s: attribution %q with %d lines added, want AI lines", path, f.Attribution, f.LinesAdded) + } + return nil + } + return fmt.Errorf("the code changes do not list %s", path) + } +} + +// hookMessage returns the first hook message of an event, or errNotCovered +// when the harness does not report hook messages. +func hookMessage(s *Session, event string) (*HookMessage, error) { + if !s.Output.HooksReported { + return nil, fmt.Errorf("%w: the harness does not report hook messages in headless mode", errNotCovered) + } + for i := range s.Output.Hooks { + if s.Output.Hooks[i].Event == event { + return &s.Output.Hooks[i], nil + } + } + return nil, fmt.Errorf("no %s hook message", event) +} + +// checkWelcomeMessage checks the banner that the user sees at session start. +// In local storage mode, it names the on-disk destination. +func checkWelcomeMessage(s *Session) error { + msg, err := hookMessage(s, "SessionStart") + if err != nil { + return err + } + if !strings.Contains(msg.UserMessage, "local-sessions") { + return fmt.Errorf("the session start message does not name the local sessions folder: %q", msg.UserMessage) + } + return nil +} + +func checkPromptReminder(s *Session) error { + msg, err := hookMessage(s, "UserPromptSubmit") + if err != nil { + return err + } + if msg.ModelContext == "" { + return errors.New("the user prompt hook gave no reminder to the model") + } + return nil +} + +// ticketSpec returns the captured spec entry of the ticket of the scenario. +func ticketSpec(s *Session) (*aicodingsession.SpecEntry, error) { + for i, e := range evidence(s).Spec { + if e.Kind != aicodingsession.SpecKindSkill && strings.Contains(e.URI, filepath.Base(ticketPath)) { + return &evidence(s).Spec[i], nil + } + } + return nil, errors.New("the evidence has no spec entry for the ticket") +} + +func checkTicketCaptured(s *Session) error { + _, err := ticketSpec(s) + return err +} + +func checkTicketContentStored(s *Session) error { + e, err := ticketSpec(s) + if err != nil { + return err + } + content, err := s.Stored.Material(e.Digest) + if err != nil { + return fmt.Errorf("read the spec material: %w", err) + } + if !bytes.Contains(content, []byte(ticketTitle)) { + return errors.New("the spec material does not hold the text of the ticket file") + } + return nil +} + +func checkSkillTracked(s *Session) error { + for _, e := range evidence(s).Spec { + if e.Kind != aicodingsession.SpecKindSkill || !strings.Contains(e.Title+e.URI, skillName) { + continue + } + if e.Metadata == nil || e.Metadata.InvocationCount == 0 { + return errors.New("the skill entry has no invocation") + } + return nil + } + return fmt.Errorf("the evidence has no entry for the skill %s", skillName) +} + +// storedContains reports whether a file of the stored session holds the text. +func storedContains(s *Session, text string) (bool, error) { + found := false + err := filepath.WalkDir(s.Stored.Dir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || found { + return err + } + data, err := os.ReadFile(path) //nolint:gosec // a throwaway workspace of the suite + if err != nil { + return err + } + found = bytes.Contains(data, []byte(text)) + return nil + }) + return found, err +} + +func checkSecretRedacted(s *Session) error { + if err := checkTranscript(s); err != nil { + return fmt.Errorf("nothing to redact: %w", err) + } + found, err := storedContains(s, fakeSecret) + if err != nil { + return err + } + if found { + return errors.New("the stored session holds the secret of the prompt") + } + // The prompt is in the transcript, so its secret must show as a + // placeholder. With no placeholder, the check proves nothing. + if found, err = storedContains(s, redactedPlaceholder); err != nil { + return err + } + if !found { + return errors.New("the stored session has no redaction placeholder") + } + return nil +} + +// redactedPlaceholder is the start of the text that replaces a redacted +// secret, as internal/redaction writes it. +const redactedPlaceholder = "[CHAINLOOP_TRACE_REDACTED" + +func checkAllowedSecretKept(s *Session) error { + found, err := storedContains(s, fakeAllowedSecret) + if err != nil { + return err + } + if !found { + return errors.New("the secret on the line with an allow marker was redacted, or the session did not read it") + } + return nil +} + +func checkCommitTrailer(s *Session) error { + ids := state.ParseSessionIDsFromTrailer(s.HeadMessage) + for _, id := range ids { + if id == s.Stored.Evidence.Data.Session.ID { + return nil + } + } + return fmt.Errorf("the commit trailer has sessions %v, not the stored session", ids) +} + +func checkLocalStorage(s *Session) error { + if !s.Stored.HasStatement() { + return errors.New("the stored session has no statement.json") + } + return nil +} diff --git a/app/cli/internal/trace/e2e/claude.go b/app/cli/internal/trace/e2e/claude.go new file mode 100644 index 000000000..d22f004f8 --- /dev/null +++ b/app/cli/internal/trace/e2e/claude.go @@ -0,0 +1,158 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "strings" +) + +// claudeTools are the tools that a scenario session may use with no +// permission prompt. The session runs in a throwaway repository, and the list +// keeps the session from asking, which a headless run cannot answer. +var claudeTools = []string{"Read", "Write", "Edit", "Glob", "Grep", "Bash", "Skill"} + +// Claude is the harness adapter of Claude Code. +type Claude struct { + // Bin is the claude executable. Empty means "claude" from PATH. + Bin string +} + +func (c *Claude) bin() string { + if c.Bin != "" { + return c.Bin + } + return "claude" +} + +// Column implements Harness. +func (c *Claude) Column() string { return HarnessClaudeCode } + +// TraceFlag implements Harness. +func (c *Claude) TraceFlag() string { return "--claude" } + +// Version implements Harness. `claude --version` prints "2.1.0 (Claude Code)". +func (c *Claude) Version(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, c.bin(), "--version").Output() //nolint:gosec // the harness binary under test + if err != nil { + return "", fmt.Errorf("claude --version: %w", err) + } + fields := strings.Fields(string(out)) + if len(fields) == 0 { + return "", fmt.Errorf("claude --version: empty output") + } + return fields[0], nil +} + +// Run implements Harness. It runs `claude -p` with the stream-json output and +// the hook events, so the output shows the hook messages that the user sees. +// +// Only the project and local settings are loaded. The settings and the plugins +// of the user who runs the suite would add their own hooks and change the +// session. +func (c *Claude) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { + args := []string{ + "-p", req.Prompt, + "--model", req.Model, + "--output-format", "stream-json", + "--verbose", + "--include-hook-events", + "--setting-sources", "project,local", + "--allowedTools", strings.Join(claudeTools, ","), + } + + cmd := exec.CommandContext(ctx, c.bin(), args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario + cmd.Dir = req.Dir + cmd.Env = req.Env + var stderr bytes.Buffer + cmd.Stderr = &stderr + + raw, err := cmd.Output() + if err != nil { + return &RunOutput{Raw: raw}, fmt.Errorf("claude -p: %w: %s", err, strings.TrimSpace(stderr.String())) + } + + hooks, err := parseClaudeHookMessages(raw) + if err != nil { + return &RunOutput{Raw: raw}, err + } + + return &RunOutput{Raw: raw, Hooks: hooks, HooksReported: true}, nil +} + +// claudeStreamEvent is the part of a stream-json line that the suite reads. A +// hook event has type "system" and subtype "hook_response". Its stdout is +// the response that the hook printed. +type claudeStreamEvent struct { + Type string `json:"type"` + Subtype string `json:"subtype"` + HookEvent string `json:"hook_event"` + Stdout string `json:"stdout"` +} + +// claudeHookResponse is the response that a Trace hook prints for Claude Code. +type claudeHookResponse struct { + SystemMessage string `json:"systemMessage"` + HookSpecificOutput struct { + AdditionalContext string `json:"additionalContext"` + } `json:"hookSpecificOutput"` +} + +// parseClaudeHookMessages reads the hook responses from the stream-json +// output. A hook that printed no JSON, for example a tool hook, gives no +// message. +func parseClaudeHookMessages(raw []byte) ([]HookMessage, error) { + var out []HookMessage + + scanner := bufio.NewScanner(bytes.NewReader(raw)) + scanner.Buffer(make([]byte, 0, 1024*1024), 64*1024*1024) + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + + var ev claudeStreamEvent + if err := json.Unmarshal(line, &ev); err != nil { + return nil, fmt.Errorf("decode claude stream event: %w", err) + } + if ev.Type != "system" || ev.Subtype != "hook_response" || strings.TrimSpace(ev.Stdout) == "" { + continue + } + + var resp claudeHookResponse + if err := json.Unmarshal([]byte(ev.Stdout), &resp); err != nil { + // Not a JSON response: the hook printed plain text, which Claude + // Code does not show to the user. + continue + } + out = append(out, HookMessage{ + Event: ev.HookEvent, + UserMessage: resp.SystemMessage, + ModelContext: resp.HookSpecificOutput.AdditionalContext, + }) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read claude output: %w", err) + } + + return out, nil +} diff --git a/app/cli/internal/trace/e2e/claude_test.go b/app/cli/internal/trace/e2e/claude_test.go new file mode 100644 index 000000000..bd818775e --- /dev/null +++ b/app/cli/internal/trace/e2e/claude_test.go @@ -0,0 +1,65 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseClaudeHookMessages(t *testing.T) { + testCases := []struct { + name string + raw string + want []HookMessage + wantErr bool + }{ + { + name: "hook response with banner and context", + raw: `{"type":"system","subtype":"init","session_id":"s1"} +{"type":"system","subtype":"hook_started","hook_event":"SessionStart"} +{"type":"system","subtype":"hook_response","hook_event":"SessionStart","stdout":"{\"systemMessage\":\"hello\",\"hookSpecificOutput\":{\"hookEventName\":\"SessionStart\",\"additionalContext\":\"capture specs\"}}\n"} +{"type":"assistant","message":{"content":[{"type":"text","text":"ok"}]}} +`, + want: []HookMessage{{Event: "SessionStart", UserMessage: "hello", ModelContext: "capture specs"}}, + }, + { + name: "hook with empty or plain stdout gives no message", + raw: `{"type":"system","subtype":"hook_response","hook_event":"PreToolUse","stdout":""} +{"type":"system","subtype":"hook_response","hook_event":"PostToolUse","stdout":"plain text"} +`, + }, + { + name: "line that is not JSON", + raw: "not json\n", + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := parseClaudeHookMessages([]byte(tc.raw)) + if tc.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/app/cli/internal/trace/e2e/config.go b/app/cli/internal/trace/e2e/config.go new file mode 100644 index 000000000..f5a1452fb --- /dev/null +++ b/app/cli/internal/trace/e2e/config.go @@ -0,0 +1,127 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + _ "embed" + "errors" + "fmt" + "time" + + "sigs.k8s.io/yaml" +) + +//go:embed config.yaml +var defaultConfig []byte + +// HarnessConfig pins a harness column and lists its models. +type HarnessConfig struct { + // Version is the pinned harness version (D-013). + Version string `json:"version"` + // Models are the models that the suite runs the harness with (D-009). + Models []string `json:"models"` +} + +// Repeat sets how many times a cell runs and how many runs must pass. +type Repeat struct { + Runs int `json:"runs"` + Threshold int `json:"threshold"` +} + +func (r Repeat) validate() error { + if r.Runs < 1 || r.Threshold < 1 || r.Threshold > r.Runs { + return fmt.Errorf("runs %d and threshold %d: want 1 <= threshold <= runs", r.Runs, r.Threshold) + } + return nil +} + +// Config is the configuration of the suite. +type Config struct { + // Harnesses are keyed by the column name in the feature table. + Harnesses map[string]HarnessConfig `json:"harnesses"` + // ModelDriven is the default repeat of a model-driven feature (R-006). + ModelDriven Repeat `json:"modelDriven"` + // Features override the repeat of a feature, keyed by feature name. + Features map[string]Repeat `json:"features"` + ScenarioTimeout Duration `json:"scenarioTimeout"` + Parallel int `json:"parallel"` +} + +// Duration is a time.Duration written as a Go duration string in YAML. +type Duration time.Duration + +// UnmarshalJSON reads a duration string such as "5m". +func (d *Duration) UnmarshalJSON(b []byte) error { + var s string + if err := yaml.Unmarshal(b, &s); err != nil { + return err + } + v, err := time.ParseDuration(s) + if err != nil { + return err + } + *d = Duration(v) + return nil +} + +// LoadConfig reads the suite configuration. A nil data reads the embedded +// config.yaml. +func LoadConfig(data []byte) (*Config, error) { + if data == nil { + data = defaultConfig + } + + var cfg Config + if err := yaml.UnmarshalStrict(data, &cfg); err != nil { + return nil, fmt.Errorf("decode suite config: %w", err) + } + + if len(cfg.Harnesses) == 0 { + return nil, errors.New("suite config: no harness") + } + for name, h := range cfg.Harnesses { + if h.Version == "" || len(h.Models) == 0 { + return nil, fmt.Errorf("suite config: harness %q needs a version and at least one model", name) + } + } + if err := cfg.ModelDriven.validate(); err != nil { + return nil, fmt.Errorf("suite config: modelDriven: %w", err) + } + for name, r := range cfg.Features { + if err := r.validate(); err != nil { + return nil, fmt.Errorf("suite config: feature %q: %w", name, err) + } + } + if cfg.ScenarioTimeout <= 0 { + return nil, errors.New("suite config: scenarioTimeout must be positive") + } + if cfg.Parallel < 1 { + cfg.Parallel = 1 + } + + return &cfg, nil +} + +// repeatFor returns how many times a feature runs and how many runs must pass. +func (c *Config) repeatFor(feature string, modelDriven bool) Repeat { + if r, ok := c.Features[feature]; ok { + return r + } + if modelDriven { + return c.ModelDriven + } + return Repeat{Runs: 1, Threshold: 1} +} diff --git a/app/cli/internal/trace/e2e/config.yaml b/app/cli/internal/trace/e2e/config.yaml new file mode 100644 index 000000000..dc8ed0887 --- /dev/null +++ b/app/cli/internal/trace/e2e/config.yaml @@ -0,0 +1,27 @@ +# Configuration of the Chainloop Trace end-to-end suite (spec issue-3635). +# +# Each harness entry pins one version for its column of the feature table +# (D-013) and lists the models that the suite runs it with (D-009). A +# maintainer bumps a pin on purpose. The key is the column name in the +# feature table of ../README.md. +harnesses: + Claude Code: + version: "2.1.301" + models: + - haiku + +# Runs and pass threshold of a cell whose feature depends on the model +# following an instruction (R-006, D-004). A cell of any other feature runs +# once. +modelDriven: + runs: 3 + threshold: 2 + +# Per-feature overrides of the runs and the threshold, keyed by feature name. +features: {} + +# Time limit of one scenario session. +scenarioTimeout: 5m + +# Scenario sessions that run at the same time. +parallel: 4 diff --git a/app/cli/internal/trace/e2e/config_test.go b/app/cli/internal/trace/e2e/config_test.go new file mode 100644 index 000000000..0cd6e4645 --- /dev/null +++ b/app/cli/internal/trace/e2e/config_test.go @@ -0,0 +1,82 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadConfig(t *testing.T) { + const valid = ` +harnesses: + Claude Code: + version: "2.1.0" + models: [haiku] +modelDriven: {runs: 3, threshold: 2} +features: + Skills tracking: {runs: 5, threshold: 4} +scenarioTimeout: 2m +parallel: 2 +` + testCases := []struct { + name string + data string + wantErr string + }{ + {name: "valid", data: valid}, + {name: "unknown field", data: valid + "extra: 1\n", wantErr: "unknown field"}, + { + name: "harness with no model", + data: "harnesses: {Cursor: {version: '1'}}\nmodelDriven: {runs: 1, threshold: 1}\nscenarioTimeout: 1m\n", + wantErr: `harness "Cursor" needs a version and at least one model`, + }, + { + name: "threshold above runs", + data: "harnesses: {Cursor: {version: '1', models: [a]}}\nmodelDriven: {runs: 1, threshold: 2}\nscenarioTimeout: 1m\n", + wantErr: "modelDriven", + }, + { + name: "missing timeout", + data: "harnesses: {Cursor: {version: '1', models: [a]}}\nmodelDriven: {runs: 1, threshold: 1}\n", + wantErr: "scenarioTimeout", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := LoadConfig([]byte(tc.data)) + if tc.wantErr != "" { + assert.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, 2*time.Minute, time.Duration(cfg.ScenarioTimeout)) + assert.Equal(t, Repeat{Runs: 5, Threshold: 4}, cfg.repeatFor("Skills tracking", true)) + assert.Equal(t, Repeat{Runs: 3, Threshold: 2}, cfg.repeatFor("Spec capture instruction", true)) + assert.Equal(t, Repeat{Runs: 1, Threshold: 1}, cfg.repeatFor("Transcript", false)) + }) + } +} + +func TestEmbeddedConfig(t *testing.T) { + cfg, err := LoadConfig(nil) + require.NoError(t, err) + assert.Contains(t, cfg.Harnesses, HarnessClaudeCode) +} diff --git a/app/cli/internal/trace/e2e/e2e_test.go b/app/cli/internal/trace/e2e/e2e_test.go new file mode 100644 index 000000000..72e8540b4 --- /dev/null +++ b/app/cli/internal/trace/e2e/e2e_test.go @@ -0,0 +1,151 @@ +//go:build e2e + +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Environment of a run of the suite. +const ( + // envHarness, envModel and envFeature filter a manual run. Each is a + // comma-separated list. A filtered run does not become the previous result. + envHarness = "CHAINLOOP_E2E_HARNESS" + envModel = "CHAINLOOP_E2E_MODEL" + envFeature = "CHAINLOOP_E2E_FEATURE" + // envResultsDir is where the results are stored. The default is the + // results folder of this package, which git ignores. + envResultsDir = "CHAINLOOP_E2E_RESULTS_DIR" + // envWorkDir keeps the workspaces and the harness output after the run. + // The default is a temporary directory that the test deletes. + envWorkDir = "CHAINLOOP_E2E_WORK_DIR" +) + +// TestTraceMatrix runs the end-to-end suite of Chainloop Trace with real +// harnesses and models (spec issue-3635). It builds the chainloop CLI, runs the +// scenarios for each harness and model of config.yaml, stores the result +// matrix, and compares it with the previous full run. +// +// It fails when a cell that the feature table marks "Yes" or "Partial" does +// not pass, or when a scenario could not run. A feature row with no check is +// logged, not failed. +func TestTraceMatrix(t *testing.T) { + ctx := t.Context() + + f, err := os.Open(traceREADMEPath) + require.NoError(t, err) + exp, err := ParseExpected(f) + f.Close() + require.NoError(t, err) + + cfg, err := LoadConfig(nil) + require.NoError(t, err) + + bin, version := buildChainloop(t) + + workDir := os.Getenv(envWorkDir) + if workDir == "" { + workDir = t.TempDir() + } + + runner := &Runner{ + Expected: exp, + Config: cfg, + Harnesses: []Harness{&Claude{}}, + Filter: Filter{Harnesses: envList(envHarness), Models: envList(envModel), Features: envList(envFeature)}, + ChainloopBin: bin, + CLIVersion: version, + WorkDir: workDir, + Logf: t.Logf, + } + + res, err := runner.Run(ctx) + require.NoError(t, err) + + resultsDir := os.Getenv(envResultsDir) + if resultsDir == "" { + resultsDir = "results" + } + store := ResultStore{Dir: resultsDir} + prev, err := store.Previous() + require.NoError(t, err) + changes := Compare(prev, res) + path, err := store.Save(res) + require.NoError(t, err) + + t.Logf("result matrix, stored in %s:\n\n%s", path, Markdown(res)) + for _, c := range changes.Regressions { + t.Logf("REGRESSION: %s / %s %s / %s: %s -> %s (%d/%d): %s", c.Cell.Feature, c.Cell.Harness, c.Cell.HarnessVersion, c.Cell.Model, c.Previous, c.Cell.Status, c.Cell.Passes, c.Cell.Runs, c.Cell.Reason) + } + for _, c := range changes.Improvements { + t.Logf("IMPROVEMENT: %s / %s %s / %s passes, the table says Unknown", c.Cell.Feature, c.Cell.Harness, c.Cell.HarnessVersion, c.Cell.Model) + } + // A row with no check is a gap of the suite, which a new check closes. A + // scenario that could not run makes the result unreliable, so it fails. + for _, c := range res.Problems() { + if c.Status == StatusNoCheck { + t.Logf("suite gap: %s / %s / %s: %s", c.Feature, c.Harness, c.Model, c.Reason) + continue + } + assert.Failf(t, "suite problem", "%s / %s / %s: %s: %s", c.Feature, c.Harness, c.Model, c.Status, c.Reason) + } + for _, c := range res.Cells { + if c.Status == StatusFail && (c.Expected == SupportYes || c.Expected == SupportPartial) { + assert.Failf(t, "regression of a supported feature", "%s / %s %s / %s: marked %s but fails (%d/%d): %s", c.Feature, c.Harness, c.HarnessVersion, c.Model, c.Expected, c.Passes, c.Runs, c.Reason) + } + } +} + +// buildChainloop builds the chainloop CLI under test and returns its path and +// its version. +func buildChainloop(t *testing.T) (string, string) { + t.Helper() + + version := "e2e" + if out, err := exec.Command("git", "describe", "--tags", "--always", "--dirty").Output(); err == nil { + version = strings.TrimSpace(string(out)) + } + + bin := filepath.Join(t.TempDir(), "chainloop") + cmd := exec.Command("go", "build", + "-ldflags", "-X github.com/chainloop-dev/chainloop/app/cli/cmd.Version="+version, + "-o", bin, "github.com/chainloop-dev/chainloop/app/cli") + out, err := cmd.CombinedOutput() + require.NoError(t, err, "build chainloop: %s", out) + + return bin, version +} + +func envList(name string) []string { + v := strings.TrimSpace(os.Getenv(name)) + if v == "" { + return nil + } + parts := strings.Split(v, ",") + for i := range parts { + parts[i] = strings.TrimSpace(parts[i]) + } + return parts +} diff --git a/app/cli/internal/trace/e2e/expected.go b/app/cli/internal/trace/e2e/expected.go new file mode 100644 index 000000000..566c5f955 --- /dev/null +++ b/app/cli/internal/trace/e2e/expected.go @@ -0,0 +1,190 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package e2e is the end-to-end test suite of Chainloop Trace (spec +// issue-3635). It runs real coding agents in headless mode, in throwaway +// repositories with Trace in local storage mode, and builds a result matrix of +// harness, model and feature. The expected matrix is the feature table of the +// Trace README. +package e2e + +import ( + "bufio" + "errors" + "fmt" + "io" + "strings" +) + +// traceREADMEPath is the Trace README, relative to this package. It holds the +// feature table that is the expected matrix (D-002). +const traceREADMEPath = "../README.md" + +// Support is the value of one cell of the feature table. +type Support string + +const ( + SupportYes Support = "Yes" + SupportPartial Support = "Partial" + SupportNo Support = "No" + SupportUnknown Support = "Unknown" + SupportNotApplicable Support = "Not applicable" +) + +// Tested reports whether the suite tests a cell with this value. "No" and "Not +// applicable" cells are skipped (R-002). +func (s Support) Tested() bool { + switch s { + case SupportYes, SupportPartial, SupportUnknown: + return true + default: + return false + } +} + +func parseSupport(v string) (Support, bool) { + s := Support(v) + switch s { + case SupportYes, SupportPartial, SupportNo, SupportUnknown, SupportNotApplicable: + return s, true + default: + return "", false + } +} + +// Expected is the expected matrix read from the feature table. +type Expected struct { + // Features are the feature names in the order of the README. + Features []string + // Harnesses are the harness column names in the order of the README. + Harnesses []string + cells map[string]map[string]Support +} + +// Support returns the value of the cell of a feature and a harness column. The +// second result is false when the table has no such cell. +func (e *Expected) Support(feature, harness string) (Support, bool) { + s, ok := e.cells[feature][harness] + return s, ok +} + +// featureHeader is the start of the header row of a feature table. Other +// tables of the README, for example how each agent is hooked in, do not start +// with it. +var featureHeader = []string{"Feature", "What it means"} + +// ParseExpected reads the feature tables of the Trace README. All feature +// tables must have the same harness columns. +func ParseExpected(r io.Reader) (*Expected, error) { + exp := &Expected{cells: make(map[string]map[string]Support)} + + var ( + columns []string // harness columns of the current table, nil outside one + inTable bool + ) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, "|") { + inTable, columns = false, nil + continue + } + + cells := splitRow(line) + if !inTable { + inTable = true + if len(cells) > len(featureHeader) && cells[0] == featureHeader[0] && cells[1] == featureHeader[1] { + columns = cells[len(featureHeader):] + if err := exp.addHarnesses(columns); err != nil { + return nil, err + } + } + continue + } + + if columns == nil || isSeparatorRow(cells) { + continue + } + + if err := exp.addRow(cells, columns); err != nil { + return nil, err + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read feature table: %w", err) + } + + if len(exp.Features) == 0 { + return nil, errors.New("no feature table found") + } + + return exp, nil +} + +func (e *Expected) addHarnesses(columns []string) error { + if e.Harnesses == nil { + e.Harnesses = columns + return nil + } + if strings.Join(e.Harnesses, "|") != strings.Join(columns, "|") { + return fmt.Errorf("feature tables have different harness columns: %v and %v", e.Harnesses, columns) + } + return nil +} + +func (e *Expected) addRow(cells, columns []string) error { + feature := cells[0] + if want := len(featureHeader) + len(columns); len(cells) != want { + return fmt.Errorf("feature %q has %d cells, want %d", feature, len(cells), want) + } + if _, dup := e.cells[feature]; dup { + return fmt.Errorf("feature %q is listed twice", feature) + } + + row := make(map[string]Support, len(columns)) + for i, harness := range columns { + raw := cells[len(featureHeader)+i] + s, ok := parseSupport(raw) + if !ok { + return fmt.Errorf("feature %q, column %q: unknown value %q", feature, harness, raw) + } + row[harness] = s + } + + e.cells[feature] = row + e.Features = append(e.Features, feature) + return nil +} + +// splitRow splits a markdown table row into its trimmed cells, with bold +// markers removed. +func splitRow(line string) []string { + line = strings.TrimSuffix(strings.TrimPrefix(line, "|"), "|") + parts := strings.Split(line, "|") + for i, p := range parts { + parts[i] = strings.TrimSpace(strings.ReplaceAll(p, "**", "")) + } + return parts +} + +func isSeparatorRow(cells []string) bool { + for _, c := range cells { + if strings.Trim(c, "-: ") != "" { + return false + } + } + return true +} diff --git a/app/cli/internal/trace/e2e/expected_test.go b/app/cli/internal/trace/e2e/expected_test.go new file mode 100644 index 000000000..a029f1b62 --- /dev/null +++ b/app/cli/internal/trace/e2e/expected_test.go @@ -0,0 +1,159 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fixture values shared by the tests of the package. +const ( + featTranscript = "Transcript" + featSecretRedaction = "Secret redaction" + colCursor = "Cursor" + testHarnessVersion = "2.1.0" + testModel = "haiku" +) + +const twoTables = `# Title + +Intro text. + +| Feature | What it means | Claude Code | Cursor | +|---------|---------------|-------------|--------| +| Transcript | The evidence holds the conversation. | Yes | Partial | +| Welcome message | The user sees a **message**. | Yes | No | + +Why not Yes: + +- **Transcript, Cursor:** reasons. + +| Agent | Hook installation | Hooks | +|-------|-------------------|-------| +| Claude Code | settings | many | + +| Feature | What it means | Claude Code | Cursor | +|---------|---------------|-------------|--------| +| Secret redaction | Secrets are redacted. | **Yes** | Not applicable | +| Media skipped | Media is skipped. | Unknown | Unknown | +` + +func TestParseExpected(t *testing.T) { + testCases := []struct { + name string + input string + wantErr string + features []string + cells map[[2]string]Support + }{ + { + name: "feature tables only, other tables ignored", + input: twoTables, + features: []string{featTranscript, "Welcome message", featSecretRedaction, "Media skipped"}, + cells: map[[2]string]Support{ + {featTranscript, "Claude Code"}: SupportYes, + {featTranscript, colCursor}: SupportPartial, + {"Welcome message", colCursor}: SupportNo, + {featSecretRedaction, "Claude Code"}: SupportYes, + {featSecretRedaction, colCursor}: SupportNotApplicable, + {"Media skipped", colCursor}: SupportUnknown, + }, + }, + { + name: "unknown cell value", + input: `| Feature | What it means | Claude Code | +|---|---|---| +| Transcript | x | Maybe | +`, + wantErr: `feature "Transcript", column "Claude Code": unknown value "Maybe"`, + }, + { + name: "duplicate feature", + input: `| Feature | What it means | Claude Code | +|---|---|---| +| Transcript | x | Yes | +| Transcript | y | Yes | +`, + wantErr: `feature "Transcript" is listed twice`, + }, + { + name: "row with missing cells", + input: `| Feature | What it means | Claude Code | Cursor | +|---|---|---|---| +| Transcript | x | Yes | +`, + wantErr: `feature "Transcript" has 3 cells, want 4`, + }, + { + name: "no feature table", + input: "# nothing here\n", + wantErr: "no feature table found", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseExpected(strings.NewReader(tc.input)) + if tc.wantErr != "" { + assert.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.features, got.Features) + for key, want := range tc.cells { + gotSupport, ok := got.Support(key[0], key[1]) + assert.True(t, ok, "cell %v", key) + assert.Equal(t, want, gotSupport, "cell %v", key) + } + }) + } +} + +func TestSupportTested(t *testing.T) { + testCases := []struct { + support Support + want bool + }{ + {SupportYes, true}, + {SupportPartial, true}, + {SupportUnknown, true}, + {SupportNo, false}, + {SupportNotApplicable, false}, + } + for _, tc := range testCases { + t.Run(string(tc.support), func(t *testing.T) { + assert.Equal(t, tc.want, tc.support.Tested()) + }) + } +} + +// TestTraceREADMEParses keeps the real feature table readable by the suite: +// a README edit that breaks the table shape fails here, not in the nightly run. +func TestTraceREADMEParses(t *testing.T) { + f, err := os.Open(traceREADMEPath) + require.NoError(t, err) + defer f.Close() + + got, err := ParseExpected(f) + require.NoError(t, err) + assert.Contains(t, got.Harnesses, HarnessClaudeCode) + assert.Contains(t, got.Features, featTranscript) +} diff --git a/app/cli/internal/trace/e2e/harness.go b/app/cli/internal/trace/e2e/harness.go new file mode 100644 index 000000000..ddba863ca --- /dev/null +++ b/app/cli/internal/trace/e2e/harness.go @@ -0,0 +1,71 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import "context" + +// HarnessClaudeCode is the column name of Claude Code in the feature table. +const HarnessClaudeCode = "Claude Code" + +// RunRequest is one headless session of a harness. +type RunRequest struct { + // Dir is the repository the session runs in. + Dir string + // Model is the model of the session, as the harness names it. + Model string + // Prompt is the only user prompt of the session. + Prompt string + // Env is the environment of the harness process. + Env []string +} + +// HookMessage is a message that a Trace hook returned to the harness. +type HookMessage struct { + // Event is the hook event, for example SessionStart. + Event string + // UserMessage is the text that the harness shows to the user. + UserMessage string + // ModelContext is the text that the harness gives to the model. + ModelContext string +} + +// RunOutput is what a harness showed during a session. +type RunOutput struct { + // Raw is the full output of the harness process. + Raw []byte + // Hooks are the hook messages that the harness reported, in order. A + // harness that does not report them leaves it empty. + Hooks []HookMessage + // HooksReported is true when the harness reports hook messages in its + // headless output. When it is false, a check on a hook message is "not + // covered" (R-004). + HooksReported bool +} + +// Harness adapts one coding agent. It is the only part of the suite that +// knows the agent (R-011). +type Harness interface { + // Column is the column name of the harness in the feature table. + Column() string + // TraceFlag is the flag of `chainloop trace init` that selects the + // harness, for example "--claude". + TraceFlag() string + // Version returns the installed version of the harness. + Version(ctx context.Context) (string, error) + // Run runs one headless session and returns its output. An error means + // that the session could not run, not that a feature failed. + Run(ctx context.Context, req RunRequest) (*RunOutput, error) +} diff --git a/app/cli/internal/trace/e2e/result.go b/app/cli/internal/trace/e2e/result.go new file mode 100644 index 000000000..5b7556200 --- /dev/null +++ b/app/cli/internal/trace/e2e/result.go @@ -0,0 +1,263 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +// Status is the outcome of one cell of the result matrix. +type Status string + +const ( + StatusPass Status = "pass" + StatusFail Status = "fail" + // StatusSkipped is a cell that the feature table marks "No" or "Not + // applicable" (R-002). + StatusSkipped Status = "skipped" + // StatusNotCovered is a cell that the suite cannot test: the feature needs + // a control plane (D-007), or the headless mode of the harness cannot show + // it (R-004). + StatusNotCovered Status = "not_covered" + // StatusNoCheck is a feature row with no registered check. It is a suite + // problem (R-002, R-008). + StatusNoCheck Status = "no_check" + // StatusError is a cell whose scenario could not run, for example because + // the harness did not start. It is a suite problem (R-008). + StatusError Status = "error" +) + +// resultSchemaVersion is the version of the stored result format. +const resultSchemaVersion = 1 + +// Cell is one cell of the result matrix: a feature for a harness and a model. +type Cell struct { + Feature string `json:"feature"` + Harness string `json:"harness"` + HarnessVersion string `json:"harness_version,omitempty"` + Model string `json:"model"` + Expected Support `json:"expected"` + Status Status `json:"status"` + // Passes and Runs give the pass rate of the cell (R-006). + Passes int `json:"passes"` + Runs int `json:"runs"` + // Reason tells why the cell did not pass, or why it was not tested. + Reason string `json:"reason,omitempty"` +} + +type cellKey struct { + feature, harness, model string +} + +func (c Cell) key() cellKey { + return cellKey{c.Feature, c.Harness, c.Model} +} + +// Result is the result matrix of one run. +type Result struct { + SchemaVersion int `json:"schema_version"` + StartedAt time.Time `json:"started_at"` + CLIVersion string `json:"cli_version"` + // Full is true for a run with no filter. Only a full run becomes the + // previous result of the next comparison (R-009). + Full bool `json:"full"` + Cells []Cell `json:"cells"` +} + +// Problems returns the cells that show a problem of the suite, not of Trace: a +// feature with no check, or a scenario that could not run (R-008). +func (r *Result) Problems() []Cell { + var out []Cell + for _, c := range r.Cells { + if c.Status == StatusNoCheck || c.Status == StatusError { + out = append(out, c) + } + } + return out +} + +// Change is a cell whose state changed from the previous run. +type Change struct { + Cell Cell + // Previous is the status in the previous run, empty when the cell is new. + Previous Status +} + +// Changes are the cells to report after a run (R-007). +type Changes struct { + // Regressions are cells that passed in the previous run and fail now. + Regressions []Change + // Improvements are cells marked "Unknown" that pass now and did not pass + // in the previous run, so the team can update the table. + Improvements []Change +} + +// Empty reports whether there is nothing to report. +func (c Changes) Empty() bool { + return len(c.Regressions) == 0 && len(c.Improvements) == 0 +} + +// Compare compares the current result with the previous one. prev is nil when +// there is no previous result. A cell that failed in the previous run too is not +// reported again (R-007, D-005). +func Compare(prev, cur *Result) Changes { + before := make(map[cellKey]Status) + if prev != nil { + for _, c := range prev.Cells { + before[c.key()] = c.Status + } + } + + var changes Changes + for _, c := range cur.Cells { + was := before[c.key()] + switch { + case was == StatusPass && c.Status == StatusFail: + changes.Regressions = append(changes.Regressions, Change{Cell: c, Previous: was}) + case c.Expected == SupportUnknown && c.Status == StatusPass && was != StatusPass: + changes.Improvements = append(changes.Improvements, Change{Cell: c, Previous: was}) + } + } + + return changes +} + +const ( + // latestResultFile is the result of the last full run: the previous result + // of the next comparison. + latestResultFile = "latest.json" + // runsDir keeps the result of every run, named by its start time. + runsDir = "runs" +) + +// ResultStore keeps the result matrices of the runs in a directory. +type ResultStore struct { + Dir string +} + +// Previous returns the result of the last full run, or nil when there is none. +func (s ResultStore) Previous() (*Result, error) { + data, err := os.ReadFile(filepath.Join(s.Dir, latestResultFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read previous result: %w", err) + } + + var res Result + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("decode previous result: %w", err) + } + return &res, nil +} + +// Save writes the result to the history and, for a full run, makes it the +// previous result of the next run. It returns the path of the history file. +func (s ResultStore) Save(res *Result) (string, error) { + res.SchemaVersion = resultSchemaVersion + data, err := json.MarshalIndent(res, "", " ") + if err != nil { + return "", fmt.Errorf("encode result: %w", err) + } + + dir := filepath.Join(s.Dir, runsDir) + if err := os.MkdirAll(dir, 0o750); err != nil { + return "", fmt.Errorf("create results directory: %w", err) + } + + path := filepath.Join(dir, res.StartedAt.UTC().Format("20060102T150405Z")+".json") + if err := os.WriteFile(path, data, 0o600); err != nil { + return "", fmt.Errorf("write result: %w", err) + } + + if res.Full { + if err := os.WriteFile(filepath.Join(s.Dir, latestResultFile), data, 0o600); err != nil { + return "", fmt.Errorf("write previous result: %w", err) + } + } + + return path, nil +} + +// Markdown renders the result matrix as a markdown table: one row per feature +// and one column per harness and model (R-010). +func Markdown(res *Result) string { + type column struct{ harness, model string } + + var ( + features []string + columns []column + labels = make(map[column]string) + cells = make(map[cellKey]Cell) + seenFeat = make(map[string]bool) + ) + for _, c := range res.Cells { + if !seenFeat[c.Feature] { + seenFeat[c.Feature] = true + features = append(features, c.Feature) + } + col := column{c.Harness, c.Model} + if _, ok := labels[col]; !ok { + columns = append(columns, col) + labels[col] = strings.TrimSpace(c.Harness+" "+c.HarnessVersion) + " / " + c.Model + } + cells[c.key()] = c + } + + var b strings.Builder + b.WriteString("| Feature |") + for _, col := range columns { + b.WriteString(" " + labels[col] + " |") + } + b.WriteString("\n|---|") + for range columns { + b.WriteString("---|") + } + b.WriteString("\n") + + for _, f := range features { + b.WriteString("| " + f + " |") + for _, col := range columns { + c, ok := cells[cellKey{f, col.harness, col.model}] + if !ok { + b.WriteString(" |") + continue + } + b.WriteString(" " + cellLabel(c) + " |") + } + b.WriteString("\n") + } + + return b.String() +} + +// cellLabel shows the status of a cell, with the pass rate when the cell ran +// more than once. +func cellLabel(c Cell) string { + label := strings.ReplaceAll(string(c.Status), "_", " ") + if c.Runs > 1 && (c.Status == StatusPass || c.Status == StatusFail) { + label += fmt.Sprintf(" %d/%d", c.Passes, c.Runs) + } + return label +} diff --git a/app/cli/internal/trace/e2e/result_test.go b/app/cli/internal/trace/e2e/result_test.go new file mode 100644 index 000000000..fc85fc9ce --- /dev/null +++ b/app/cli/internal/trace/e2e/result_test.go @@ -0,0 +1,181 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func cell(feature, model string, expected Support, status Status) Cell { + return Cell{Feature: feature, Harness: HarnessClaudeCode, Model: model, Expected: expected, Status: status} +} + +func TestCompare(t *testing.T) { + testCases := []struct { + name string + prev []Cell + cur []Cell + wantRegressions []string + wantImprovements []string + }{ + { + name: "pass to fail is a regression", + prev: []Cell{cell(featTranscript, testModel, SupportYes, StatusPass)}, + cur: []Cell{cell(featTranscript, testModel, SupportYes, StatusFail)}, + wantRegressions: []string{featTranscript}, + }, + { + name: "fail twice in a row is not reported again", + prev: []Cell{cell(featTranscript, testModel, SupportYes, StatusFail)}, + cur: []Cell{cell(featTranscript, testModel, SupportYes, StatusFail)}, + }, + { + name: "same feature, other model, is another cell", + prev: []Cell{cell(featTranscript, testModel, SupportYes, StatusPass)}, + cur: []Cell{cell(featTranscript, "sonnet", SupportYes, StatusFail)}, + }, + { + name: "unknown cell that passes is an improvement", + prev: []Cell{cell("Media", testModel, SupportUnknown, StatusFail)}, + cur: []Cell{cell("Media", testModel, SupportUnknown, StatusPass)}, + wantImprovements: []string{"Media"}, + }, + { + name: "unknown cell that passes with no previous run is an improvement", + cur: []Cell{cell("Media", testModel, SupportUnknown, StatusPass)}, + wantImprovements: []string{"Media"}, + }, + { + name: "unknown cell that passed before is not reported again", + prev: []Cell{cell("Media", testModel, SupportUnknown, StatusPass)}, + cur: []Cell{cell("Media", testModel, SupportUnknown, StatusPass)}, + }, + { + name: "pass to not covered is not a regression", + prev: []Cell{cell("Welcome message", testModel, SupportYes, StatusPass)}, + cur: []Cell{cell("Welcome message", testModel, SupportYes, StatusNotCovered)}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var prev *Result + if tc.prev != nil { + prev = &Result{Cells: tc.prev} + } + changes := Compare(prev, &Result{Cells: tc.cur}) + + assert.ElementsMatch(t, tc.wantRegressions, changeFeatures(changes.Regressions)) + assert.ElementsMatch(t, tc.wantImprovements, changeFeatures(changes.Improvements)) + }) + } +} + +func changeFeatures(changes []Change) []string { + out := make([]string, 0, len(changes)) + for _, c := range changes { + out = append(out, c.Cell.Feature) + } + return out +} + +func TestResultStore(t *testing.T) { + started := time.Date(2026, 10, 16, 3, 0, 0, 0, time.UTC) + + testCases := []struct { + name string + full bool + wantPrevious bool + }{ + {name: "full run becomes the previous result", full: true, wantPrevious: true}, + {name: "filtered run does not change the previous result", full: false, wantPrevious: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + store := ResultStore{Dir: t.TempDir()} + + prev, err := store.Previous() + require.NoError(t, err) + assert.Nil(t, prev, "an empty store has no previous result") + + res := &Result{ + StartedAt: started, + Full: tc.full, + Cells: []Cell{cell(featTranscript, testModel, SupportYes, StatusPass)}, + } + path, err := store.Save(res) + require.NoError(t, err) + assert.FileExists(t, path, "every run is kept in the history") + assert.Equal(t, filepath.Join(store.Dir, "runs", "20261016T030000Z.json"), path) + + prev, err = store.Previous() + require.NoError(t, err) + if !tc.wantPrevious { + assert.Nil(t, prev) + return + } + require.NotNil(t, prev) + assert.Equal(t, resultSchemaVersion, prev.SchemaVersion) + assert.Equal(t, res.Cells, prev.Cells) + }) + } + + t.Run("unreadable previous result is an error", func(t *testing.T) { + store := ResultStore{Dir: t.TempDir()} + require.NoError(t, os.WriteFile(filepath.Join(store.Dir, latestResultFile), []byte("{"), 0o600)) + _, err := store.Previous() + assert.Error(t, err) + }) +} + +func TestMarkdown(t *testing.T) { + res := &Result{ + CLIVersion: "v1.2.3", + Cells: []Cell{ + {Feature: featTranscript, Harness: HarnessClaudeCode, HarnessVersion: testHarnessVersion, Model: testModel, Status: StatusPass, Passes: 1, Runs: 1}, + {Feature: "Spec capture instruction", Harness: HarnessClaudeCode, HarnessVersion: testHarnessVersion, Model: testModel, Status: StatusFail, Passes: 1, Runs: 3}, + {Feature: "Session link after push", Harness: HarnessClaudeCode, HarnessVersion: testHarnessVersion, Model: testModel, Status: StatusNotCovered}, + {Feature: featTranscript, Harness: colCursor, Model: "auto", Status: StatusSkipped}, + }, + } + + want := `| Feature | Claude Code 2.1.0 / haiku | Cursor / auto | +|---|---|---| +| Transcript | pass | skipped | +| Spec capture instruction | fail 1/3 | | +| Session link after push | not covered | | +` + assert.Equal(t, want, Markdown(res)) +} + +func TestProblems(t *testing.T) { + res := &Result{Cells: []Cell{ + cell(featTranscript, testModel, SupportYes, StatusPass), + cell("Subagents", testModel, SupportYes, StatusNoCheck), + cell("Skills tracking", testModel, SupportYes, StatusError), + }} + got := res.Problems() + require.Len(t, got, 2) + assert.Equal(t, "Subagents", got[0].Feature) + assert.Equal(t, "Skills tracking", got[1].Feature) +} diff --git a/app/cli/internal/trace/e2e/runner.go b/app/cli/internal/trace/e2e/runner.go new file mode 100644 index 000000000..2164210bc --- /dev/null +++ b/app/cli/internal/trace/e2e/runner.go @@ -0,0 +1,308 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/sync/errgroup" +) + +// Filter limits a manual run to some harnesses, models or features (R-009). +// An empty list matches all. +type Filter struct { + Harnesses []string + Models []string + Features []string +} + +// Empty reports whether the filter matches all cells. Only a run with an empty +// filter is a full run. +func (f Filter) Empty() bool { + return len(f.Harnesses) == 0 && len(f.Models) == 0 && len(f.Features) == 0 +} + +func matches(list []string, v string) bool { + return len(list) == 0 || slices.Contains(list, v) +} + +// Runner expands the expected matrix into scenario sessions, runs them, and +// builds the result matrix. +type Runner struct { + Expected *Expected + Config *Config + Harnesses []Harness + Filter Filter + // ChainloopBin is the chainloop CLI under test. + ChainloopBin string + CLIVersion string + // WorkDir holds the workspaces and the harness output of each session. + WorkDir string + Logf func(format string, args ...any) +} + +// sessionResult is one run of a scenario. err is set when the session could +// not run, which is a suite problem and not a feature failure. +type sessionResult struct { + session *Session + err error +} + +// plannedCell is a cell to test, with the repeat of its feature. +type plannedCell struct { + cell Cell + feature Feature + repeat Repeat +} + +// Run runs the suite and returns the result matrix. +func (r *Runner) Run(ctx context.Context) (*Result, error) { + res := &Result{StartedAt: time.Now().UTC(), CLIVersion: r.CLIVersion, Full: r.Filter.Empty()} + + for _, h := range r.Harnesses { + if !matches(r.Filter.Harnesses, h.Column()) { + continue + } + if !slices.Contains(r.Expected.Harnesses, h.Column()) { + return nil, fmt.Errorf("harness %q has no column in the feature table", h.Column()) + } + hc, ok := r.Config.Harnesses[h.Column()] + if !ok { + return nil, fmt.Errorf("harness %q has no entry in the suite config", h.Column()) + } + + version, versionErr := h.Version(ctx) + if versionErr == nil && version != hc.Version { + r.Logf("%s: installed version %s is not the pinned version %s", h.Column(), version, hc.Version) + } + + for _, model := range hc.Models { + if !matches(r.Filter.Models, model) { + continue + } + cells, err := r.runHarnessModel(ctx, h, version, versionErr, model) + if err != nil { + return nil, err + } + res.Cells = append(res.Cells, cells...) + } + } + + return res, nil +} + +// runHarnessModel tests all the cells of one harness and one model. +func (r *Runner) runHarnessModel(ctx context.Context, h Harness, version string, versionErr error, model string) ([]Cell, error) { + var ( + cells []Cell + planned []plannedCell + runs = make(map[string]int) // scenario name to the runs it needs + ) + + for _, name := range r.Expected.Features { + if !matches(r.Filter.Features, name) { + continue + } + support, _ := r.Expected.Support(name, h.Column()) + c := Cell{Feature: name, Harness: h.Column(), HarnessVersion: version, Model: model, Expected: support} + + def, hasDef := features[name] + switch { + case !support.Tested(): + c.Status, c.Reason = StatusSkipped, "the feature table marks it "+string(support) + case !hasDef || (!def.ControlPlane && len(def.Checks) == 0): + c.Status, c.Reason = StatusNoCheck, "no check is registered for this feature" + case def.ControlPlane: + c.Status, c.Reason = StatusNotCovered, "the feature needs a control plane" + case versionErr != nil: + c.Status, c.Reason = StatusError, versionErr.Error() + default: + repeat := r.Config.repeatFor(name, def.ModelDriven) + planned = append(planned, plannedCell{cell: c, feature: def, repeat: repeat}) + for _, chk := range def.Checks { + runs[chk.Scenario] = max(runs[chk.Scenario], repeat.Runs) + } + continue + } + cells = append(cells, c) + } + + sessions, err := r.runScenarios(ctx, h, model, runs) + if err != nil { + return nil, err + } + + for _, p := range planned { + cells = append(cells, evaluate(p, sessions)) + } + + // Keep the order of the feature table. + order := make(map[string]int, len(r.Expected.Features)) + for i, f := range r.Expected.Features { + order[f] = i + } + slices.SortStableFunc(cells, func(a, b Cell) int { return order[a.Feature] - order[b.Feature] }) + + return cells, nil +} + +// runScenarios runs each scenario the given number of times, in parallel. +func (r *Runner) runScenarios(ctx context.Context, h Harness, model string, runs map[string]int) (map[string][]sessionResult, error) { + var ( + mu sync.Mutex + out = make(map[string][]sessionResult, len(runs)) + ) + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(r.Config.Parallel) + + for name, n := range runs { + sc, ok := scenarios[name] + if !ok { + return nil, fmt.Errorf("unknown scenario %q", name) + } + out[name] = make([]sessionResult, n) + for i := range n { + g.Go(func() error { + s, err := r.runScenario(gctx, h, model, sc, i) + if err != nil { + r.Logf("%s / %s / %s #%d: %v", h.Column(), model, sc.Name, i+1, err) + } + mu.Lock() + out[name][i] = sessionResult{session: s, err: err} + mu.Unlock() + return nil + }) + } + } + _ = g.Wait() + + return out, ctx.Err() +} + +// runScenario runs one session of a scenario: it creates the workspace, runs +// the harness, pushes the commit of the agent, and reads the stored session. +func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Scenario, i int) (*Session, error) { + ctx, cancel := context.WithTimeout(ctx, time.Duration(r.Config.ScenarioTimeout)) + defer cancel() + + base := filepath.Join(r.WorkDir, dirName(h.Column()), dirName(model), sc.Name, strconv.Itoa(i+1)) + r.Logf("%s / %s / %s #%d: start in %s", h.Column(), model, sc.Name, i+1, base) + + ws, err := NewWorkspace(ctx, base, r.ChainloopBin, h, sc.Seed) + if err != nil { + return nil, fmt.Errorf("create workspace: %w", err) + } + + out, runErr := h.Run(ctx, RunRequest{Dir: ws.Dir, Model: model, Prompt: sc.Prompt, Env: ws.Env}) + if out != nil { + // The harness output stays next to the workspace to debug a failure. + if err := os.WriteFile(filepath.Join(base, "harness-output.jsonl"), out.Raw, 0o600); err != nil { + return nil, err + } + } + if runErr != nil { + return nil, runErr + } + + s := &Session{Scenario: sc.Name, Model: model, Workspace: ws, Output: out} + + // The agent pushes in the session, as a user does. The pre-push hook + // stores the session in local storage mode. The suite does not push after + // the session: the session end removes the captured specs, so a later push + // stores the session with no spec. A session that did not push stores + // nothing, and its checks fail. + if s.HeadMessage, err = ws.Git(ctx, "log", "-1", "--format=%B"); err != nil { + return nil, err + } + + stored, err := ws.StoredSessions() + if err != nil { + return nil, err + } + switch len(stored) { + case 0: + case 1: + s.Stored = &stored[0] + default: + return nil, fmt.Errorf("the push stored %d sessions, want 1", len(stored)) + } + + r.Logf("%s / %s / %s #%d: done", h.Column(), model, sc.Name, i+1) + return s, nil +} + +// errNoStoredSession is the failure of a check when the push stored no +// session, for example because the agent did not commit or push. +var errNoStoredSession = errors.New("the session stored nothing: the agent did not commit and push") + +// evaluate runs the checks of a planned cell on its sessions and sets its +// status and pass rate. +func evaluate(p plannedCell, sessions map[string][]sessionResult) Cell { + c := p.cell + c.Runs = p.repeat.Runs + + var reasons []string + for i := range p.repeat.Runs { + ok := true + for _, chk := range p.feature.Checks { + sr := sessions[chk.Scenario][i] + if sr.err != nil { + c.Status, c.Reason = StatusError, fmt.Sprintf("scenario %s could not run: %v", chk.Scenario, sr.err) + return c + } + + var err error + if sr.session.Stored == nil { + err = errNoStoredSession + } else { + err = chk.Verify(sr.session) + } + if errors.Is(err, errNotCovered) { + c.Status, c.Reason = StatusNotCovered, err.Error() + return c + } + if err != nil { + ok = false + reasons = append(reasons, fmt.Sprintf("run %d: %v", i+1, err)) + } + } + if ok { + c.Passes++ + } + } + + c.Status = StatusFail + if c.Passes >= p.repeat.Threshold { + c.Status = StatusPass + } + c.Reason = strings.Join(reasons, "; ") + return c +} + +// dirName turns a harness or model name into a directory name. +func dirName(s string) string { + return strings.NewReplacer(" ", "-", "/", "-", ":", "-").Replace(strings.ToLower(s)) +} diff --git a/app/cli/internal/trace/e2e/runner_test.go b/app/cli/internal/trace/e2e/runner_test.go new file mode 100644 index 000000000..668f69e27 --- /dev/null +++ b/app/cli/internal/trace/e2e/runner_test.go @@ -0,0 +1,184 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEvaluate(t *testing.T) { + pass := func(*Session) error { return nil } + fail := func(*Session) error { return errors.New("boom") } + notCovered := func(*Session) error { return fmt.Errorf("%w: headless", errNotCovered) } + stored := &Session{Stored: &StoredSession{}} + + // failOn fails the given runs of a scenario, counted from 1. + failOn := func(runs ...int) func(*Session) error { + return func(s *Session) error { + for _, r := range runs { + if s.Model == fmt.Sprint(r) { + return errors.New("boom") + } + } + return nil + } + } + numbered := func(n int) []sessionResult { + out := make([]sessionResult, n) + for i := range out { + out[i] = sessionResult{session: &Session{Model: fmt.Sprint(i + 1), Stored: &StoredSession{}}} + } + return out + } + + testCases := []struct { + name string + checks []Check + repeat Repeat + sessions map[string][]sessionResult + wantStatus Status + wantPasses int + }{ + { + name: "single run passes", + checks: []Check{{"a", pass}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{session: stored}}}, + wantStatus: StatusPass, + wantPasses: 1, + }, + { + name: "all checks must pass", + checks: []Check{{"a", pass}, {"a", fail}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{session: stored}}}, + wantStatus: StatusFail, + }, + { + name: "two of three runs pass", + checks: []Check{{"a", failOn(2)}}, + repeat: Repeat{3, 2}, + sessions: map[string][]sessionResult{"a": numbered(3)}, + wantStatus: StatusPass, + wantPasses: 2, + }, + { + name: "one of three runs passes", + checks: []Check{{"a", failOn(1, 3)}}, + repeat: Repeat{3, 2}, + sessions: map[string][]sessionResult{"a": numbered(3)}, + wantStatus: StatusFail, + wantPasses: 1, + }, + { + name: "scenario that could not run is a suite error", + checks: []Check{{"a", pass}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{err: errors.New("no harness")}}}, + wantStatus: StatusError, + }, + { + name: "check that cannot see the feature", + checks: []Check{{"a", notCovered}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{session: stored}}}, + wantStatus: StatusNotCovered, + }, + { + name: "no stored session fails", + checks: []Check{{"a", pass}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{session: &Session{}}}}, + wantStatus: StatusFail, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := evaluate(plannedCell{feature: Feature{Checks: tc.checks}, repeat: tc.repeat}, tc.sessions) + assert.Equal(t, tc.wantStatus, got.Status, got.Reason) + assert.Equal(t, tc.wantPasses, got.Passes) + assert.Equal(t, tc.repeat.Runs, got.Runs) + }) + } +} + +// fakeHarness fails to report its version, so the runner tests the planning +// of the cells and runs no session. +type fakeHarness struct{} + +func (fakeHarness) Column() string { return HarnessClaudeCode } +func (fakeHarness) TraceFlag() string { return "--claude" } +func (fakeHarness) Version(context.Context) (string, error) { + return "", errors.New("not installed") +} +func (fakeHarness) Run(context.Context, RunRequest) (*RunOutput, error) { + return nil, errors.New("not installed") +} + +func TestRunnerPlansCells(t *testing.T) { + exp, err := ParseExpected(strings.NewReader(`| Feature | What it means | Claude Code | Cursor | +|---|---|---|---| +| Transcript | x | Yes | Yes | +| Subagents | x | No | No | +| Attestation at push | x | Yes | Yes | +| Brand new feature | x | Unknown | Unknown | +`)) + require.NoError(t, err) + cfg, err := LoadConfig(nil) + require.NoError(t, err) + + r := &Runner{Expected: exp, Config: cfg, Harnesses: []Harness{fakeHarness{}}, WorkDir: t.TempDir(), Logf: t.Logf} + res, err := r.Run(context.Background()) + require.NoError(t, err) + + got := make(map[string]Status) + for _, c := range res.Cells { + got[c.Feature] = c.Status + } + assert.Equal(t, map[string]Status{ + featTranscript: StatusError, // the harness is not installed + "Subagents": StatusSkipped, + "Attestation at push": StatusNotCovered, + "Brand new feature": StatusNoCheck, + }, got) + assert.True(t, res.Full) +} + +// TestFeaturesMatchREADME keeps the registered checks in line with the feature +// table: a check whose feature was renamed in the table would never run. +func TestFeaturesMatchREADME(t *testing.T) { + f, err := os.Open(traceREADMEPath) + require.NoError(t, err) + defer f.Close() + exp, err := ParseExpected(f) + require.NoError(t, err) + + for name, def := range features { + assert.Contains(t, exp.Features, name, "registered feature is not in the table") + for _, chk := range def.Checks { + assert.Contains(t, scenarios, chk.Scenario, "feature %q", name) + } + } +} diff --git a/app/cli/internal/trace/e2e/scenarios.go b/app/cli/internal/trace/e2e/scenarios.go new file mode 100644 index 000000000..b8b65ec25 --- /dev/null +++ b/app/cli/internal/trace/e2e/scenarios.go @@ -0,0 +1,98 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +// Scenario is a scripted task in a throwaway repository. One session runs one +// scenario, and the checks of all the features that it touches read its +// result (D-008). Scenarios are shared by all harnesses (R-011). +type Scenario struct { + Name string + // Seed are the files of the repository before the session, by path. + Seed map[string]string + // Prompt is the only user prompt of the session. + Prompt string +} + +// The fake secrets of the scenarios. They match the GitHub token rule of the +// secret scanner, and they are not real tokens. The allow markers on these +// source lines keep the repository secret scan quiet. They do not reach the +// session: the prompt and the seed files carry only the token values. +const ( + // fakeSecret is in the prompt. The stored session must not hold it. + fakeSecret = "ghp_8fK2mQ9xLp4ZrT7vWc1NbH6yJd3sGa5eUo0i" //nolint:gosec // gitleaks:allow a fake token that the suite expects redaction to remove + // fakeAllowedSecret is on a line with an allow marker in the seed file. + // The stored session keeps it. + fakeAllowedSecret = "ghp_Zx7Lq2Wm9Rt4Yp6Vn3Bk8Hd1Jf5Gs0CaEu2o" //nolint:gosec // gitleaks:allow a fake token on a line with an allow marker +) + +// The README of every scenario repository. +const ( + readmePath = "README.md" + seedReadme = "# Trace E2E\n" +) + +const ( + scenarioEditAndCommit = "edit-and-commit" + scenarioTicket = "ticket" + scenarioSkill = "skill" +) + +const ( + ticketPath = "docs/ticket-e2e-42.md" + ticketTitle = "E2E-42: Add a farewell file" + skillName = "e2e-farewell" +) + +// scenarios are the scenarios of the suite, by name. +var scenarios = map[string]Scenario{ + scenarioEditAndCommit: { + Name: scenarioEditAndCommit, + Seed: map[string]string{ + readmePath: seedReadme, + "fixture.txt": "token = " + fakeAllowedSecret + " # gitleaks:allow\n", + }, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. Read the file fixture.txt. +2. With your file-writing tool, create the file greeting.txt with exactly these three lines: hello, from, the agent. +3. Run this shell command: printf 'one\ntwo\n' > shell.txt +4. Run this shell command: git add greeting.txt shell.txt && git commit -m "add greeting" +5. Run this shell command: git push +The deploy token ` + fakeSecret + ` is for a later task: do not use it.`, + }, + scenarioTicket: { + Name: scenarioTicket, + Seed: map[string]string{ + readmePath: seedReadme, + ticketPath: "# " + ticketTitle + "\n\nCreate the file farewell.txt with the single line: goodbye\n", + }, + Prompt: "This is an automated test. Implement the ticket in " + ticketPath + ". Then commit your change and push it with git.", + }, + scenarioSkill: { + Name: scenarioSkill, + Seed: map[string]string{ + readmePath: seedReadme, + ".claude/skills/" + skillName + "/SKILL.md": `--- +name: ` + skillName + ` +description: Use when asked to add the farewell file to the repository. +--- + +Create the file farewell.txt with the single line: goodbye +Then run: git add farewell.txt && git commit -m "add farewell" && git push +`, + }, + Prompt: "This is an automated test. Use the " + skillName + " skill.", + }, +} diff --git a/app/cli/internal/trace/e2e/workspace.go b/app/cli/internal/trace/e2e/workspace.go new file mode 100644 index 000000000..b37e8498e --- /dev/null +++ b/app/cli/internal/trace/e2e/workspace.go @@ -0,0 +1,209 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +// localSessionsDir is where local storage mode writes the sessions, under the +// default trace directory. +var localSessionsDir = filepath.Join(".git", "chainloop-trace", "local-sessions") + +// gitConfig isolates the git commands of a workspace, and of the harness +// that runs in it, from the git configuration of the user who runs the suite. +// A global hooks path would hide the Trace git hooks, and commit signing could +// ask for a passphrase. +const gitConfig = `[user] + name = Chainloop Trace E2E + email = trace-e2e@chainloop.local +[commit] + gpgsign = false +[tag] + gpgsign = false +[init] + defaultBranch = main +` + +// Workspace is a throwaway git repository with Trace in local storage mode, +// and a local bare remote to push to. +type Workspace struct { + // Dir is the repository. + Dir string + // Env is the environment of every command in the workspace, the harness + // included: the chainloop binary under test first in PATH, and an + // isolated git configuration. + Env []string +} + +// NewWorkspace creates a workspace under base. It writes the seed files, +// runs `chainloop trace init --local` for the harness, commits the result and +// pushes it, so the session starts from a clean, pushed tree. +func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, seed map[string]string) (*Workspace, error) { + repo := filepath.Join(base, "repo") + remote := filepath.Join(base, "remote.git") + gitConfigPath := filepath.Join(base, "gitconfig") + + if err := os.MkdirAll(repo, 0o750); err != nil { + return nil, err + } + if err := os.WriteFile(gitConfigPath, []byte(gitConfig), 0o600); err != nil { + return nil, err + } + + w := &Workspace{ + Dir: repo, + Env: append(os.Environ(), + "PATH="+filepath.Dir(chainloopBin)+string(os.PathListSeparator)+os.Getenv("PATH"), + "GIT_CONFIG_GLOBAL="+gitConfigPath, + "GIT_CONFIG_NOSYSTEM=1", + "CHAINLOOP_NO_PROMPT=1", + "DO_NOT_TRACK=1", + ), + } + + if _, err := w.run(ctx, base, "git", "init", "--bare", "--quiet", remote); err != nil { + return nil, err + } + if _, err := w.Git(ctx, "init", "--quiet"); err != nil { + return nil, err + } + + for name, content := range seed { + path := filepath.Join(repo, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return nil, err + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + return nil, err + } + } + + if _, err := w.run(ctx, repo, chainloopBin, "trace", "init", "--local", h.TraceFlag()); err != nil { + return nil, err + } + + for _, args := range [][]string{ + {"add", "-A"}, + {"commit", "--quiet", "-m", "seed"}, + {"remote", "add", "origin", remote}, + {"push", "--quiet", "-u", "origin", "main"}, + } { + if _, err := w.Git(ctx, args...); err != nil { + return nil, err + } + } + + return w, nil +} + +// Git runs a git command in the repository and returns its output. +func (w *Workspace) Git(ctx context.Context, args ...string) (string, error) { + return w.run(ctx, w.Dir, "git", args...) +} + +func (w *Workspace) run(ctx context.Context, dir, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + cmd.Env = w.Env + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return string(out), nil +} + +// StoredSession is one session that the pre-push hook stored in local storage +// mode. +type StoredSession struct { + // Dir is the directory of the stored session. + Dir string + // Evidence is the AI coding session evidence. + Evidence aicodingsession.Evidence +} + +// Material returns the stored content of a material, by its digest. +func (s *StoredSession) Material(digest string) ([]byte, error) { + return os.ReadFile(filepath.Join(s.Dir, "evidence", strings.ReplaceAll(digest, ":", "-"))) +} + +// HasStatement reports whether the stored session has its statement index. +func (s *StoredSession) HasStatement() bool { + _, err := os.Stat(filepath.Join(s.Dir, "statement.json")) + return err == nil +} + +// StoredSessions reads the sessions that local storage mode stored. Each +// session directory holds the AI coding session evidence among its materials. +func (w *Workspace) StoredSessions() ([]StoredSession, error) { + root := filepath.Join(w.Dir, localSessionsDir) + dirs, err := os.ReadDir(root) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + + var out []StoredSession + for _, d := range dirs { + if !d.IsDir() { + continue + } + dir := filepath.Join(root, d.Name()) + ev, err := findSessionEvidence(filepath.Join(dir, "evidence")) + if err != nil { + return nil, fmt.Errorf("stored session %s: %w", d.Name(), err) + } + out = append(out, StoredSession{Dir: dir, Evidence: *ev}) + } + + return out, nil +} + +// findSessionEvidence finds the AI coding session evidence among the +// materials of a stored session. The other materials, for example spec +// sources, are not JSON or have another ID. +func findSessionEvidence(dir string) (*aicodingsession.Evidence, error) { + files, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + for _, f := range files { + data, err := os.ReadFile(filepath.Join(dir, f.Name())) + if err != nil { + return nil, err + } + var ev aicodingsession.Evidence + if json.Unmarshal(data, &ev) == nil && ev.ID == aicodingsession.EvidenceID { + return &ev, nil + } + } + return nil, errors.New("no AI coding session evidence") +}