From 8a38b059339818370e1820c8d2758021004498b6 Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Mon, 19 Oct 2026 09:50:06 +0200 Subject: [PATCH 1/2] feat(trace): end-to-end test matrix for Claude Code Add a local end-to-end suite for Chainloop Trace (spec issue-3635, milestone 2). The suite reads the expected matrix from the feature table of the Trace README, runs Claude Code headless with real models in throwaway repositories in local storage mode, and stores the result matrix. It compares each full run with the previous one. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: be5af72c-30e7-48f4-b46c-dc7a5a754c41 --- app/cli/Makefile | 5 + app/cli/internal/trace/README.md | 12 + app/cli/internal/trace/e2e/.gitignore | 1 + app/cli/internal/trace/e2e/checks.go | 309 ++++++++++++++++++++ app/cli/internal/trace/e2e/claude.go | 158 ++++++++++ app/cli/internal/trace/e2e/claude_test.go | 65 ++++ app/cli/internal/trace/e2e/config.go | 127 ++++++++ app/cli/internal/trace/e2e/config.yaml | 27 ++ app/cli/internal/trace/e2e/config_test.go | 82 ++++++ app/cli/internal/trace/e2e/e2e_test.go | 151 ++++++++++ app/cli/internal/trace/e2e/expected.go | 190 ++++++++++++ app/cli/internal/trace/e2e/expected_test.go | 159 ++++++++++ app/cli/internal/trace/e2e/harness.go | 71 +++++ app/cli/internal/trace/e2e/result.go | 263 +++++++++++++++++ app/cli/internal/trace/e2e/result_test.go | 181 ++++++++++++ app/cli/internal/trace/e2e/runner.go | 308 +++++++++++++++++++ app/cli/internal/trace/e2e/runner_test.go | 184 ++++++++++++ app/cli/internal/trace/e2e/scenarios.go | 96 ++++++ app/cli/internal/trace/e2e/workspace.go | 209 +++++++++++++ 19 files changed, 2598 insertions(+) create mode 100644 app/cli/internal/trace/e2e/.gitignore create mode 100644 app/cli/internal/trace/e2e/checks.go create mode 100644 app/cli/internal/trace/e2e/claude.go create mode 100644 app/cli/internal/trace/e2e/claude_test.go create mode 100644 app/cli/internal/trace/e2e/config.go create mode 100644 app/cli/internal/trace/e2e/config.yaml create mode 100644 app/cli/internal/trace/e2e/config_test.go create mode 100644 app/cli/internal/trace/e2e/e2e_test.go create mode 100644 app/cli/internal/trace/e2e/expected.go create mode 100644 app/cli/internal/trace/e2e/expected_test.go create mode 100644 app/cli/internal/trace/e2e/harness.go create mode 100644 app/cli/internal/trace/e2e/result.go create mode 100644 app/cli/internal/trace/e2e/result_test.go create mode 100644 app/cli/internal/trace/e2e/runner.go create mode 100644 app/cli/internal/trace/e2e/runner_test.go create mode 100644 app/cli/internal/trace/e2e/scenarios.go create mode 100644 app/cli/internal/trace/e2e/workspace.go diff --git a/app/cli/Makefile b/app/cli/Makefile index be3e15ce2..db2eee42f 100644 --- a/app/cli/Makefile +++ b/app/cli/Makefile @@ -12,6 +12,11 @@ build: test: go test ./... +.PHONY: test-trace-e2e +# run the Chainloop Trace end-to-end suite with real agents and models (uses model tokens) +test-trace-e2e: + go test -tags e2e -run TestTraceMatrix -timeout 60m -v ./internal/trace/e2e/ + .PHONY: lint # lint lint: diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index fa8d84a1d..1c73bd937 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -128,3 +128,15 @@ Why not Yes: | OpenCode | `.opencode/plugins/chainloop-trace.ts` | Session start and end, user prompt, and before and after each tool. One plugin covers both versions: `server()` for 1.x (1.3.4 or later) and `setup()` for 2. The provider picks the export command from `opencode --version`. | All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`. + +## End-to-end tests + +The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. It tests each cell marked **Yes**, **Partial**, or **Unknown**, and it skips the other cells. A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. + +`e2e/config.yaml` pins the version of each agent and lists the models. The suite uses model tokens. To run it: + +```bash +make -C app/cli test-trace-e2e +``` + +`CHAINLOOP_E2E_HARNESS`, `CHAINLOOP_E2E_MODEL`, and `CHAINLOOP_E2E_FEATURE` limit a run to some columns, models, or features (comma-separated). Set `CHAINLOOP_E2E_WORK_DIR` to keep the workspaces and the agent output. The results go to `e2e/results/`, or to `CHAINLOOP_E2E_RESULTS_DIR`. Only a run with no filter becomes the previous result of the next comparison. diff --git a/app/cli/internal/trace/e2e/.gitignore b/app/cli/internal/trace/e2e/.gitignore new file mode 100644 index 000000000..fbca22537 --- /dev/null +++ b/app/cli/internal/trace/e2e/.gitignore @@ -0,0 +1 @@ +results/ diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go new file mode 100644 index 000000000..97ed067a4 --- /dev/null +++ b/app/cli/internal/trace/e2e/checks.go @@ -0,0 +1,309 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/app/cli/internal/trace" + "github.com/chainloop-dev/chainloop/app/cli/internal/trace/state" + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +// errNotCovered marks a check that cannot see the feature in this run, for +// example because the harness does not report hook messages in headless mode +// (R-004). +var errNotCovered = errors.New("not covered") + +// Session is the result of one scenario session. +type Session struct { + Scenario string + Model string + Workspace *Workspace + Output *RunOutput + // Stored is the session that the push stored in local storage mode. + Stored *StoredSession + // HeadMessage is the message of the commit that the agent made. + HeadMessage string +} + +// Check verifies one feature on the result of one scenario session. +type Check struct { + Scenario string + Verify func(s *Session) error +} + +// Feature is the test definition of one feature row of the table. The key of +// the features map is the feature name in the table (R-002). +type Feature struct { + // ControlPlane is true for a feature that needs a control plane. The suite + // runs in local storage mode only, so its cells are "not covered" (D-007). + ControlPlane bool + // ModelDriven is true for a feature that depends on the model following an + // instruction. Its cells run more than once (R-006). + ModelDriven bool + // Checks must all pass for a run of the feature to pass. + Checks []Check +} + +// features are the test definitions of the feature rows. A row of the table +// with no entry here is reported as "no check". +var features = map[string]Feature{ + // Session collection + "Transcript": {Checks: []Check{{scenarioEditAndCommit, checkTranscript}}}, + "Token usage and cost": {Checks: []Check{{scenarioEditAndCommit, checkTokenUsage}}}, + "Model detection": {Checks: []Check{{scenarioEditAndCommit, checkModel}}}, + + // Code attribution + "Line attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("greeting.txt")}}}, + "Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}}, + + // User experience + "Welcome message": {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}}, + "Session link after push": {ControlPlane: true}, + "Credentials warning": {ControlPlane: true}, + + // Specs and skills + "Spec capture instruction": {ModelDriven: true, Checks: []Check{{scenarioTicket, checkTicketCaptured}}}, + "Spec capture reminder": {Checks: []Check{{scenarioEditAndCommit, checkPromptReminder}}}, + "Local spec sources read at push": {ModelDriven: true, Checks: []Check{{scenarioTicket, checkTicketContentStored}}}, + "Skills tracking": {ModelDriven: true, Checks: []Check{{scenarioSkill, checkSkillTracked}}}, + + // Security + "Secret redaction": {Checks: []Check{{scenarioEditAndCommit, checkSecretRedacted}}}, + "Allow markers in redaction": {Checks: []Check{{scenarioEditAndCommit, checkAllowedSecretKept}}}, + + // Git integration + "Commit trailer": {Checks: []Check{{scenarioEditAndCommit, checkCommitTrailer}}}, + "Attestation at push": {ControlPlane: true}, + "Local storage at push": {Checks: []Check{{scenarioEditAndCommit, checkLocalStorage}}}, +} + +func evidence(s *Session) *aicodingsession.Data { + return &s.Stored.Evidence.Data +} + +func checkTranscript(s *Session) error { + main := evidence(s).RawSession["main"] + if len(main) == 0 { + return errors.New("the evidence has no main transcript") + } + for _, msg := range main { + if bytes.Contains(msg, []byte("greeting.txt")) { + return nil + } + } + return errors.New("the transcript does not show the work of the session") +} + +func checkTokenUsage(s *Session) error { + u := evidence(s).Usage + if u == nil || u.TotalTokens == 0 { + return errors.New("the evidence has no token usage") + } + if u.EstimatedCostUSD <= 0 { + return errors.New("the evidence has no cost") + } + return nil +} + +func checkModel(s *Session) error { + m := evidence(s).Model + if m == nil || m.Primary == "" { + return errors.New("the evidence has no primary model") + } + if !strings.Contains(strings.ToLower(m.Primary), strings.ToLower(s.Model)) { + return fmt.Errorf("primary model %q is not the model of the session %q", m.Primary, s.Model) + } + return nil +} + +func checkFileAttributedToAI(path string) func(s *Session) error { + return func(s *Session) error { + cc := evidence(s).CodeChanges + if cc == nil { + return errors.New("the evidence has no code changes") + } + for _, f := range cc.Files { + if f.Path != path { + continue + } + if f.Attribution != trace.AttributionAI || f.LinesAdded == 0 { + return fmt.Errorf("%s: attribution %q with %d lines added, want AI lines", path, f.Attribution, f.LinesAdded) + } + return nil + } + return fmt.Errorf("the code changes do not list %s", path) + } +} + +// hookMessage returns the first hook message of an event, or errNotCovered +// when the harness does not report hook messages. +func hookMessage(s *Session, event string) (*HookMessage, error) { + if !s.Output.HooksReported { + return nil, fmt.Errorf("%w: the harness does not report hook messages in headless mode", errNotCovered) + } + for i := range s.Output.Hooks { + if s.Output.Hooks[i].Event == event { + return &s.Output.Hooks[i], nil + } + } + return nil, fmt.Errorf("no %s hook message", event) +} + +// checkWelcomeMessage checks the banner that the user sees at session start. +// In local storage mode, it names the on-disk destination. +func checkWelcomeMessage(s *Session) error { + msg, err := hookMessage(s, "SessionStart") + if err != nil { + return err + } + if !strings.Contains(msg.UserMessage, "local-sessions") { + return fmt.Errorf("the session start message does not name the local sessions folder: %q", msg.UserMessage) + } + return nil +} + +func checkPromptReminder(s *Session) error { + msg, err := hookMessage(s, "UserPromptSubmit") + if err != nil { + return err + } + if msg.ModelContext == "" { + return errors.New("the user prompt hook gave no reminder to the model") + } + return nil +} + +// ticketSpec returns the captured spec entry of the ticket of the scenario. +func ticketSpec(s *Session) (*aicodingsession.SpecEntry, error) { + for i, e := range evidence(s).Spec { + if e.Kind != aicodingsession.SpecKindSkill && strings.Contains(e.URI, filepath.Base(ticketPath)) { + return &evidence(s).Spec[i], nil + } + } + return nil, errors.New("the evidence has no spec entry for the ticket") +} + +func checkTicketCaptured(s *Session) error { + _, err := ticketSpec(s) + return err +} + +func checkTicketContentStored(s *Session) error { + e, err := ticketSpec(s) + if err != nil { + return err + } + content, err := s.Stored.Material(e.Digest) + if err != nil { + return fmt.Errorf("read the spec material: %w", err) + } + if !bytes.Contains(content, []byte(ticketTitle)) { + return errors.New("the spec material does not hold the text of the ticket file") + } + return nil +} + +func checkSkillTracked(s *Session) error { + for _, e := range evidence(s).Spec { + if e.Kind != aicodingsession.SpecKindSkill || !strings.Contains(e.Title+e.URI, skillName) { + continue + } + if e.Metadata == nil || e.Metadata.InvocationCount == 0 { + return errors.New("the skill entry has no invocation") + } + return nil + } + return fmt.Errorf("the evidence has no entry for the skill %s", skillName) +} + +// storedContains reports whether a file of the stored session holds the text. +func storedContains(s *Session, text string) (bool, error) { + found := false + err := filepath.WalkDir(s.Stored.Dir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || found { + return err + } + data, err := os.ReadFile(path) //nolint:gosec // a throwaway workspace of the suite + if err != nil { + return err + } + found = bytes.Contains(data, []byte(text)) + return nil + }) + return found, err +} + +func checkSecretRedacted(s *Session) error { + if err := checkTranscript(s); err != nil { + return fmt.Errorf("nothing to redact: %w", err) + } + found, err := storedContains(s, fakeSecret) + if err != nil { + return err + } + if found { + return errors.New("the stored session holds the secret of the prompt") + } + // The prompt is in the transcript, so its secret must show as a + // placeholder. With no placeholder, the check proves nothing. + if found, err = storedContains(s, redactedPlaceholder); err != nil { + return err + } + if !found { + return errors.New("the stored session has no redaction placeholder") + } + return nil +} + +// redactedPlaceholder is the start of the text that replaces a redacted +// secret, as internal/redaction writes it. +const redactedPlaceholder = "[CHAINLOOP_TRACE_REDACTED" + +func checkAllowedSecretKept(s *Session) error { + found, err := storedContains(s, fakeAllowedSecret) + if err != nil { + return err + } + if !found { + return errors.New("the secret on the line with an allow marker was redacted, or the session did not read it") + } + return nil +} + +func checkCommitTrailer(s *Session) error { + ids := state.ParseSessionIDsFromTrailer(s.HeadMessage) + for _, id := range ids { + if id == s.Stored.Evidence.Data.Session.ID { + return nil + } + } + return fmt.Errorf("the commit trailer has sessions %v, not the stored session", ids) +} + +func checkLocalStorage(s *Session) error { + if !s.Stored.HasStatement() { + return errors.New("the stored session has no statement.json") + } + return nil +} diff --git a/app/cli/internal/trace/e2e/claude.go b/app/cli/internal/trace/e2e/claude.go new file mode 100644 index 000000000..d22f004f8 --- /dev/null +++ b/app/cli/internal/trace/e2e/claude.go @@ -0,0 +1,158 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "strings" +) + +// claudeTools are the tools that a scenario session may use with no +// permission prompt. The session runs in a throwaway repository, and the list +// keeps the session from asking, which a headless run cannot answer. +var claudeTools = []string{"Read", "Write", "Edit", "Glob", "Grep", "Bash", "Skill"} + +// Claude is the harness adapter of Claude Code. +type Claude struct { + // Bin is the claude executable. Empty means "claude" from PATH. + Bin string +} + +func (c *Claude) bin() string { + if c.Bin != "" { + return c.Bin + } + return "claude" +} + +// Column implements Harness. +func (c *Claude) Column() string { return HarnessClaudeCode } + +// TraceFlag implements Harness. +func (c *Claude) TraceFlag() string { return "--claude" } + +// Version implements Harness. `claude --version` prints "2.1.0 (Claude Code)". +func (c *Claude) Version(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, c.bin(), "--version").Output() //nolint:gosec // the harness binary under test + if err != nil { + return "", fmt.Errorf("claude --version: %w", err) + } + fields := strings.Fields(string(out)) + if len(fields) == 0 { + return "", fmt.Errorf("claude --version: empty output") + } + return fields[0], nil +} + +// Run implements Harness. It runs `claude -p` with the stream-json output and +// the hook events, so the output shows the hook messages that the user sees. +// +// Only the project and local settings are loaded. The settings and the plugins +// of the user who runs the suite would add their own hooks and change the +// session. +func (c *Claude) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { + args := []string{ + "-p", req.Prompt, + "--model", req.Model, + "--output-format", "stream-json", + "--verbose", + "--include-hook-events", + "--setting-sources", "project,local", + "--allowedTools", strings.Join(claudeTools, ","), + } + + cmd := exec.CommandContext(ctx, c.bin(), args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario + cmd.Dir = req.Dir + cmd.Env = req.Env + var stderr bytes.Buffer + cmd.Stderr = &stderr + + raw, err := cmd.Output() + if err != nil { + return &RunOutput{Raw: raw}, fmt.Errorf("claude -p: %w: %s", err, strings.TrimSpace(stderr.String())) + } + + hooks, err := parseClaudeHookMessages(raw) + if err != nil { + return &RunOutput{Raw: raw}, err + } + + return &RunOutput{Raw: raw, Hooks: hooks, HooksReported: true}, nil +} + +// claudeStreamEvent is the part of a stream-json line that the suite reads. A +// hook event has type "system" and subtype "hook_response". Its stdout is +// the response that the hook printed. +type claudeStreamEvent struct { + Type string `json:"type"` + Subtype string `json:"subtype"` + HookEvent string `json:"hook_event"` + Stdout string `json:"stdout"` +} + +// claudeHookResponse is the response that a Trace hook prints for Claude Code. +type claudeHookResponse struct { + SystemMessage string `json:"systemMessage"` + HookSpecificOutput struct { + AdditionalContext string `json:"additionalContext"` + } `json:"hookSpecificOutput"` +} + +// parseClaudeHookMessages reads the hook responses from the stream-json +// output. A hook that printed no JSON, for example a tool hook, gives no +// message. +func parseClaudeHookMessages(raw []byte) ([]HookMessage, error) { + var out []HookMessage + + scanner := bufio.NewScanner(bytes.NewReader(raw)) + scanner.Buffer(make([]byte, 0, 1024*1024), 64*1024*1024) + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + + var ev claudeStreamEvent + if err := json.Unmarshal(line, &ev); err != nil { + return nil, fmt.Errorf("decode claude stream event: %w", err) + } + if ev.Type != "system" || ev.Subtype != "hook_response" || strings.TrimSpace(ev.Stdout) == "" { + continue + } + + var resp claudeHookResponse + if err := json.Unmarshal([]byte(ev.Stdout), &resp); err != nil { + // Not a JSON response: the hook printed plain text, which Claude + // Code does not show to the user. + continue + } + out = append(out, HookMessage{ + Event: ev.HookEvent, + UserMessage: resp.SystemMessage, + ModelContext: resp.HookSpecificOutput.AdditionalContext, + }) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read claude output: %w", err) + } + + return out, nil +} diff --git a/app/cli/internal/trace/e2e/claude_test.go b/app/cli/internal/trace/e2e/claude_test.go new file mode 100644 index 000000000..bd818775e --- /dev/null +++ b/app/cli/internal/trace/e2e/claude_test.go @@ -0,0 +1,65 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseClaudeHookMessages(t *testing.T) { + testCases := []struct { + name string + raw string + want []HookMessage + wantErr bool + }{ + { + name: "hook response with banner and context", + raw: `{"type":"system","subtype":"init","session_id":"s1"} +{"type":"system","subtype":"hook_started","hook_event":"SessionStart"} +{"type":"system","subtype":"hook_response","hook_event":"SessionStart","stdout":"{\"systemMessage\":\"hello\",\"hookSpecificOutput\":{\"hookEventName\":\"SessionStart\",\"additionalContext\":\"capture specs\"}}\n"} +{"type":"assistant","message":{"content":[{"type":"text","text":"ok"}]}} +`, + want: []HookMessage{{Event: "SessionStart", UserMessage: "hello", ModelContext: "capture specs"}}, + }, + { + name: "hook with empty or plain stdout gives no message", + raw: `{"type":"system","subtype":"hook_response","hook_event":"PreToolUse","stdout":""} +{"type":"system","subtype":"hook_response","hook_event":"PostToolUse","stdout":"plain text"} +`, + }, + { + name: "line that is not JSON", + raw: "not json\n", + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := parseClaudeHookMessages([]byte(tc.raw)) + if tc.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/app/cli/internal/trace/e2e/config.go b/app/cli/internal/trace/e2e/config.go new file mode 100644 index 000000000..f5a1452fb --- /dev/null +++ b/app/cli/internal/trace/e2e/config.go @@ -0,0 +1,127 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + _ "embed" + "errors" + "fmt" + "time" + + "sigs.k8s.io/yaml" +) + +//go:embed config.yaml +var defaultConfig []byte + +// HarnessConfig pins a harness column and lists its models. +type HarnessConfig struct { + // Version is the pinned harness version (D-013). + Version string `json:"version"` + // Models are the models that the suite runs the harness with (D-009). + Models []string `json:"models"` +} + +// Repeat sets how many times a cell runs and how many runs must pass. +type Repeat struct { + Runs int `json:"runs"` + Threshold int `json:"threshold"` +} + +func (r Repeat) validate() error { + if r.Runs < 1 || r.Threshold < 1 || r.Threshold > r.Runs { + return fmt.Errorf("runs %d and threshold %d: want 1 <= threshold <= runs", r.Runs, r.Threshold) + } + return nil +} + +// Config is the configuration of the suite. +type Config struct { + // Harnesses are keyed by the column name in the feature table. + Harnesses map[string]HarnessConfig `json:"harnesses"` + // ModelDriven is the default repeat of a model-driven feature (R-006). + ModelDriven Repeat `json:"modelDriven"` + // Features override the repeat of a feature, keyed by feature name. + Features map[string]Repeat `json:"features"` + ScenarioTimeout Duration `json:"scenarioTimeout"` + Parallel int `json:"parallel"` +} + +// Duration is a time.Duration written as a Go duration string in YAML. +type Duration time.Duration + +// UnmarshalJSON reads a duration string such as "5m". +func (d *Duration) UnmarshalJSON(b []byte) error { + var s string + if err := yaml.Unmarshal(b, &s); err != nil { + return err + } + v, err := time.ParseDuration(s) + if err != nil { + return err + } + *d = Duration(v) + return nil +} + +// LoadConfig reads the suite configuration. A nil data reads the embedded +// config.yaml. +func LoadConfig(data []byte) (*Config, error) { + if data == nil { + data = defaultConfig + } + + var cfg Config + if err := yaml.UnmarshalStrict(data, &cfg); err != nil { + return nil, fmt.Errorf("decode suite config: %w", err) + } + + if len(cfg.Harnesses) == 0 { + return nil, errors.New("suite config: no harness") + } + for name, h := range cfg.Harnesses { + if h.Version == "" || len(h.Models) == 0 { + return nil, fmt.Errorf("suite config: harness %q needs a version and at least one model", name) + } + } + if err := cfg.ModelDriven.validate(); err != nil { + return nil, fmt.Errorf("suite config: modelDriven: %w", err) + } + for name, r := range cfg.Features { + if err := r.validate(); err != nil { + return nil, fmt.Errorf("suite config: feature %q: %w", name, err) + } + } + if cfg.ScenarioTimeout <= 0 { + return nil, errors.New("suite config: scenarioTimeout must be positive") + } + if cfg.Parallel < 1 { + cfg.Parallel = 1 + } + + return &cfg, nil +} + +// repeatFor returns how many times a feature runs and how many runs must pass. +func (c *Config) repeatFor(feature string, modelDriven bool) Repeat { + if r, ok := c.Features[feature]; ok { + return r + } + if modelDriven { + return c.ModelDriven + } + return Repeat{Runs: 1, Threshold: 1} +} diff --git a/app/cli/internal/trace/e2e/config.yaml b/app/cli/internal/trace/e2e/config.yaml new file mode 100644 index 000000000..dc8ed0887 --- /dev/null +++ b/app/cli/internal/trace/e2e/config.yaml @@ -0,0 +1,27 @@ +# Configuration of the Chainloop Trace end-to-end suite (spec issue-3635). +# +# Each harness entry pins one version for its column of the feature table +# (D-013) and lists the models that the suite runs it with (D-009). A +# maintainer bumps a pin on purpose. The key is the column name in the +# feature table of ../README.md. +harnesses: + Claude Code: + version: "2.1.301" + models: + - haiku + +# Runs and pass threshold of a cell whose feature depends on the model +# following an instruction (R-006, D-004). A cell of any other feature runs +# once. +modelDriven: + runs: 3 + threshold: 2 + +# Per-feature overrides of the runs and the threshold, keyed by feature name. +features: {} + +# Time limit of one scenario session. +scenarioTimeout: 5m + +# Scenario sessions that run at the same time. +parallel: 4 diff --git a/app/cli/internal/trace/e2e/config_test.go b/app/cli/internal/trace/e2e/config_test.go new file mode 100644 index 000000000..0cd6e4645 --- /dev/null +++ b/app/cli/internal/trace/e2e/config_test.go @@ -0,0 +1,82 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadConfig(t *testing.T) { + const valid = ` +harnesses: + Claude Code: + version: "2.1.0" + models: [haiku] +modelDriven: {runs: 3, threshold: 2} +features: + Skills tracking: {runs: 5, threshold: 4} +scenarioTimeout: 2m +parallel: 2 +` + testCases := []struct { + name string + data string + wantErr string + }{ + {name: "valid", data: valid}, + {name: "unknown field", data: valid + "extra: 1\n", wantErr: "unknown field"}, + { + name: "harness with no model", + data: "harnesses: {Cursor: {version: '1'}}\nmodelDriven: {runs: 1, threshold: 1}\nscenarioTimeout: 1m\n", + wantErr: `harness "Cursor" needs a version and at least one model`, + }, + { + name: "threshold above runs", + data: "harnesses: {Cursor: {version: '1', models: [a]}}\nmodelDriven: {runs: 1, threshold: 2}\nscenarioTimeout: 1m\n", + wantErr: "modelDriven", + }, + { + name: "missing timeout", + data: "harnesses: {Cursor: {version: '1', models: [a]}}\nmodelDriven: {runs: 1, threshold: 1}\n", + wantErr: "scenarioTimeout", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := LoadConfig([]byte(tc.data)) + if tc.wantErr != "" { + assert.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, 2*time.Minute, time.Duration(cfg.ScenarioTimeout)) + assert.Equal(t, Repeat{Runs: 5, Threshold: 4}, cfg.repeatFor("Skills tracking", true)) + assert.Equal(t, Repeat{Runs: 3, Threshold: 2}, cfg.repeatFor("Spec capture instruction", true)) + assert.Equal(t, Repeat{Runs: 1, Threshold: 1}, cfg.repeatFor("Transcript", false)) + }) + } +} + +func TestEmbeddedConfig(t *testing.T) { + cfg, err := LoadConfig(nil) + require.NoError(t, err) + assert.Contains(t, cfg.Harnesses, HarnessClaudeCode) +} diff --git a/app/cli/internal/trace/e2e/e2e_test.go b/app/cli/internal/trace/e2e/e2e_test.go new file mode 100644 index 000000000..72e8540b4 --- /dev/null +++ b/app/cli/internal/trace/e2e/e2e_test.go @@ -0,0 +1,151 @@ +//go:build e2e + +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Environment of a run of the suite. +const ( + // envHarness, envModel and envFeature filter a manual run. Each is a + // comma-separated list. A filtered run does not become the previous result. + envHarness = "CHAINLOOP_E2E_HARNESS" + envModel = "CHAINLOOP_E2E_MODEL" + envFeature = "CHAINLOOP_E2E_FEATURE" + // envResultsDir is where the results are stored. The default is the + // results folder of this package, which git ignores. + envResultsDir = "CHAINLOOP_E2E_RESULTS_DIR" + // envWorkDir keeps the workspaces and the harness output after the run. + // The default is a temporary directory that the test deletes. + envWorkDir = "CHAINLOOP_E2E_WORK_DIR" +) + +// TestTraceMatrix runs the end-to-end suite of Chainloop Trace with real +// harnesses and models (spec issue-3635). It builds the chainloop CLI, runs the +// scenarios for each harness and model of config.yaml, stores the result +// matrix, and compares it with the previous full run. +// +// It fails when a cell that the feature table marks "Yes" or "Partial" does +// not pass, or when a scenario could not run. A feature row with no check is +// logged, not failed. +func TestTraceMatrix(t *testing.T) { + ctx := t.Context() + + f, err := os.Open(traceREADMEPath) + require.NoError(t, err) + exp, err := ParseExpected(f) + f.Close() + require.NoError(t, err) + + cfg, err := LoadConfig(nil) + require.NoError(t, err) + + bin, version := buildChainloop(t) + + workDir := os.Getenv(envWorkDir) + if workDir == "" { + workDir = t.TempDir() + } + + runner := &Runner{ + Expected: exp, + Config: cfg, + Harnesses: []Harness{&Claude{}}, + Filter: Filter{Harnesses: envList(envHarness), Models: envList(envModel), Features: envList(envFeature)}, + ChainloopBin: bin, + CLIVersion: version, + WorkDir: workDir, + Logf: t.Logf, + } + + res, err := runner.Run(ctx) + require.NoError(t, err) + + resultsDir := os.Getenv(envResultsDir) + if resultsDir == "" { + resultsDir = "results" + } + store := ResultStore{Dir: resultsDir} + prev, err := store.Previous() + require.NoError(t, err) + changes := Compare(prev, res) + path, err := store.Save(res) + require.NoError(t, err) + + t.Logf("result matrix, stored in %s:\n\n%s", path, Markdown(res)) + for _, c := range changes.Regressions { + t.Logf("REGRESSION: %s / %s %s / %s: %s -> %s (%d/%d): %s", c.Cell.Feature, c.Cell.Harness, c.Cell.HarnessVersion, c.Cell.Model, c.Previous, c.Cell.Status, c.Cell.Passes, c.Cell.Runs, c.Cell.Reason) + } + for _, c := range changes.Improvements { + t.Logf("IMPROVEMENT: %s / %s %s / %s passes, the table says Unknown", c.Cell.Feature, c.Cell.Harness, c.Cell.HarnessVersion, c.Cell.Model) + } + // A row with no check is a gap of the suite, which a new check closes. A + // scenario that could not run makes the result unreliable, so it fails. + for _, c := range res.Problems() { + if c.Status == StatusNoCheck { + t.Logf("suite gap: %s / %s / %s: %s", c.Feature, c.Harness, c.Model, c.Reason) + continue + } + assert.Failf(t, "suite problem", "%s / %s / %s: %s: %s", c.Feature, c.Harness, c.Model, c.Status, c.Reason) + } + for _, c := range res.Cells { + if c.Status == StatusFail && (c.Expected == SupportYes || c.Expected == SupportPartial) { + assert.Failf(t, "regression of a supported feature", "%s / %s %s / %s: marked %s but fails (%d/%d): %s", c.Feature, c.Harness, c.HarnessVersion, c.Model, c.Expected, c.Passes, c.Runs, c.Reason) + } + } +} + +// buildChainloop builds the chainloop CLI under test and returns its path and +// its version. +func buildChainloop(t *testing.T) (string, string) { + t.Helper() + + version := "e2e" + if out, err := exec.Command("git", "describe", "--tags", "--always", "--dirty").Output(); err == nil { + version = strings.TrimSpace(string(out)) + } + + bin := filepath.Join(t.TempDir(), "chainloop") + cmd := exec.Command("go", "build", + "-ldflags", "-X github.com/chainloop-dev/chainloop/app/cli/cmd.Version="+version, + "-o", bin, "github.com/chainloop-dev/chainloop/app/cli") + out, err := cmd.CombinedOutput() + require.NoError(t, err, "build chainloop: %s", out) + + return bin, version +} + +func envList(name string) []string { + v := strings.TrimSpace(os.Getenv(name)) + if v == "" { + return nil + } + parts := strings.Split(v, ",") + for i := range parts { + parts[i] = strings.TrimSpace(parts[i]) + } + return parts +} diff --git a/app/cli/internal/trace/e2e/expected.go b/app/cli/internal/trace/e2e/expected.go new file mode 100644 index 000000000..566c5f955 --- /dev/null +++ b/app/cli/internal/trace/e2e/expected.go @@ -0,0 +1,190 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package e2e is the end-to-end test suite of Chainloop Trace (spec +// issue-3635). It runs real coding agents in headless mode, in throwaway +// repositories with Trace in local storage mode, and builds a result matrix of +// harness, model and feature. The expected matrix is the feature table of the +// Trace README. +package e2e + +import ( + "bufio" + "errors" + "fmt" + "io" + "strings" +) + +// traceREADMEPath is the Trace README, relative to this package. It holds the +// feature table that is the expected matrix (D-002). +const traceREADMEPath = "../README.md" + +// Support is the value of one cell of the feature table. +type Support string + +const ( + SupportYes Support = "Yes" + SupportPartial Support = "Partial" + SupportNo Support = "No" + SupportUnknown Support = "Unknown" + SupportNotApplicable Support = "Not applicable" +) + +// Tested reports whether the suite tests a cell with this value. "No" and "Not +// applicable" cells are skipped (R-002). +func (s Support) Tested() bool { + switch s { + case SupportYes, SupportPartial, SupportUnknown: + return true + default: + return false + } +} + +func parseSupport(v string) (Support, bool) { + s := Support(v) + switch s { + case SupportYes, SupportPartial, SupportNo, SupportUnknown, SupportNotApplicable: + return s, true + default: + return "", false + } +} + +// Expected is the expected matrix read from the feature table. +type Expected struct { + // Features are the feature names in the order of the README. + Features []string + // Harnesses are the harness column names in the order of the README. + Harnesses []string + cells map[string]map[string]Support +} + +// Support returns the value of the cell of a feature and a harness column. The +// second result is false when the table has no such cell. +func (e *Expected) Support(feature, harness string) (Support, bool) { + s, ok := e.cells[feature][harness] + return s, ok +} + +// featureHeader is the start of the header row of a feature table. Other +// tables of the README, for example how each agent is hooked in, do not start +// with it. +var featureHeader = []string{"Feature", "What it means"} + +// ParseExpected reads the feature tables of the Trace README. All feature +// tables must have the same harness columns. +func ParseExpected(r io.Reader) (*Expected, error) { + exp := &Expected{cells: make(map[string]map[string]Support)} + + var ( + columns []string // harness columns of the current table, nil outside one + inTable bool + ) + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, "|") { + inTable, columns = false, nil + continue + } + + cells := splitRow(line) + if !inTable { + inTable = true + if len(cells) > len(featureHeader) && cells[0] == featureHeader[0] && cells[1] == featureHeader[1] { + columns = cells[len(featureHeader):] + if err := exp.addHarnesses(columns); err != nil { + return nil, err + } + } + continue + } + + if columns == nil || isSeparatorRow(cells) { + continue + } + + if err := exp.addRow(cells, columns); err != nil { + return nil, err + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read feature table: %w", err) + } + + if len(exp.Features) == 0 { + return nil, errors.New("no feature table found") + } + + return exp, nil +} + +func (e *Expected) addHarnesses(columns []string) error { + if e.Harnesses == nil { + e.Harnesses = columns + return nil + } + if strings.Join(e.Harnesses, "|") != strings.Join(columns, "|") { + return fmt.Errorf("feature tables have different harness columns: %v and %v", e.Harnesses, columns) + } + return nil +} + +func (e *Expected) addRow(cells, columns []string) error { + feature := cells[0] + if want := len(featureHeader) + len(columns); len(cells) != want { + return fmt.Errorf("feature %q has %d cells, want %d", feature, len(cells), want) + } + if _, dup := e.cells[feature]; dup { + return fmt.Errorf("feature %q is listed twice", feature) + } + + row := make(map[string]Support, len(columns)) + for i, harness := range columns { + raw := cells[len(featureHeader)+i] + s, ok := parseSupport(raw) + if !ok { + return fmt.Errorf("feature %q, column %q: unknown value %q", feature, harness, raw) + } + row[harness] = s + } + + e.cells[feature] = row + e.Features = append(e.Features, feature) + return nil +} + +// splitRow splits a markdown table row into its trimmed cells, with bold +// markers removed. +func splitRow(line string) []string { + line = strings.TrimSuffix(strings.TrimPrefix(line, "|"), "|") + parts := strings.Split(line, "|") + for i, p := range parts { + parts[i] = strings.TrimSpace(strings.ReplaceAll(p, "**", "")) + } + return parts +} + +func isSeparatorRow(cells []string) bool { + for _, c := range cells { + if strings.Trim(c, "-: ") != "" { + return false + } + } + return true +} diff --git a/app/cli/internal/trace/e2e/expected_test.go b/app/cli/internal/trace/e2e/expected_test.go new file mode 100644 index 000000000..a029f1b62 --- /dev/null +++ b/app/cli/internal/trace/e2e/expected_test.go @@ -0,0 +1,159 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fixture values shared by the tests of the package. +const ( + featTranscript = "Transcript" + featSecretRedaction = "Secret redaction" + colCursor = "Cursor" + testHarnessVersion = "2.1.0" + testModel = "haiku" +) + +const twoTables = `# Title + +Intro text. + +| Feature | What it means | Claude Code | Cursor | +|---------|---------------|-------------|--------| +| Transcript | The evidence holds the conversation. | Yes | Partial | +| Welcome message | The user sees a **message**. | Yes | No | + +Why not Yes: + +- **Transcript, Cursor:** reasons. + +| Agent | Hook installation | Hooks | +|-------|-------------------|-------| +| Claude Code | settings | many | + +| Feature | What it means | Claude Code | Cursor | +|---------|---------------|-------------|--------| +| Secret redaction | Secrets are redacted. | **Yes** | Not applicable | +| Media skipped | Media is skipped. | Unknown | Unknown | +` + +func TestParseExpected(t *testing.T) { + testCases := []struct { + name string + input string + wantErr string + features []string + cells map[[2]string]Support + }{ + { + name: "feature tables only, other tables ignored", + input: twoTables, + features: []string{featTranscript, "Welcome message", featSecretRedaction, "Media skipped"}, + cells: map[[2]string]Support{ + {featTranscript, "Claude Code"}: SupportYes, + {featTranscript, colCursor}: SupportPartial, + {"Welcome message", colCursor}: SupportNo, + {featSecretRedaction, "Claude Code"}: SupportYes, + {featSecretRedaction, colCursor}: SupportNotApplicable, + {"Media skipped", colCursor}: SupportUnknown, + }, + }, + { + name: "unknown cell value", + input: `| Feature | What it means | Claude Code | +|---|---|---| +| Transcript | x | Maybe | +`, + wantErr: `feature "Transcript", column "Claude Code": unknown value "Maybe"`, + }, + { + name: "duplicate feature", + input: `| Feature | What it means | Claude Code | +|---|---|---| +| Transcript | x | Yes | +| Transcript | y | Yes | +`, + wantErr: `feature "Transcript" is listed twice`, + }, + { + name: "row with missing cells", + input: `| Feature | What it means | Claude Code | Cursor | +|---|---|---|---| +| Transcript | x | Yes | +`, + wantErr: `feature "Transcript" has 3 cells, want 4`, + }, + { + name: "no feature table", + input: "# nothing here\n", + wantErr: "no feature table found", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseExpected(strings.NewReader(tc.input)) + if tc.wantErr != "" { + assert.ErrorContains(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.features, got.Features) + for key, want := range tc.cells { + gotSupport, ok := got.Support(key[0], key[1]) + assert.True(t, ok, "cell %v", key) + assert.Equal(t, want, gotSupport, "cell %v", key) + } + }) + } +} + +func TestSupportTested(t *testing.T) { + testCases := []struct { + support Support + want bool + }{ + {SupportYes, true}, + {SupportPartial, true}, + {SupportUnknown, true}, + {SupportNo, false}, + {SupportNotApplicable, false}, + } + for _, tc := range testCases { + t.Run(string(tc.support), func(t *testing.T) { + assert.Equal(t, tc.want, tc.support.Tested()) + }) + } +} + +// TestTraceREADMEParses keeps the real feature table readable by the suite: +// a README edit that breaks the table shape fails here, not in the nightly run. +func TestTraceREADMEParses(t *testing.T) { + f, err := os.Open(traceREADMEPath) + require.NoError(t, err) + defer f.Close() + + got, err := ParseExpected(f) + require.NoError(t, err) + assert.Contains(t, got.Harnesses, HarnessClaudeCode) + assert.Contains(t, got.Features, featTranscript) +} diff --git a/app/cli/internal/trace/e2e/harness.go b/app/cli/internal/trace/e2e/harness.go new file mode 100644 index 000000000..ddba863ca --- /dev/null +++ b/app/cli/internal/trace/e2e/harness.go @@ -0,0 +1,71 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import "context" + +// HarnessClaudeCode is the column name of Claude Code in the feature table. +const HarnessClaudeCode = "Claude Code" + +// RunRequest is one headless session of a harness. +type RunRequest struct { + // Dir is the repository the session runs in. + Dir string + // Model is the model of the session, as the harness names it. + Model string + // Prompt is the only user prompt of the session. + Prompt string + // Env is the environment of the harness process. + Env []string +} + +// HookMessage is a message that a Trace hook returned to the harness. +type HookMessage struct { + // Event is the hook event, for example SessionStart. + Event string + // UserMessage is the text that the harness shows to the user. + UserMessage string + // ModelContext is the text that the harness gives to the model. + ModelContext string +} + +// RunOutput is what a harness showed during a session. +type RunOutput struct { + // Raw is the full output of the harness process. + Raw []byte + // Hooks are the hook messages that the harness reported, in order. A + // harness that does not report them leaves it empty. + Hooks []HookMessage + // HooksReported is true when the harness reports hook messages in its + // headless output. When it is false, a check on a hook message is "not + // covered" (R-004). + HooksReported bool +} + +// Harness adapts one coding agent. It is the only part of the suite that +// knows the agent (R-011). +type Harness interface { + // Column is the column name of the harness in the feature table. + Column() string + // TraceFlag is the flag of `chainloop trace init` that selects the + // harness, for example "--claude". + TraceFlag() string + // Version returns the installed version of the harness. + Version(ctx context.Context) (string, error) + // Run runs one headless session and returns its output. An error means + // that the session could not run, not that a feature failed. + Run(ctx context.Context, req RunRequest) (*RunOutput, error) +} diff --git a/app/cli/internal/trace/e2e/result.go b/app/cli/internal/trace/e2e/result.go new file mode 100644 index 000000000..5b7556200 --- /dev/null +++ b/app/cli/internal/trace/e2e/result.go @@ -0,0 +1,263 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +// Status is the outcome of one cell of the result matrix. +type Status string + +const ( + StatusPass Status = "pass" + StatusFail Status = "fail" + // StatusSkipped is a cell that the feature table marks "No" or "Not + // applicable" (R-002). + StatusSkipped Status = "skipped" + // StatusNotCovered is a cell that the suite cannot test: the feature needs + // a control plane (D-007), or the headless mode of the harness cannot show + // it (R-004). + StatusNotCovered Status = "not_covered" + // StatusNoCheck is a feature row with no registered check. It is a suite + // problem (R-002, R-008). + StatusNoCheck Status = "no_check" + // StatusError is a cell whose scenario could not run, for example because + // the harness did not start. It is a suite problem (R-008). + StatusError Status = "error" +) + +// resultSchemaVersion is the version of the stored result format. +const resultSchemaVersion = 1 + +// Cell is one cell of the result matrix: a feature for a harness and a model. +type Cell struct { + Feature string `json:"feature"` + Harness string `json:"harness"` + HarnessVersion string `json:"harness_version,omitempty"` + Model string `json:"model"` + Expected Support `json:"expected"` + Status Status `json:"status"` + // Passes and Runs give the pass rate of the cell (R-006). + Passes int `json:"passes"` + Runs int `json:"runs"` + // Reason tells why the cell did not pass, or why it was not tested. + Reason string `json:"reason,omitempty"` +} + +type cellKey struct { + feature, harness, model string +} + +func (c Cell) key() cellKey { + return cellKey{c.Feature, c.Harness, c.Model} +} + +// Result is the result matrix of one run. +type Result struct { + SchemaVersion int `json:"schema_version"` + StartedAt time.Time `json:"started_at"` + CLIVersion string `json:"cli_version"` + // Full is true for a run with no filter. Only a full run becomes the + // previous result of the next comparison (R-009). + Full bool `json:"full"` + Cells []Cell `json:"cells"` +} + +// Problems returns the cells that show a problem of the suite, not of Trace: a +// feature with no check, or a scenario that could not run (R-008). +func (r *Result) Problems() []Cell { + var out []Cell + for _, c := range r.Cells { + if c.Status == StatusNoCheck || c.Status == StatusError { + out = append(out, c) + } + } + return out +} + +// Change is a cell whose state changed from the previous run. +type Change struct { + Cell Cell + // Previous is the status in the previous run, empty when the cell is new. + Previous Status +} + +// Changes are the cells to report after a run (R-007). +type Changes struct { + // Regressions are cells that passed in the previous run and fail now. + Regressions []Change + // Improvements are cells marked "Unknown" that pass now and did not pass + // in the previous run, so the team can update the table. + Improvements []Change +} + +// Empty reports whether there is nothing to report. +func (c Changes) Empty() bool { + return len(c.Regressions) == 0 && len(c.Improvements) == 0 +} + +// Compare compares the current result with the previous one. prev is nil when +// there is no previous result. A cell that failed in the previous run too is not +// reported again (R-007, D-005). +func Compare(prev, cur *Result) Changes { + before := make(map[cellKey]Status) + if prev != nil { + for _, c := range prev.Cells { + before[c.key()] = c.Status + } + } + + var changes Changes + for _, c := range cur.Cells { + was := before[c.key()] + switch { + case was == StatusPass && c.Status == StatusFail: + changes.Regressions = append(changes.Regressions, Change{Cell: c, Previous: was}) + case c.Expected == SupportUnknown && c.Status == StatusPass && was != StatusPass: + changes.Improvements = append(changes.Improvements, Change{Cell: c, Previous: was}) + } + } + + return changes +} + +const ( + // latestResultFile is the result of the last full run: the previous result + // of the next comparison. + latestResultFile = "latest.json" + // runsDir keeps the result of every run, named by its start time. + runsDir = "runs" +) + +// ResultStore keeps the result matrices of the runs in a directory. +type ResultStore struct { + Dir string +} + +// Previous returns the result of the last full run, or nil when there is none. +func (s ResultStore) Previous() (*Result, error) { + data, err := os.ReadFile(filepath.Join(s.Dir, latestResultFile)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("read previous result: %w", err) + } + + var res Result + if err := json.Unmarshal(data, &res); err != nil { + return nil, fmt.Errorf("decode previous result: %w", err) + } + return &res, nil +} + +// Save writes the result to the history and, for a full run, makes it the +// previous result of the next run. It returns the path of the history file. +func (s ResultStore) Save(res *Result) (string, error) { + res.SchemaVersion = resultSchemaVersion + data, err := json.MarshalIndent(res, "", " ") + if err != nil { + return "", fmt.Errorf("encode result: %w", err) + } + + dir := filepath.Join(s.Dir, runsDir) + if err := os.MkdirAll(dir, 0o750); err != nil { + return "", fmt.Errorf("create results directory: %w", err) + } + + path := filepath.Join(dir, res.StartedAt.UTC().Format("20060102T150405Z")+".json") + if err := os.WriteFile(path, data, 0o600); err != nil { + return "", fmt.Errorf("write result: %w", err) + } + + if res.Full { + if err := os.WriteFile(filepath.Join(s.Dir, latestResultFile), data, 0o600); err != nil { + return "", fmt.Errorf("write previous result: %w", err) + } + } + + return path, nil +} + +// Markdown renders the result matrix as a markdown table: one row per feature +// and one column per harness and model (R-010). +func Markdown(res *Result) string { + type column struct{ harness, model string } + + var ( + features []string + columns []column + labels = make(map[column]string) + cells = make(map[cellKey]Cell) + seenFeat = make(map[string]bool) + ) + for _, c := range res.Cells { + if !seenFeat[c.Feature] { + seenFeat[c.Feature] = true + features = append(features, c.Feature) + } + col := column{c.Harness, c.Model} + if _, ok := labels[col]; !ok { + columns = append(columns, col) + labels[col] = strings.TrimSpace(c.Harness+" "+c.HarnessVersion) + " / " + c.Model + } + cells[c.key()] = c + } + + var b strings.Builder + b.WriteString("| Feature |") + for _, col := range columns { + b.WriteString(" " + labels[col] + " |") + } + b.WriteString("\n|---|") + for range columns { + b.WriteString("---|") + } + b.WriteString("\n") + + for _, f := range features { + b.WriteString("| " + f + " |") + for _, col := range columns { + c, ok := cells[cellKey{f, col.harness, col.model}] + if !ok { + b.WriteString(" |") + continue + } + b.WriteString(" " + cellLabel(c) + " |") + } + b.WriteString("\n") + } + + return b.String() +} + +// cellLabel shows the status of a cell, with the pass rate when the cell ran +// more than once. +func cellLabel(c Cell) string { + label := strings.ReplaceAll(string(c.Status), "_", " ") + if c.Runs > 1 && (c.Status == StatusPass || c.Status == StatusFail) { + label += fmt.Sprintf(" %d/%d", c.Passes, c.Runs) + } + return label +} diff --git a/app/cli/internal/trace/e2e/result_test.go b/app/cli/internal/trace/e2e/result_test.go new file mode 100644 index 000000000..fc85fc9ce --- /dev/null +++ b/app/cli/internal/trace/e2e/result_test.go @@ -0,0 +1,181 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func cell(feature, model string, expected Support, status Status) Cell { + return Cell{Feature: feature, Harness: HarnessClaudeCode, Model: model, Expected: expected, Status: status} +} + +func TestCompare(t *testing.T) { + testCases := []struct { + name string + prev []Cell + cur []Cell + wantRegressions []string + wantImprovements []string + }{ + { + name: "pass to fail is a regression", + prev: []Cell{cell(featTranscript, testModel, SupportYes, StatusPass)}, + cur: []Cell{cell(featTranscript, testModel, SupportYes, StatusFail)}, + wantRegressions: []string{featTranscript}, + }, + { + name: "fail twice in a row is not reported again", + prev: []Cell{cell(featTranscript, testModel, SupportYes, StatusFail)}, + cur: []Cell{cell(featTranscript, testModel, SupportYes, StatusFail)}, + }, + { + name: "same feature, other model, is another cell", + prev: []Cell{cell(featTranscript, testModel, SupportYes, StatusPass)}, + cur: []Cell{cell(featTranscript, "sonnet", SupportYes, StatusFail)}, + }, + { + name: "unknown cell that passes is an improvement", + prev: []Cell{cell("Media", testModel, SupportUnknown, StatusFail)}, + cur: []Cell{cell("Media", testModel, SupportUnknown, StatusPass)}, + wantImprovements: []string{"Media"}, + }, + { + name: "unknown cell that passes with no previous run is an improvement", + cur: []Cell{cell("Media", testModel, SupportUnknown, StatusPass)}, + wantImprovements: []string{"Media"}, + }, + { + name: "unknown cell that passed before is not reported again", + prev: []Cell{cell("Media", testModel, SupportUnknown, StatusPass)}, + cur: []Cell{cell("Media", testModel, SupportUnknown, StatusPass)}, + }, + { + name: "pass to not covered is not a regression", + prev: []Cell{cell("Welcome message", testModel, SupportYes, StatusPass)}, + cur: []Cell{cell("Welcome message", testModel, SupportYes, StatusNotCovered)}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var prev *Result + if tc.prev != nil { + prev = &Result{Cells: tc.prev} + } + changes := Compare(prev, &Result{Cells: tc.cur}) + + assert.ElementsMatch(t, tc.wantRegressions, changeFeatures(changes.Regressions)) + assert.ElementsMatch(t, tc.wantImprovements, changeFeatures(changes.Improvements)) + }) + } +} + +func changeFeatures(changes []Change) []string { + out := make([]string, 0, len(changes)) + for _, c := range changes { + out = append(out, c.Cell.Feature) + } + return out +} + +func TestResultStore(t *testing.T) { + started := time.Date(2026, 10, 16, 3, 0, 0, 0, time.UTC) + + testCases := []struct { + name string + full bool + wantPrevious bool + }{ + {name: "full run becomes the previous result", full: true, wantPrevious: true}, + {name: "filtered run does not change the previous result", full: false, wantPrevious: false}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + store := ResultStore{Dir: t.TempDir()} + + prev, err := store.Previous() + require.NoError(t, err) + assert.Nil(t, prev, "an empty store has no previous result") + + res := &Result{ + StartedAt: started, + Full: tc.full, + Cells: []Cell{cell(featTranscript, testModel, SupportYes, StatusPass)}, + } + path, err := store.Save(res) + require.NoError(t, err) + assert.FileExists(t, path, "every run is kept in the history") + assert.Equal(t, filepath.Join(store.Dir, "runs", "20261016T030000Z.json"), path) + + prev, err = store.Previous() + require.NoError(t, err) + if !tc.wantPrevious { + assert.Nil(t, prev) + return + } + require.NotNil(t, prev) + assert.Equal(t, resultSchemaVersion, prev.SchemaVersion) + assert.Equal(t, res.Cells, prev.Cells) + }) + } + + t.Run("unreadable previous result is an error", func(t *testing.T) { + store := ResultStore{Dir: t.TempDir()} + require.NoError(t, os.WriteFile(filepath.Join(store.Dir, latestResultFile), []byte("{"), 0o600)) + _, err := store.Previous() + assert.Error(t, err) + }) +} + +func TestMarkdown(t *testing.T) { + res := &Result{ + CLIVersion: "v1.2.3", + Cells: []Cell{ + {Feature: featTranscript, Harness: HarnessClaudeCode, HarnessVersion: testHarnessVersion, Model: testModel, Status: StatusPass, Passes: 1, Runs: 1}, + {Feature: "Spec capture instruction", Harness: HarnessClaudeCode, HarnessVersion: testHarnessVersion, Model: testModel, Status: StatusFail, Passes: 1, Runs: 3}, + {Feature: "Session link after push", Harness: HarnessClaudeCode, HarnessVersion: testHarnessVersion, Model: testModel, Status: StatusNotCovered}, + {Feature: featTranscript, Harness: colCursor, Model: "auto", Status: StatusSkipped}, + }, + } + + want := `| Feature | Claude Code 2.1.0 / haiku | Cursor / auto | +|---|---|---| +| Transcript | pass | skipped | +| Spec capture instruction | fail 1/3 | | +| Session link after push | not covered | | +` + assert.Equal(t, want, Markdown(res)) +} + +func TestProblems(t *testing.T) { + res := &Result{Cells: []Cell{ + cell(featTranscript, testModel, SupportYes, StatusPass), + cell("Subagents", testModel, SupportYes, StatusNoCheck), + cell("Skills tracking", testModel, SupportYes, StatusError), + }} + got := res.Problems() + require.Len(t, got, 2) + assert.Equal(t, "Subagents", got[0].Feature) + assert.Equal(t, "Skills tracking", got[1].Feature) +} diff --git a/app/cli/internal/trace/e2e/runner.go b/app/cli/internal/trace/e2e/runner.go new file mode 100644 index 000000000..2164210bc --- /dev/null +++ b/app/cli/internal/trace/e2e/runner.go @@ -0,0 +1,308 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/sync/errgroup" +) + +// Filter limits a manual run to some harnesses, models or features (R-009). +// An empty list matches all. +type Filter struct { + Harnesses []string + Models []string + Features []string +} + +// Empty reports whether the filter matches all cells. Only a run with an empty +// filter is a full run. +func (f Filter) Empty() bool { + return len(f.Harnesses) == 0 && len(f.Models) == 0 && len(f.Features) == 0 +} + +func matches(list []string, v string) bool { + return len(list) == 0 || slices.Contains(list, v) +} + +// Runner expands the expected matrix into scenario sessions, runs them, and +// builds the result matrix. +type Runner struct { + Expected *Expected + Config *Config + Harnesses []Harness + Filter Filter + // ChainloopBin is the chainloop CLI under test. + ChainloopBin string + CLIVersion string + // WorkDir holds the workspaces and the harness output of each session. + WorkDir string + Logf func(format string, args ...any) +} + +// sessionResult is one run of a scenario. err is set when the session could +// not run, which is a suite problem and not a feature failure. +type sessionResult struct { + session *Session + err error +} + +// plannedCell is a cell to test, with the repeat of its feature. +type plannedCell struct { + cell Cell + feature Feature + repeat Repeat +} + +// Run runs the suite and returns the result matrix. +func (r *Runner) Run(ctx context.Context) (*Result, error) { + res := &Result{StartedAt: time.Now().UTC(), CLIVersion: r.CLIVersion, Full: r.Filter.Empty()} + + for _, h := range r.Harnesses { + if !matches(r.Filter.Harnesses, h.Column()) { + continue + } + if !slices.Contains(r.Expected.Harnesses, h.Column()) { + return nil, fmt.Errorf("harness %q has no column in the feature table", h.Column()) + } + hc, ok := r.Config.Harnesses[h.Column()] + if !ok { + return nil, fmt.Errorf("harness %q has no entry in the suite config", h.Column()) + } + + version, versionErr := h.Version(ctx) + if versionErr == nil && version != hc.Version { + r.Logf("%s: installed version %s is not the pinned version %s", h.Column(), version, hc.Version) + } + + for _, model := range hc.Models { + if !matches(r.Filter.Models, model) { + continue + } + cells, err := r.runHarnessModel(ctx, h, version, versionErr, model) + if err != nil { + return nil, err + } + res.Cells = append(res.Cells, cells...) + } + } + + return res, nil +} + +// runHarnessModel tests all the cells of one harness and one model. +func (r *Runner) runHarnessModel(ctx context.Context, h Harness, version string, versionErr error, model string) ([]Cell, error) { + var ( + cells []Cell + planned []plannedCell + runs = make(map[string]int) // scenario name to the runs it needs + ) + + for _, name := range r.Expected.Features { + if !matches(r.Filter.Features, name) { + continue + } + support, _ := r.Expected.Support(name, h.Column()) + c := Cell{Feature: name, Harness: h.Column(), HarnessVersion: version, Model: model, Expected: support} + + def, hasDef := features[name] + switch { + case !support.Tested(): + c.Status, c.Reason = StatusSkipped, "the feature table marks it "+string(support) + case !hasDef || (!def.ControlPlane && len(def.Checks) == 0): + c.Status, c.Reason = StatusNoCheck, "no check is registered for this feature" + case def.ControlPlane: + c.Status, c.Reason = StatusNotCovered, "the feature needs a control plane" + case versionErr != nil: + c.Status, c.Reason = StatusError, versionErr.Error() + default: + repeat := r.Config.repeatFor(name, def.ModelDriven) + planned = append(planned, plannedCell{cell: c, feature: def, repeat: repeat}) + for _, chk := range def.Checks { + runs[chk.Scenario] = max(runs[chk.Scenario], repeat.Runs) + } + continue + } + cells = append(cells, c) + } + + sessions, err := r.runScenarios(ctx, h, model, runs) + if err != nil { + return nil, err + } + + for _, p := range planned { + cells = append(cells, evaluate(p, sessions)) + } + + // Keep the order of the feature table. + order := make(map[string]int, len(r.Expected.Features)) + for i, f := range r.Expected.Features { + order[f] = i + } + slices.SortStableFunc(cells, func(a, b Cell) int { return order[a.Feature] - order[b.Feature] }) + + return cells, nil +} + +// runScenarios runs each scenario the given number of times, in parallel. +func (r *Runner) runScenarios(ctx context.Context, h Harness, model string, runs map[string]int) (map[string][]sessionResult, error) { + var ( + mu sync.Mutex + out = make(map[string][]sessionResult, len(runs)) + ) + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(r.Config.Parallel) + + for name, n := range runs { + sc, ok := scenarios[name] + if !ok { + return nil, fmt.Errorf("unknown scenario %q", name) + } + out[name] = make([]sessionResult, n) + for i := range n { + g.Go(func() error { + s, err := r.runScenario(gctx, h, model, sc, i) + if err != nil { + r.Logf("%s / %s / %s #%d: %v", h.Column(), model, sc.Name, i+1, err) + } + mu.Lock() + out[name][i] = sessionResult{session: s, err: err} + mu.Unlock() + return nil + }) + } + } + _ = g.Wait() + + return out, ctx.Err() +} + +// runScenario runs one session of a scenario: it creates the workspace, runs +// the harness, pushes the commit of the agent, and reads the stored session. +func (r *Runner) runScenario(ctx context.Context, h Harness, model string, sc Scenario, i int) (*Session, error) { + ctx, cancel := context.WithTimeout(ctx, time.Duration(r.Config.ScenarioTimeout)) + defer cancel() + + base := filepath.Join(r.WorkDir, dirName(h.Column()), dirName(model), sc.Name, strconv.Itoa(i+1)) + r.Logf("%s / %s / %s #%d: start in %s", h.Column(), model, sc.Name, i+1, base) + + ws, err := NewWorkspace(ctx, base, r.ChainloopBin, h, sc.Seed) + if err != nil { + return nil, fmt.Errorf("create workspace: %w", err) + } + + out, runErr := h.Run(ctx, RunRequest{Dir: ws.Dir, Model: model, Prompt: sc.Prompt, Env: ws.Env}) + if out != nil { + // The harness output stays next to the workspace to debug a failure. + if err := os.WriteFile(filepath.Join(base, "harness-output.jsonl"), out.Raw, 0o600); err != nil { + return nil, err + } + } + if runErr != nil { + return nil, runErr + } + + s := &Session{Scenario: sc.Name, Model: model, Workspace: ws, Output: out} + + // The agent pushes in the session, as a user does. The pre-push hook + // stores the session in local storage mode. The suite does not push after + // the session: the session end removes the captured specs, so a later push + // stores the session with no spec. A session that did not push stores + // nothing, and its checks fail. + if s.HeadMessage, err = ws.Git(ctx, "log", "-1", "--format=%B"); err != nil { + return nil, err + } + + stored, err := ws.StoredSessions() + if err != nil { + return nil, err + } + switch len(stored) { + case 0: + case 1: + s.Stored = &stored[0] + default: + return nil, fmt.Errorf("the push stored %d sessions, want 1", len(stored)) + } + + r.Logf("%s / %s / %s #%d: done", h.Column(), model, sc.Name, i+1) + return s, nil +} + +// errNoStoredSession is the failure of a check when the push stored no +// session, for example because the agent did not commit or push. +var errNoStoredSession = errors.New("the session stored nothing: the agent did not commit and push") + +// evaluate runs the checks of a planned cell on its sessions and sets its +// status and pass rate. +func evaluate(p plannedCell, sessions map[string][]sessionResult) Cell { + c := p.cell + c.Runs = p.repeat.Runs + + var reasons []string + for i := range p.repeat.Runs { + ok := true + for _, chk := range p.feature.Checks { + sr := sessions[chk.Scenario][i] + if sr.err != nil { + c.Status, c.Reason = StatusError, fmt.Sprintf("scenario %s could not run: %v", chk.Scenario, sr.err) + return c + } + + var err error + if sr.session.Stored == nil { + err = errNoStoredSession + } else { + err = chk.Verify(sr.session) + } + if errors.Is(err, errNotCovered) { + c.Status, c.Reason = StatusNotCovered, err.Error() + return c + } + if err != nil { + ok = false + reasons = append(reasons, fmt.Sprintf("run %d: %v", i+1, err)) + } + } + if ok { + c.Passes++ + } + } + + c.Status = StatusFail + if c.Passes >= p.repeat.Threshold { + c.Status = StatusPass + } + c.Reason = strings.Join(reasons, "; ") + return c +} + +// dirName turns a harness or model name into a directory name. +func dirName(s string) string { + return strings.NewReplacer(" ", "-", "/", "-", ":", "-").Replace(strings.ToLower(s)) +} diff --git a/app/cli/internal/trace/e2e/runner_test.go b/app/cli/internal/trace/e2e/runner_test.go new file mode 100644 index 000000000..668f69e27 --- /dev/null +++ b/app/cli/internal/trace/e2e/runner_test.go @@ -0,0 +1,184 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEvaluate(t *testing.T) { + pass := func(*Session) error { return nil } + fail := func(*Session) error { return errors.New("boom") } + notCovered := func(*Session) error { return fmt.Errorf("%w: headless", errNotCovered) } + stored := &Session{Stored: &StoredSession{}} + + // failOn fails the given runs of a scenario, counted from 1. + failOn := func(runs ...int) func(*Session) error { + return func(s *Session) error { + for _, r := range runs { + if s.Model == fmt.Sprint(r) { + return errors.New("boom") + } + } + return nil + } + } + numbered := func(n int) []sessionResult { + out := make([]sessionResult, n) + for i := range out { + out[i] = sessionResult{session: &Session{Model: fmt.Sprint(i + 1), Stored: &StoredSession{}}} + } + return out + } + + testCases := []struct { + name string + checks []Check + repeat Repeat + sessions map[string][]sessionResult + wantStatus Status + wantPasses int + }{ + { + name: "single run passes", + checks: []Check{{"a", pass}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{session: stored}}}, + wantStatus: StatusPass, + wantPasses: 1, + }, + { + name: "all checks must pass", + checks: []Check{{"a", pass}, {"a", fail}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{session: stored}}}, + wantStatus: StatusFail, + }, + { + name: "two of three runs pass", + checks: []Check{{"a", failOn(2)}}, + repeat: Repeat{3, 2}, + sessions: map[string][]sessionResult{"a": numbered(3)}, + wantStatus: StatusPass, + wantPasses: 2, + }, + { + name: "one of three runs passes", + checks: []Check{{"a", failOn(1, 3)}}, + repeat: Repeat{3, 2}, + sessions: map[string][]sessionResult{"a": numbered(3)}, + wantStatus: StatusFail, + wantPasses: 1, + }, + { + name: "scenario that could not run is a suite error", + checks: []Check{{"a", pass}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{err: errors.New("no harness")}}}, + wantStatus: StatusError, + }, + { + name: "check that cannot see the feature", + checks: []Check{{"a", notCovered}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{session: stored}}}, + wantStatus: StatusNotCovered, + }, + { + name: "no stored session fails", + checks: []Check{{"a", pass}}, + repeat: Repeat{1, 1}, + sessions: map[string][]sessionResult{"a": {{session: &Session{}}}}, + wantStatus: StatusFail, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := evaluate(plannedCell{feature: Feature{Checks: tc.checks}, repeat: tc.repeat}, tc.sessions) + assert.Equal(t, tc.wantStatus, got.Status, got.Reason) + assert.Equal(t, tc.wantPasses, got.Passes) + assert.Equal(t, tc.repeat.Runs, got.Runs) + }) + } +} + +// fakeHarness fails to report its version, so the runner tests the planning +// of the cells and runs no session. +type fakeHarness struct{} + +func (fakeHarness) Column() string { return HarnessClaudeCode } +func (fakeHarness) TraceFlag() string { return "--claude" } +func (fakeHarness) Version(context.Context) (string, error) { + return "", errors.New("not installed") +} +func (fakeHarness) Run(context.Context, RunRequest) (*RunOutput, error) { + return nil, errors.New("not installed") +} + +func TestRunnerPlansCells(t *testing.T) { + exp, err := ParseExpected(strings.NewReader(`| Feature | What it means | Claude Code | Cursor | +|---|---|---|---| +| Transcript | x | Yes | Yes | +| Subagents | x | No | No | +| Attestation at push | x | Yes | Yes | +| Brand new feature | x | Unknown | Unknown | +`)) + require.NoError(t, err) + cfg, err := LoadConfig(nil) + require.NoError(t, err) + + r := &Runner{Expected: exp, Config: cfg, Harnesses: []Harness{fakeHarness{}}, WorkDir: t.TempDir(), Logf: t.Logf} + res, err := r.Run(context.Background()) + require.NoError(t, err) + + got := make(map[string]Status) + for _, c := range res.Cells { + got[c.Feature] = c.Status + } + assert.Equal(t, map[string]Status{ + featTranscript: StatusError, // the harness is not installed + "Subagents": StatusSkipped, + "Attestation at push": StatusNotCovered, + "Brand new feature": StatusNoCheck, + }, got) + assert.True(t, res.Full) +} + +// TestFeaturesMatchREADME keeps the registered checks in line with the feature +// table: a check whose feature was renamed in the table would never run. +func TestFeaturesMatchREADME(t *testing.T) { + f, err := os.Open(traceREADMEPath) + require.NoError(t, err) + defer f.Close() + exp, err := ParseExpected(f) + require.NoError(t, err) + + for name, def := range features { + assert.Contains(t, exp.Features, name, "registered feature is not in the table") + for _, chk := range def.Checks { + assert.Contains(t, scenarios, chk.Scenario, "feature %q", name) + } + } +} diff --git a/app/cli/internal/trace/e2e/scenarios.go b/app/cli/internal/trace/e2e/scenarios.go new file mode 100644 index 000000000..52ed8bab9 --- /dev/null +++ b/app/cli/internal/trace/e2e/scenarios.go @@ -0,0 +1,96 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +// Scenario is a scripted task in a throwaway repository. One session runs one +// scenario, and the checks of all the features that it touches read its +// result (D-008). Scenarios are shared by all harnesses (R-011). +type Scenario struct { + Name string + // Seed are the files of the repository before the session, by path. + Seed map[string]string + // Prompt is the only user prompt of the session. + Prompt string +} + +// The fake secrets of the scenarios. They match the GitHub token rule of the +// secret scanner, and they are not real tokens. +const ( + // fakeSecret is in the prompt. The stored session must not hold it. + fakeSecret = "ghp_8fK2mQ9xLp4ZrT7vWc1NbH6yJd3sGa5eUo0i" //nolint:gosec // a fake token that the suite expects redaction to remove + // fakeAllowedSecret is on a line with an allow marker. The stored session + // keeps it. + fakeAllowedSecret = "ghp_Zx7Lq2Wm9Rt4Yp6Vn3Bk8Hd1Jf5Gs0CaEu2o" //nolint:gosec // a fake token on a line with an allow marker +) + +// The README of every scenario repository. +const ( + readmePath = "README.md" + seedReadme = "# Trace E2E\n" +) + +const ( + scenarioEditAndCommit = "edit-and-commit" + scenarioTicket = "ticket" + scenarioSkill = "skill" +) + +const ( + ticketPath = "docs/ticket-e2e-42.md" + ticketTitle = "E2E-42: Add a farewell file" + skillName = "e2e-farewell" +) + +// scenarios are the scenarios of the suite, by name. +var scenarios = map[string]Scenario{ + scenarioEditAndCommit: { + Name: scenarioEditAndCommit, + Seed: map[string]string{ + readmePath: seedReadme, + "fixture.txt": "token = " + fakeAllowedSecret + " # gitleaks:allow\n", + }, + Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: +1. Read the file fixture.txt. +2. With your file-writing tool, create the file greeting.txt with exactly these three lines: hello, from, the agent. +3. Run this shell command: printf 'one\ntwo\n' > shell.txt +4. Run this shell command: git add greeting.txt shell.txt && git commit -m "add greeting" +5. Run this shell command: git push +The deploy token ` + fakeSecret + ` is for a later task: do not use it.`, + }, + scenarioTicket: { + Name: scenarioTicket, + Seed: map[string]string{ + readmePath: seedReadme, + ticketPath: "# " + ticketTitle + "\n\nCreate the file farewell.txt with the single line: goodbye\n", + }, + Prompt: "This is an automated test. Implement the ticket in " + ticketPath + ". Then commit your change and push it with git.", + }, + scenarioSkill: { + Name: scenarioSkill, + Seed: map[string]string{ + readmePath: seedReadme, + ".claude/skills/" + skillName + "/SKILL.md": `--- +name: ` + skillName + ` +description: Use when asked to add the farewell file to the repository. +--- + +Create the file farewell.txt with the single line: goodbye +Then run: git add farewell.txt && git commit -m "add farewell" && git push +`, + }, + Prompt: "This is an automated test. Use the " + skillName + " skill.", + }, +} diff --git a/app/cli/internal/trace/e2e/workspace.go b/app/cli/internal/trace/e2e/workspace.go new file mode 100644 index 000000000..b37e8498e --- /dev/null +++ b/app/cli/internal/trace/e2e/workspace.go @@ -0,0 +1,209 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession" +) + +// localSessionsDir is where local storage mode writes the sessions, under the +// default trace directory. +var localSessionsDir = filepath.Join(".git", "chainloop-trace", "local-sessions") + +// gitConfig isolates the git commands of a workspace, and of the harness +// that runs in it, from the git configuration of the user who runs the suite. +// A global hooks path would hide the Trace git hooks, and commit signing could +// ask for a passphrase. +const gitConfig = `[user] + name = Chainloop Trace E2E + email = trace-e2e@chainloop.local +[commit] + gpgsign = false +[tag] + gpgsign = false +[init] + defaultBranch = main +` + +// Workspace is a throwaway git repository with Trace in local storage mode, +// and a local bare remote to push to. +type Workspace struct { + // Dir is the repository. + Dir string + // Env is the environment of every command in the workspace, the harness + // included: the chainloop binary under test first in PATH, and an + // isolated git configuration. + Env []string +} + +// NewWorkspace creates a workspace under base. It writes the seed files, +// runs `chainloop trace init --local` for the harness, commits the result and +// pushes it, so the session starts from a clean, pushed tree. +func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, seed map[string]string) (*Workspace, error) { + repo := filepath.Join(base, "repo") + remote := filepath.Join(base, "remote.git") + gitConfigPath := filepath.Join(base, "gitconfig") + + if err := os.MkdirAll(repo, 0o750); err != nil { + return nil, err + } + if err := os.WriteFile(gitConfigPath, []byte(gitConfig), 0o600); err != nil { + return nil, err + } + + w := &Workspace{ + Dir: repo, + Env: append(os.Environ(), + "PATH="+filepath.Dir(chainloopBin)+string(os.PathListSeparator)+os.Getenv("PATH"), + "GIT_CONFIG_GLOBAL="+gitConfigPath, + "GIT_CONFIG_NOSYSTEM=1", + "CHAINLOOP_NO_PROMPT=1", + "DO_NOT_TRACK=1", + ), + } + + if _, err := w.run(ctx, base, "git", "init", "--bare", "--quiet", remote); err != nil { + return nil, err + } + if _, err := w.Git(ctx, "init", "--quiet"); err != nil { + return nil, err + } + + for name, content := range seed { + path := filepath.Join(repo, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return nil, err + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + return nil, err + } + } + + if _, err := w.run(ctx, repo, chainloopBin, "trace", "init", "--local", h.TraceFlag()); err != nil { + return nil, err + } + + for _, args := range [][]string{ + {"add", "-A"}, + {"commit", "--quiet", "-m", "seed"}, + {"remote", "add", "origin", remote}, + {"push", "--quiet", "-u", "origin", "main"}, + } { + if _, err := w.Git(ctx, args...); err != nil { + return nil, err + } + } + + return w, nil +} + +// Git runs a git command in the repository and returns its output. +func (w *Workspace) Git(ctx context.Context, args ...string) (string, error) { + return w.run(ctx, w.Dir, "git", args...) +} + +func (w *Workspace) run(ctx context.Context, dir, name string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + cmd.Env = w.Env + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return string(out), nil +} + +// StoredSession is one session that the pre-push hook stored in local storage +// mode. +type StoredSession struct { + // Dir is the directory of the stored session. + Dir string + // Evidence is the AI coding session evidence. + Evidence aicodingsession.Evidence +} + +// Material returns the stored content of a material, by its digest. +func (s *StoredSession) Material(digest string) ([]byte, error) { + return os.ReadFile(filepath.Join(s.Dir, "evidence", strings.ReplaceAll(digest, ":", "-"))) +} + +// HasStatement reports whether the stored session has its statement index. +func (s *StoredSession) HasStatement() bool { + _, err := os.Stat(filepath.Join(s.Dir, "statement.json")) + return err == nil +} + +// StoredSessions reads the sessions that local storage mode stored. Each +// session directory holds the AI coding session evidence among its materials. +func (w *Workspace) StoredSessions() ([]StoredSession, error) { + root := filepath.Join(w.Dir, localSessionsDir) + dirs, err := os.ReadDir(root) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + + var out []StoredSession + for _, d := range dirs { + if !d.IsDir() { + continue + } + dir := filepath.Join(root, d.Name()) + ev, err := findSessionEvidence(filepath.Join(dir, "evidence")) + if err != nil { + return nil, fmt.Errorf("stored session %s: %w", d.Name(), err) + } + out = append(out, StoredSession{Dir: dir, Evidence: *ev}) + } + + return out, nil +} + +// findSessionEvidence finds the AI coding session evidence among the +// materials of a stored session. The other materials, for example spec +// sources, are not JSON or have another ID. +func findSessionEvidence(dir string) (*aicodingsession.Evidence, error) { + files, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + for _, f := range files { + data, err := os.ReadFile(filepath.Join(dir, f.Name())) + if err != nil { + return nil, err + } + var ev aicodingsession.Evidence + if json.Unmarshal(data, &ev) == nil && ev.ID == aicodingsession.EvidenceID { + return &ev, nil + } + } + return nil, errors.New("no AI coding session evidence") +} From b7504e5cb3767f16f2ce8f6058e594c037b91cca Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Mon, 19 Oct 2026 10:30:54 +0200 Subject: [PATCH 2/2] fix(trace): e2e README coverage and secret scan markers State in the README which cells the e2e suite tests and how it reports the others, and that it does not install the configured agent version. Mark the fake tokens of the scenarios for the repository secret scan. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: be5af72c-30e7-48f4-b46c-dc7a5a754c41 --- app/cli/internal/trace/README.md | 10 ++++++++-- app/cli/internal/trace/e2e/scenarios.go | 12 +++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 1c73bd937..e7729c171 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -131,9 +131,15 @@ All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`. ## End-to-end tests -The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. It tests each cell marked **Yes**, **Partial**, or **Unknown**, and it skips the other cells. A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. +The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code column only. -`e2e/config.yaml` pins the version of each agent and lists the models. The suite uses model tokens. To run it: +A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. The suite tests a cell marked **Yes**, **Partial**, or **Unknown** only when the row has a registered check. It reports the other cells as follows: + +- A cell marked **No** or **Not applicable** is `skipped`. +- A feature that needs a control plane is `not covered`, because the suite uses local storage mode only. +- A row with no registered check is `no check`. + +`e2e/config.yaml` sets the version of each agent that the suite expects, and the models. The suite does not install the agent. It runs the installed version, and it logs a warning when that version is not the one in the configuration. The suite uses model tokens. To run it: ```bash make -C app/cli test-trace-e2e diff --git a/app/cli/internal/trace/e2e/scenarios.go b/app/cli/internal/trace/e2e/scenarios.go index 52ed8bab9..b8b65ec25 100644 --- a/app/cli/internal/trace/e2e/scenarios.go +++ b/app/cli/internal/trace/e2e/scenarios.go @@ -27,13 +27,15 @@ type Scenario struct { } // The fake secrets of the scenarios. They match the GitHub token rule of the -// secret scanner, and they are not real tokens. +// secret scanner, and they are not real tokens. The allow markers on these +// source lines keep the repository secret scan quiet. They do not reach the +// session: the prompt and the seed files carry only the token values. const ( // fakeSecret is in the prompt. The stored session must not hold it. - fakeSecret = "ghp_8fK2mQ9xLp4ZrT7vWc1NbH6yJd3sGa5eUo0i" //nolint:gosec // a fake token that the suite expects redaction to remove - // fakeAllowedSecret is on a line with an allow marker. The stored session - // keeps it. - fakeAllowedSecret = "ghp_Zx7Lq2Wm9Rt4Yp6Vn3Bk8Hd1Jf5Gs0CaEu2o" //nolint:gosec // a fake token on a line with an allow marker + fakeSecret = "ghp_8fK2mQ9xLp4ZrT7vWc1NbH6yJd3sGa5eUo0i" //nolint:gosec // gitleaks:allow a fake token that the suite expects redaction to remove + // fakeAllowedSecret is on a line with an allow marker in the seed file. + // The stored session keeps it. + fakeAllowedSecret = "ghp_Zx7Lq2Wm9Rt4Yp6Vn3Bk8Hd1Jf5Gs0CaEu2o" //nolint:gosec // gitleaks:allow a fake token on a line with an allow marker ) // The README of every scenario repository.