feat(trace): end-to-end test matrix for Claude Code

app/cli/Makefile

@@ -12,6 +12,11 @@ build:
 test:
 	go test ./...
 
+.PHONY: test-trace-e2e
+# run the Chainloop Trace end-to-end suite with real agents and models (uses model tokens)
+test-trace-e2e:
+	go test -tags e2e -run TestTraceMatrix -timeout 60m -v ./internal/trace/e2e/
+
 .PHONY: lint
 # lint
 lint:

app/cli/internal/trace/README.md

@@ -128,3 +128,21 @@ Why not Yes:
 | OpenCode | `.opencode/plugins/chainloop-trace.ts` | Session start and end, user prompt, and before and after each tool. One plugin covers both versions: `server()` for 1.x (1.3.4 or later) and `setup()` for 2. The provider picks the export command from `opencode --version`. |
 
 All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`.
+
+## End-to-end tests
+
+The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code column only.
+
+A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. The suite tests a cell marked **Yes**, **Partial**, or **Unknown** only when the row has a registered check. It reports the other cells as follows:
+
+- A cell marked **No** or **Not applicable** is `skipped`.
+- A feature that needs a control plane is `not covered`, because the suite uses local storage mode only.
+- A row with no registered check is `no check`.
+
+`e2e/config.yaml` sets the version of each agent that the suite expects, and the models. The suite does not install the agent. It runs the installed version, and it logs a warning when that version is not the one in the configuration. The suite uses model tokens. To run it:
+
+```bash
+make -C app/cli test-trace-e2e
+```
+
+`CHAINLOOP_E2E_HARNESS`, `CHAINLOOP_E2E_MODEL`, and `CHAINLOOP_E2E_FEATURE` limit a run to some columns, models, or features (comma-separated). Set `CHAINLOOP_E2E_WORK_DIR` to keep the workspaces and the agent output. The results go to `e2e/results/`, or to `CHAINLOOP_E2E_RESULTS_DIR`. Only a run with no filter becomes the previous result of the next comparison.

app/cli/internal/trace/e2e/.gitignore

@@ -0,0 +1 @@
+results/

app/cli/internal/trace/e2e/checks.go

@@ -0,0 +1,309 @@
+//
+// Copyright 2026 The Chainloop Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/chainloop-dev/chainloop/app/cli/internal/trace"
+	"github.com/chainloop-dev/chainloop/app/cli/internal/trace/state"
+	"github.com/chainloop-dev/chainloop/pkg/attestation/crafter/materials/aicodingsession"
+)
+
+// errNotCovered marks a check that cannot see the feature in this run, for
+// example because the harness does not report hook messages in headless mode
+// (R-004).
+var errNotCovered = errors.New("not covered")
+
+// Session is the result of one scenario session.
+type Session struct {
+	Scenario  string
+	Model     string
+	Workspace *Workspace
+	Output    *RunOutput
+	// Stored is the session that the push stored in local storage mode.
+	Stored *StoredSession
+	// HeadMessage is the message of the commit that the agent made.
+	HeadMessage string
+}
+
+// Check verifies one feature on the result of one scenario session.
+type Check struct {
+	Scenario string
+	Verify   func(s *Session) error
+}
+
+// Feature is the test definition of one feature row of the table. The key of
+// the features map is the feature name in the table (R-002).
+type Feature struct {
+	// ControlPlane is true for a feature that needs a control plane. The suite
+	// runs in local storage mode only, so its cells are "not covered" (D-007).
+	ControlPlane bool
+	// ModelDriven is true for a feature that depends on the model following an
+	// instruction. Its cells run more than once (R-006).
+	ModelDriven bool
+	// Checks must all pass for a run of the feature to pass.
+	Checks []Check
+}
+
+// features are the test definitions of the feature rows. A row of the table
+// with no entry here is reported as "no check".
+var features = map[string]Feature{
+	// Session collection
+	"Transcript":           {Checks: []Check{{scenarioEditAndCommit, checkTranscript}}},
+	"Token usage and cost": {Checks: []Check{{scenarioEditAndCommit, checkTokenUsage}}},
+	"Model detection":      {Checks: []Check{{scenarioEditAndCommit, checkModel}}},
+
+	// Code attribution
+	"Line attribution":          {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("greeting.txt")}}},
+	"Shell changes attribution": {Checks: []Check{{scenarioEditAndCommit, checkFileAttributedToAI("shell.txt")}}},
+
+	// User experience
+	"Welcome message":         {Checks: []Check{{scenarioEditAndCommit, checkWelcomeMessage}}},
+	"Session link after push": {ControlPlane: true},
+	"Credentials warning":     {ControlPlane: true},
+
+	// Specs and skills
+	"Spec capture instruction":        {ModelDriven: true, Checks: []Check{{scenarioTicket, checkTicketCaptured}}},
+	"Spec capture reminder":           {Checks: []Check{{scenarioEditAndCommit, checkPromptReminder}}},
+	"Local spec sources read at push": {ModelDriven: true, Checks: []Check{{scenarioTicket, checkTicketContentStored}}},
+	"Skills tracking":                 {ModelDriven: true, Checks: []Check{{scenarioSkill, checkSkillTracked}}},
+
+	// Security
+	"Secret redaction":           {Checks: []Check{{scenarioEditAndCommit, checkSecretRedacted}}},
+	"Allow markers in redaction": {Checks: []Check{{scenarioEditAndCommit, checkAllowedSecretKept}}},
+
+	// Git integration
+	"Commit trailer":        {Checks: []Check{{scenarioEditAndCommit, checkCommitTrailer}}},
+	"Attestation at push":   {ControlPlane: true},
+	"Local storage at push": {Checks: []Check{{scenarioEditAndCommit, checkLocalStorage}}},
+}
+
+func evidence(s *Session) *aicodingsession.Data {
+	return &s.Stored.Evidence.Data
+}
+
+func checkTranscript(s *Session) error {
+	main := evidence(s).RawSession["main"]
+	if len(main) == 0 {
+		return errors.New("the evidence has no main transcript")
+	}
+	for _, msg := range main {
+		if bytes.Contains(msg, []byte("greeting.txt")) {
+			return nil
+		}
+	}
+	return errors.New("the transcript does not show the work of the session")
+}
+
+func checkTokenUsage(s *Session) error {
+	u := evidence(s).Usage
+	if u == nil || u.TotalTokens == 0 {
+		return errors.New("the evidence has no token usage")
+	}
+	if u.EstimatedCostUSD <= 0 {
+		return errors.New("the evidence has no cost")
+	}
+	return nil
+}
+
+func checkModel(s *Session) error {
+	m := evidence(s).Model
+	if m == nil || m.Primary == "" {
+		return errors.New("the evidence has no primary model")
+	}
+	if !strings.Contains(strings.ToLower(m.Primary), strings.ToLower(s.Model)) {
+		return fmt.Errorf("primary model %q is not the model of the session %q", m.Primary, s.Model)
+	}
+	return nil
+}
+
+func checkFileAttributedToAI(path string) func(s *Session) error {
+	return func(s *Session) error {
+		cc := evidence(s).CodeChanges
+		if cc == nil {
+			return errors.New("the evidence has no code changes")
+		}
+		for _, f := range cc.Files {
+			if f.Path != path {
+				continue
+			}
+			if f.Attribution != trace.AttributionAI || f.LinesAdded == 0 {
+				return fmt.Errorf("%s: attribution %q with %d lines added, want AI lines", path, f.Attribution, f.LinesAdded)
+			}
+			return nil
+		}
+		return fmt.Errorf("the code changes do not list %s", path)
+	}
+}
+
+// hookMessage returns the first hook message of an event, or errNotCovered
+// when the harness does not report hook messages.
+func hookMessage(s *Session, event string) (*HookMessage, error) {
+	if !s.Output.HooksReported {
+		return nil, fmt.Errorf("%w: the harness does not report hook messages in headless mode", errNotCovered)
+	}
+	for i := range s.Output.Hooks {
+		if s.Output.Hooks[i].Event == event {
+			return &s.Output.Hooks[i], nil
+		}
+	}
+	return nil, fmt.Errorf("no %s hook message", event)
+}
+
+// checkWelcomeMessage checks the banner that the user sees at session start.
+// In local storage mode, it names the on-disk destination.
+func checkWelcomeMessage(s *Session) error {
+	msg, err := hookMessage(s, "SessionStart")
+	if err != nil {
+		return err
+	}
+	if !strings.Contains(msg.UserMessage, "local-sessions") {
+		return fmt.Errorf("the session start message does not name the local sessions folder: %q", msg.UserMessage)
+	}
+	return nil
+}
+
+func checkPromptReminder(s *Session) error {
+	msg, err := hookMessage(s, "UserPromptSubmit")
+	if err != nil {
+		return err
+	}
+	if msg.ModelContext == "" {
+		return errors.New("the user prompt hook gave no reminder to the model")
+	}
+	return nil
+}
+
+// ticketSpec returns the captured spec entry of the ticket of the scenario.
+func ticketSpec(s *Session) (*aicodingsession.SpecEntry, error) {
+	for i, e := range evidence(s).Spec {
+		if e.Kind != aicodingsession.SpecKindSkill && strings.Contains(e.URI, filepath.Base(ticketPath)) {
+			return &evidence(s).Spec[i], nil
+		}
+	}
+	return nil, errors.New("the evidence has no spec entry for the ticket")
+}
+
+func checkTicketCaptured(s *Session) error {
+	_, err := ticketSpec(s)
+	return err
+}
+
+func checkTicketContentStored(s *Session) error {
+	e, err := ticketSpec(s)
+	if err != nil {
+		return err
+	}
+	content, err := s.Stored.Material(e.Digest)
+	if err != nil {
+		return fmt.Errorf("read the spec material: %w", err)
+	}
+	if !bytes.Contains(content, []byte(ticketTitle)) {
+		return errors.New("the spec material does not hold the text of the ticket file")
+	}
+	return nil
+}
+
+func checkSkillTracked(s *Session) error {
+	for _, e := range evidence(s).Spec {
+		if e.Kind != aicodingsession.SpecKindSkill || !strings.Contains(e.Title+e.URI, skillName) {
+			continue
+		}
+		if e.Metadata == nil || e.Metadata.InvocationCount == 0 {
+			return errors.New("the skill entry has no invocation")
+		}
+		return nil
+	}
+	return fmt.Errorf("the evidence has no entry for the skill %s", skillName)
+}
+
+// storedContains reports whether a file of the stored session holds the text.
+func storedContains(s *Session, text string) (bool, error) {
+	found := false
+	err := filepath.WalkDir(s.Stored.Dir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() || found {
+			return err
+		}
+		data, err := os.ReadFile(path) //nolint:gosec // a throwaway workspace of the suite
+		if err != nil {
+			return err
+		}
+		found = bytes.Contains(data, []byte(text))
+		return nil
+	})
+	return found, err
+}
+
+func checkSecretRedacted(s *Session) error {
+	if err := checkTranscript(s); err != nil {
+		return fmt.Errorf("nothing to redact: %w", err)
+	}
+	found, err := storedContains(s, fakeSecret)
+	if err != nil {
+		return err
+	}
+	if found {
+		return errors.New("the stored session holds the secret of the prompt")
+	}
+	// The prompt is in the transcript, so its secret must show as a
+	// placeholder. With no placeholder, the check proves nothing.
+	if found, err = storedContains(s, redactedPlaceholder); err != nil {
+		return err
+	}
+	if !found {
+		return errors.New("the stored session has no redaction placeholder")
+	}
+	return nil
+}
+
+// redactedPlaceholder is the start of the text that replaces a redacted
+// secret, as internal/redaction writes it.
+const redactedPlaceholder = "[CHAINLOOP_TRACE_REDACTED"
+
+func checkAllowedSecretKept(s *Session) error {
+	found, err := storedContains(s, fakeAllowedSecret)
+	if err != nil {
+		return err
+	}
+	if !found {
+		return errors.New("the secret on the line with an allow marker was redacted, or the session did not read it")
+	}
+	return nil
+}
+
+func checkCommitTrailer(s *Session) error {
+	ids := state.ParseSessionIDsFromTrailer(s.HeadMessage)
+	for _, id := range ids {
+		if id == s.Stored.Evidence.Data.Session.ID {
+			return nil
+		}
+	}
+	return fmt.Errorf("the commit trailer has sessions %v, not the stored session", ids)
+}
+
+func checkLocalStorage(s *Session) error {
+	if !s.Stored.HasStatement() {
+		return errors.New("the stored session has no statement.json")
+	}
+	return nil
+}