From 94fcd695ab4351544a26178eb73a83ebea80f46c Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Mon, 19 Oct 2026 20:38:57 +0200 Subject: [PATCH 1/3] feat(trace): e2e adapter for OpenCode 2 Add the OpenCode 2 harness to the Trace end-to-end suite. The suite runs the harnesses that config.yaml lists. The skill files and the user hook file come from the harness, and the suite fails when an agent commits in the checkout that runs it. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: be5af72c-30e7-48f4-b46c-dc7a5a754c41 --- app/cli/internal/trace/README.md | 14 +- app/cli/internal/trace/e2e/checks.go | 21 +- app/cli/internal/trace/e2e/checks_more.go | 39 ++- app/cli/internal/trace/e2e/claude.go | 5 + app/cli/internal/trace/e2e/config.yaml | 5 + app/cli/internal/trace/e2e/e2e_test.go | 32 ++- app/cli/internal/trace/e2e/harness.go | 30 +- app/cli/internal/trace/e2e/opencode.go | 299 ++++++++++++++++++++ app/cli/internal/trace/e2e/opencode_test.go | 145 ++++++++++ app/cli/internal/trace/e2e/runner_test.go | 1 + app/cli/internal/trace/e2e/scenarios.go | 15 +- app/cli/internal/trace/e2e/workspace.go | 8 +- 12 files changed, 587 insertions(+), 27 deletions(-) create mode 100644 app/cli/internal/trace/e2e/opencode.go create mode 100644 app/cli/internal/trace/e2e/opencode_test.go diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index cd9faccfd..874cdb057 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -14,8 +14,8 @@ How Trace is added to a repository and to an agent. | Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | |---------|---------------|-------------|--------|--------------|------------| -| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Unknown | Unknown | Unknown | -| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Unknown | Unknown | Unknown | +| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Unknown | Unknown | Yes | +| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Unknown | Unknown | Yes | | More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | Why not Yes: @@ -33,7 +33,7 @@ What the session evidence holds about the conversation. | Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | | Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | | Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | -| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Unknown | +| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Yes | Why not Yes: @@ -42,7 +42,7 @@ Why not Yes: - **Model detection, Cursor:** the primary model comes from the session-start hook. The transcript does not tell the models, so the evidence does not list the other models. - **Subagents, Cursor:** the provider does not read subagent conversations. - **Subagents, OpenCode:** a subagent runs in a child session with its own export. It is not part of the evidence of its parent session. -- **Missed session start, Cursor and OpenCode:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **Missed session start, Cursor and OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). ## Code attribution @@ -91,7 +91,7 @@ What the session was asked to build, and which skills it used. | Spec capture instruction | At session start, the agent is told to capture the specs of the session (tickets, documents, plans, images) in the spec folder. A plan is captured only when it comes from an external source or from the plan mode of the agent: an agreement in the chat is not a plan. A pasted image is not captured, also when the agent has a file path for it. The agent tells the user in one line when it captures a file. | Yes | Yes | Yes | Yes | | Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | | Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | -| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Unknown | Unknown | Unknown | +| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Unknown | Unknown | Yes | | Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | | Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | | Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | @@ -99,7 +99,7 @@ What the session was asked to build, and which skills it used. Why not Yes: - **Spec capture reminder, Cursor:** Cursor has no hook at each user prompt. The agent gets the session-start instruction only. -- **Spec source role, title, and description, Cursor and OpenCode:** not verified yet. The end-to-end suite finds the values (spec issue-3635). +- **Spec source role, title, and description, Cursor and OpenCode 1.x:** not verified yet. The end-to-end suite finds the values (spec issue-3635). - **Skills tracking, Cursor:** the provider does not track skills. The evidence has no skill entries, which means "not recorded", not "no skill used". - **Skills tracking, OpenCode:** only the skills that the model starts with the `skill` tool are counted. Skills that the user starts, and skills used in subagents, are not. - **Spec source pointers, Cursor and OpenCode:** the provider has no finder for the copies in its transcript yet. For OpenCode, the rebuilt transcript holds no tool outputs, so only the write copies would apply. @@ -157,7 +157,7 @@ All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`. ## End-to-end tests -The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code column only. +The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code and OpenCode 2 columns. OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace, because OpenCode takes its project folder from `PWD`. The OpenCode 2 model in `e2e/config.yaml` is an OpenRouter model, so the run needs `OPENROUTER_API_KEY`. A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. The suite tests a cell marked **Yes**, **Partial**, or **Unknown** only when the row has a registered check. It reports the other cells as follows: diff --git a/app/cli/internal/trace/e2e/checks.go b/app/cli/internal/trace/e2e/checks.go index 09e569f89..28ffb8226 100644 --- a/app/cli/internal/trace/e2e/checks.go +++ b/app/cli/internal/trace/e2e/checks.go @@ -21,6 +21,7 @@ import ( "fmt" "io/fs" "os" + "path" "path/filepath" "strings" @@ -165,7 +166,11 @@ func checkModel(s *Session) error { if m == nil || m.Primary == "" { return errors.New("the evidence has no primary model") } - if !strings.Contains(strings.ToLower(m.Primary), strings.ToLower(s.Model)) { + // A model can have a provider prefix, for example + // amazon-bedrock/eu.anthropic.claude-haiku-5-5, that the evidence leaves + // out, and a short name, for example haiku, that the evidence spells out. + primary, want := strings.ToLower(m.Primary), strings.ToLower(path.Base(s.Model)) + if !strings.Contains(primary, want) && !strings.Contains(want, primary) { return fmt.Errorf("primary model %q is not the model of the session %q", m.Primary, s.Model) } return nil @@ -356,15 +361,23 @@ func checkSecretRedacted(s *Session) error { // secret, as internal/redaction writes it. const redactedPlaceholder = "[CHAINLOOP_TRACE_REDACTED" +// checkAllowedSecretKept checks that the secret on the line with an allow +// marker is in the stored session. The line comes from the output of the +// tool that read the fixture. A transcript with no tool outputs holds no +// copy of the line, so it cannot show the feature. func checkAllowedSecretKept(s *Session) error { found, err := storedContains(s, fakeAllowedSecret) + if err != nil || found { + return err + } + marked, err := storedContains(s, allowMarker) if err != nil { return err } - if !found { - return errors.New("the secret on the line with an allow marker was redacted, or the session did not read it") + if !marked { + return fmt.Errorf("%w: the stored session holds no copy of the fixture line, for example because the transcript has no tool outputs", errNotCovered) } - return nil + return errors.New("the secret on the line with an allow marker was redacted") } func checkCommitTrailer(s *Session) error { diff --git a/app/cli/internal/trace/e2e/checks_more.go b/app/cli/internal/trace/e2e/checks_more.go index 9f3fad2a6..ebcb8c5fb 100644 --- a/app/cli/internal/trace/e2e/checks_more.go +++ b/app/cli/internal/trace/e2e/checks_more.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "image/png" + "io/fs" "os" "path/filepath" "slices" @@ -36,9 +37,29 @@ import ( // commandTimeout limits a command that a check runs in the workspace. const commandTimeout = 2 * time.Minute -// readHookConfig reads the hook configuration file of the harness. +// readHookFile reads a hook file of the harness. A missing file is empty. +func readHookFile(s *Session, path string) ([]byte, error) { + data, err := os.ReadFile(filepath.Join(s.Workspace.Dir, path)) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return data, err +} + +// readHookConfig reads the hook configuration of the harness: the file of +// the Chainloop hooks, and the file of the hook of the user when it is +// another file. func readHookConfig(s *Session) ([]byte, error) { - return os.ReadFile(filepath.Join(s.Workspace.Dir, s.Harness.HookConfig().Path)) + hc := s.Harness.HookConfig() + out, err := readHookFile(s, hc.Path) + if err != nil || hc.UserFile() == hc.Path { + return out, err + } + user, err := readHookFile(s, hc.UserFile()) + if err != nil { + return nil, err + } + return append(out, user...), nil } // checkHookInstallation checks that `trace init` installs the Chainloop @@ -90,13 +111,23 @@ func checkHookInstallation(s *Session) error { // checkHookConfigRestored checks that the hook configuration is the same as // before the wrapped session. func checkHookConfigRestored(s *Session) error { - got, err := readHookConfig(s) + hc := s.Harness.HookConfig() + got, err := readHookFile(s, hc.UserFile()) if err != nil { return err } - if string(got) != s.Harness.HookConfig().UserConfig { + if string(got) != hc.UserConfig { return fmt.Errorf("the hook configuration changed after trace run: %s", got) } + if hc.UserFile() == hc.Path { + return nil + } + if got, err = readHookFile(s, hc.Path); err != nil { + return err + } + if bytes.Contains(got, []byte(hc.ChainloopMarker)) { + return fmt.Errorf("trace run left Chainloop hooks in %s", hc.Path) + } return nil } diff --git a/app/cli/internal/trace/e2e/claude.go b/app/cli/internal/trace/e2e/claude.go index dfc5d5ff4..1d1cbc8b1 100644 --- a/app/cli/internal/trace/e2e/claude.go +++ b/app/cli/internal/trace/e2e/claude.go @@ -217,6 +217,11 @@ func (c *Claude) DisableSessionStart(repo string) error { return os.WriteFile(path, data, 0o600) } +// SkillPath implements Harness. +func (c *Claude) SkillPath(name string) string { + return ".claude/skills/" + name + "/SKILL.md" +} + // claudeStreamEvent is the part of a stream-json line that the suite reads. A // hook event has type "system" and subtype "hook_response". Its stdout is // the response that the hook printed. diff --git a/app/cli/internal/trace/e2e/config.yaml b/app/cli/internal/trace/e2e/config.yaml index dc8ed0887..937becf5c 100644 --- a/app/cli/internal/trace/e2e/config.yaml +++ b/app/cli/internal/trace/e2e/config.yaml @@ -9,6 +9,11 @@ harnesses: version: "2.1.301" models: - haiku + # The OpenRouter models need OPENROUTER_API_KEY in the environment. + OpenCode 2: + version: "2.0.22" + models: + - openrouter/anthropic/claude-haiku-5.5 # Runs and pass threshold of a cell whose feature depends on the model # following an instruction (R-006, D-004). A cell of any other feature runs diff --git a/app/cli/internal/trace/e2e/e2e_test.go b/app/cli/internal/trace/e2e/e2e_test.go index abba8b5d9..9c1a4239a 100644 --- a/app/cli/internal/trace/e2e/e2e_test.go +++ b/app/cli/internal/trace/e2e/e2e_test.go @@ -18,9 +18,11 @@ package e2e import ( + "maps" "os" "os/exec" "path/filepath" + "slices" "strings" "testing" @@ -74,7 +76,7 @@ func TestTraceMatrix(t *testing.T) { runner := &Runner{ Expected: exp, Config: cfg, - Harnesses: []Harness{&Claude{}}, + Harnesses: configuredHarnesses(t, cfg), Filter: Filter{Harnesses: envList(envHarness), Models: envList(envModel), Features: envList(envFeature)}, ChainloopBin: bin, CLIVersion: version, @@ -82,8 +84,12 @@ func TestTraceMatrix(t *testing.T) { Logf: t.Logf, } + // An agent that does not work in its workspace commits in the checkout + // that runs the suite. The suite cannot stop that, but it must not pass. + head := sourceHead(t) res, err := runner.Run(ctx) require.NoError(t, err) + require.Equal(t, head, sourceHead(t), "an agent committed in the checkout that runs the suite, not in its workspace") resultsDir := os.Getenv(envResultsDir) if resultsDir == "" { @@ -119,6 +125,30 @@ func TestTraceMatrix(t *testing.T) { } } +// configuredHarnesses returns the adapters of the harnesses that the suite +// config lists, in the order of their names. +func configuredHarnesses(t *testing.T, cfg *Config) []Harness { + t.Helper() + + names := slices.Sorted(maps.Keys(cfg.Harnesses)) + out := make([]Harness, 0, len(names)) + for _, name := range names { + h, ok := harnesses[name] + require.True(t, ok, "the suite config lists harness %q, which has no adapter", name) + out = append(out, h) + } + return out +} + +// sourceHead returns the HEAD commit of the checkout that runs the suite. +func sourceHead(t *testing.T) string { + t.Helper() + + out, err := exec.Command("git", "rev-parse", "HEAD").Output() + require.NoError(t, err) + return strings.TrimSpace(string(out)) +} + // buildChainloop builds the chainloop CLI under test and returns its path and // its version. func buildChainloop(t *testing.T) (string, string) { diff --git a/app/cli/internal/trace/e2e/harness.go b/app/cli/internal/trace/e2e/harness.go index 5240b7047..90ce3772d 100644 --- a/app/cli/internal/trace/e2e/harness.go +++ b/app/cli/internal/trace/e2e/harness.go @@ -20,6 +20,13 @@ import "context" // HarnessClaudeCode is the column name of Claude Code in the feature table. const HarnessClaudeCode = "Claude Code" +// harnesses are the adapters of the suite, by column name. The suite runs the +// ones that config.yaml lists. +var harnesses = map[string]Harness{ + HarnessClaudeCode: &Claude{}, + "OpenCode 2": &OpenCode{Major: "2"}, +} + // RunRequest is one headless session of a harness. type RunRequest struct { // Dir is the repository the session runs in. @@ -38,19 +45,31 @@ type RunRequest struct { Images [][]byte } -// HookConfig describes the file where the harness reads its hooks. +// HookConfig describes the files where the harness reads its hooks. type HookConfig struct { - // Path is the file, relative to the repository root. + // Path is the file of the Chainloop hooks, relative to the repository + // root. Path string // ChainloopMarker is a text that only a Chainloop hook in the file holds. ChainloopMarker string - // UserConfig is the content of the file with one hook of the user and no - // Chainloop hook. + // UserPath is the file of the hook of the user, relative to the + // repository root. Empty means Path: the hooks share one file. + UserPath string + // UserConfig is the content of the user file with one hook of the user + // and no Chainloop hook. UserConfig string // UserMarker is a text that only the hook of the user holds. UserMarker string } +// UserFile returns the file of the hook of the user. +func (c HookConfig) UserFile() string { + if c.UserPath != "" { + return c.UserPath + } + return c.Path +} + // HookMessage is a message that a Trace hook returned to the harness. type HookMessage struct { // Event is the hook event, for example SessionStart. @@ -93,4 +112,7 @@ type Harness interface { // installed in the repository, so that a session starts with no // session-start hook. DisableSessionStart(repo string) error + // SkillPath returns the file of a project skill, relative to the + // repository root. + SkillPath(name string) string } diff --git a/app/cli/internal/trace/e2e/opencode.go b/app/cli/internal/trace/e2e/opencode.go new file mode 100644 index 000000000..f1750c80a --- /dev/null +++ b/app/cli/internal/trace/e2e/opencode.go @@ -0,0 +1,299 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" +) + +// openCodePluginPath is the plugin file where `trace init` installs the +// OpenCode hooks. +const openCodePluginPath = ".opencode/plugins/chainloop-trace.ts" + +// openCodeReminderMarker is a text that only the spec capture reminder holds, +// not the session-start instruction. The plugin posts both to the session as +// synthetic messages, and this text tells them apart. +const openCodeReminderMarker = "Capture only in these cases" + +// openCodeSessionStartHandler is the line of the plugin that runs the +// session-start hook. DisableSessionStart replaces it. +const openCodeSessionStartHandler = `if (type === "session.created") {` + +var openCodeVersionRe = regexp.MustCompile(`(\d+)\.\d+\.\d+`) + +// OpenCode is the harness adapter of OpenCode. One adapter serves one major +// version, which is one column of the feature table. +type OpenCode struct { + // Major is the major version of OpenCode that the column covers. + Major string + // Bin is the opencode executable. Empty means "opencode" from PATH. + Bin string +} + +func (o *OpenCode) bin() string { + if o.Bin != "" { + return o.Bin + } + return "opencode" +} + +// Column implements Harness. +func (o *OpenCode) Column() string { return "OpenCode " + o.Major } + +// TraceFlag implements Harness. +func (o *OpenCode) TraceFlag() string { return "--opencode" } + +// Version implements Harness. `opencode --version` prints "opencode v2.0.22". +// An installed major version that is not the one of the column is an error, +// because the cells would test the wrong column. +func (o *OpenCode) Version(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, o.bin(), "--version").Output() //nolint:gosec // the harness binary under test + if err != nil { + return "", fmt.Errorf("opencode --version: %w", err) + } + m := openCodeVersionRe.FindStringSubmatch(string(out)) + if m == nil { + return "", fmt.Errorf("opencode --version: no version in %q", strings.TrimSpace(string(out))) + } + if m[1] != o.Major { + return "", fmt.Errorf("installed opencode %s is not the major version %s of the column %q", m[0], o.Major, o.Column()) + } + return m[0], nil +} + +// Run implements Harness. It runs `opencode run` with the JSON output. Each +// session gets a private server (--standalone): the shared background server +// keeps the environment and the plugins of the directory that started it. +// --auto approves the tool calls, which a headless run cannot answer. +// +// The plugin posts the instruction and the reminder of Trace to the session +// as synthetic messages, which the run output does not show. Run exports the +// session after the run and reads them from the export. +func (o *OpenCode) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { + args := []string{"run", "--standalone", "--format", "json", "--auto", "--model", req.Model} + + // A pasted image goes in as an attached file, from outside the + // repository, so that the agent cannot commit it. + if len(req.Images) > 0 { + dir, err := os.MkdirTemp("", "chainloop-trace-e2e-") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + for i, img := range req.Images { + path := filepath.Join(dir, fmt.Sprintf("pasted-%d.png", i+1)) + if err := os.WriteFile(path, img, 0o600); err != nil { + return nil, err + } + args = append(args, "--file", path) + } + } + args = append(args, req.Prompt) + + name := o.bin() + if len(req.Wrapper) > 0 { + name = req.Wrapper[0] + args = append(append(append([]string{}, req.Wrapper[1:]...), o.bin()), args...) + } + + raw, err := o.command(ctx, req, name, args...) + if err != nil { + return &RunOutput{Raw: raw}, fmt.Errorf("opencode run: %w", err) + } + if err := openCodeError(raw); err != nil { + return &RunOutput{Raw: raw}, err + } + + sessionID, err := openCodeSessionID(raw) + if err != nil { + return &RunOutput{Raw: raw}, err + } + export, err := o.command(ctx, req, o.bin(), "session", "export", sessionID) + if err != nil { + return &RunOutput{Raw: raw}, fmt.Errorf("opencode session export: %w", err) + } + + // The export follows the run events in the harness output, on one line. + var line bytes.Buffer + if err := json.Compact(&line, export); err != nil { + return &RunOutput{Raw: raw}, fmt.Errorf("decode opencode export: %w", err) + } + raw = append(append(raw, line.Bytes()...), '\n') + + hooks, err := parseOpenCodeHookMessages(export) + if err != nil { + return &RunOutput{Raw: raw}, err + } + + return &RunOutput{Raw: raw, Hooks: hooks, HooksReported: true}, nil +} + +func (o *OpenCode) command(ctx context.Context, req RunRequest, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario // nosemgrep + cmd.Dir = req.Dir + // OpenCode takes its project directory from PWD, not from the working + // directory of the process. With the PWD of the suite, the agent works + // and pushes in the checkout that runs the suite. + cmd.Env = append(slices.Clone(req.Env), "PWD="+req.Dir) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return out, fmt.Errorf("%w: %s", err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +// openCodeEvent is the part of a JSON event of `opencode run` that the suite +// reads. +type openCodeEvent struct { + Type string `json:"type"` + SessionID string `json:"sessionID"` + Error *struct { + Type string `json:"type"` + Message string `json:"message"` + } `json:"error"` +} + +func openCodeEvents(raw []byte) ([]openCodeEvent, error) { + var out []openCodeEvent + scanner := bufio.NewScanner(bytes.NewReader(raw)) + scanner.Buffer(make([]byte, 0, 1024*1024), 64*1024*1024) + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + var ev openCodeEvent + if err := json.Unmarshal(line, &ev); err != nil { + return nil, fmt.Errorf("decode opencode event: %w", err) + } + out = append(out, ev) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read opencode output: %w", err) + } + return out, nil +} + +// openCodeError returns the error event of a run. `opencode run` exits with +// status 0 for some errors, for example a provider that does not accept +// the credentials. +func openCodeError(raw []byte) error { + events, err := openCodeEvents(raw) + if err != nil { + return err + } + for _, ev := range events { + if ev.Type == "error" && ev.Error != nil { + return fmt.Errorf("opencode run: %s: %s", ev.Error.Type, ev.Error.Message) + } + } + return nil +} + +// openCodeSessionID returns the session of a run, from its first event. +func openCodeSessionID(raw []byte) (string, error) { + events, err := openCodeEvents(raw) + if err != nil { + return "", err + } + for _, ev := range events { + if ev.SessionID != "" { + return ev.SessionID, nil + } + } + return "", errors.New("opencode run: no session in the output") +} + +// openCodeExport is the part of an OpenCode 2 session export that the suite +// reads. +type openCodeExport struct { + Messages []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"messages"` +} + +// parseOpenCodeHookMessages reads the synthetic messages of a session export. +// The plugin posts them for the model only, so each one is model context. +func parseOpenCodeHookMessages(export []byte) ([]HookMessage, error) { + var ex openCodeExport + if err := json.Unmarshal(export, &ex); err != nil { + return nil, fmt.Errorf("decode opencode export: %w", err) + } + + var out []HookMessage + for _, m := range ex.Messages { + if m.Type != "synthetic" || m.Text == "" { + continue + } + event := "SessionStart" + if strings.Contains(m.Text, openCodeReminderMarker) { + event = "UserPromptSubmit" + } + out = append(out, HookMessage{Event: event, ModelContext: m.Text}) + } + return out, nil +} + +// HookConfig implements Harness. The Chainloop plugin is a file of its own, +// so the plugin of the user is in another file. +func (o *OpenCode) HookConfig() HookConfig { + return HookConfig{ + Path: openCodePluginPath, + ChainloopMarker: `"trace", "hook", "opencode"`, + UserPath: ".opencode/plugins/e2e-user.ts", + UserConfig: `// e2e-user-hook +export default { + id: "e2e-user", + async setup() {}, +} +`, + UserMarker: "e2e-user-hook", + } +} + +// DisableSessionStart implements Harness. It changes the session.created +// handler of the plugin, so that the session-start hook never runs. +func (o *OpenCode) DisableSessionStart(repo string) error { + path := filepath.Join(repo, openCodePluginPath) + data, err := os.ReadFile(path) + if err != nil { + return err + } + if n := bytes.Count(data, []byte(openCodeSessionStartHandler)); n != 1 { + return fmt.Errorf("%s has %d session.created handlers, want 1", openCodePluginPath, n) + } + data = bytes.Replace(data, []byte(openCodeSessionStartHandler), []byte(`if (type === "e2e.disabled") {`), 1) + return os.WriteFile(path, data, 0o600) //nolint:gosec // the plugin file of a throwaway workspace +} + +// SkillPath implements Harness. +func (o *OpenCode) SkillPath(name string) string { + return ".opencode/skills/" + name + "/SKILL.md" +} diff --git a/app/cli/internal/trace/e2e/opencode_test.go b/app/cli/internal/trace/e2e/opencode_test.go new file mode 100644 index 000000000..1a9d7d29e --- /dev/null +++ b/app/cli/internal/trace/e2e/opencode_test.go @@ -0,0 +1,145 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOpenCodeRunOutput(t *testing.T) { + cases := []struct { + name string + raw string + wantID string + wantIDErr bool + wantErr string + }{ + { + name: "session", + raw: `{"type":"step_start","sessionID":"ses_1","part":{}} +{"type":"text","sessionID":"ses_1","part":{"text":"done"}} +`, + wantID: "ses_1", + }, + { + name: "error event", + raw: `{"type":"step_start","sessionID":"ses_2","part":{}} +{"type":"error","sessionID":"ses_2","error":{"type":"provider.auth","message":"Token is expired."}} +`, + wantID: "ses_2", + wantErr: "provider.auth: Token is expired.", + }, + { + name: "no session", + raw: "\n", + wantIDErr: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := openCodeError([]byte(tc.raw)) + if tc.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tc.wantErr) + } else { + require.NoError(t, err) + } + + id, err := openCodeSessionID([]byte(tc.raw)) + if tc.wantIDErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantID, id) + }) + } +} + +func TestParseOpenCodeHookMessages(t *testing.T) { + cases := []struct { + name string + export string + want []HookMessage + }{ + { + name: "instruction and reminder", + export: `{"info":{"id":"ses_1"},"messages":[ +{"type":"synthetic","text":"Chainloop spec capture. Folder: /x\nCapture the specs."}, +{"type":"synthetic","text":"Chainloop spec capture. Folder: /x\n` + openCodeReminderMarker + `, one file per source."}, +{"type":"user","text":"do it"}, +{"type":"assistant","text":""}]}`, + want: []HookMessage{ + {Event: "SessionStart", ModelContext: "Chainloop spec capture. Folder: /x\nCapture the specs."}, + {Event: "UserPromptSubmit", ModelContext: "Chainloop spec capture. Folder: /x\n" + openCodeReminderMarker + ", one file per source."}, + }, + }, + { + name: "no synthetic message", + export: `{"info":{"id":"ses_1"},"messages":[{"type":"user","text":"do it"}]}`, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := parseOpenCodeHookMessages([]byte(tc.export)) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestOpenCodeDisableSessionStart(t *testing.T) { + cases := []struct { + name string + plugin string + want string + wantErr bool + }{ + { + name: "handler", + plugin: "async function sessionEvent() {\n " + openCodeSessionStartHandler + "\n await start()\n }\n}\n", + want: "async function sessionEvent() {\n if (type === \"e2e.disabled\") {\n await start()\n }\n}\n", + }, + { + name: "no handler", + plugin: "export default {}\n", + wantErr: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + repo := t.TempDir() + path := filepath.Join(repo, openCodePluginPath) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750)) + require.NoError(t, os.WriteFile(path, []byte(tc.plugin), 0o600)) + + err := (&OpenCode{Major: "2"}).DisableSessionStart(repo) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, tc.want, string(got)) + }) + } +} diff --git a/app/cli/internal/trace/e2e/runner_test.go b/app/cli/internal/trace/e2e/runner_test.go index cfc06b97b..e3a08b5d9 100644 --- a/app/cli/internal/trace/e2e/runner_test.go +++ b/app/cli/internal/trace/e2e/runner_test.go @@ -138,6 +138,7 @@ func (fakeHarness) Run(context.Context, RunRequest) (*RunOutput, error) { } func (fakeHarness) HookConfig() HookConfig { return HookConfig{} } func (fakeHarness) DisableSessionStart(string) error { return errors.New("not installed") } +func (fakeHarness) SkillPath(string) string { return "" } func TestRunnerPlansCells(t *testing.T) { exp, err := ParseExpected(strings.NewReader(`| Feature | What it means | Claude Code | Cursor | diff --git a/app/cli/internal/trace/e2e/scenarios.go b/app/cli/internal/trace/e2e/scenarios.go index 5c404783f..dce4ab2aa 100644 --- a/app/cli/internal/trace/e2e/scenarios.go +++ b/app/cli/internal/trace/e2e/scenarios.go @@ -29,6 +29,9 @@ type Scenario struct { Checkout string // Images are PNG images that the user pastes with the prompt. Images [][]byte + // Skills are the project skills of the repository, by name. The harness + // gives the path of each skill file. + Skills map[string]string // NoAgent runs no harness session. The checks run commands in the // workspace themselves. NoAgent bool @@ -56,6 +59,10 @@ const ( fakeAllowedSecret = "ghp_Zx7Lq2Wm9Rt4Yp6Vn3Bk8Hd1Jf5Gs0CaEu2o" //nolint:gosec // gitleaks:allow a fake token on a line with an allow marker ) +// allowMarker is the inline marker that keeps a secret on its line from +// redaction. +const allowMarker = "gitleaks:allow" + // The README of every scenario repository. const ( readmePath = "README.md" @@ -99,7 +106,7 @@ var scenarios = map[string]Scenario{ Name: scenarioEditAndCommit, Seed: map[string]string{ readmePath: seedReadme, - "fixture.txt": "token = " + fakeAllowedSecret + " # gitleaks:allow\n", + "fixture.txt": "token = " + fakeAllowedSecret + " # " + allowMarker + "\n", }, Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: 1. Read the file fixture.txt. @@ -119,9 +126,9 @@ The deploy token ` + fakeSecret + ` is for a later task: do not use it.`, }, scenarioSkill: { Name: scenarioSkill, - Seed: map[string]string{ - readmePath: seedReadme, - ".claude/skills/" + skillName + "/SKILL.md": `--- + Seed: map[string]string{readmePath: seedReadme}, + Skills: map[string]string{ + skillName: `--- name: ` + skillName + ` description: Use when asked to add the farewell file to the repository. --- diff --git a/app/cli/internal/trace/e2e/workspace.go b/app/cli/internal/trace/e2e/workspace.go index afb6a98ad..f41397b89 100644 --- a/app/cli/internal/trace/e2e/workspace.go +++ b/app/cli/internal/trace/e2e/workspace.go @@ -98,11 +98,13 @@ func NewWorkspace(ctx context.Context, base, chainloopBin string, h Harness, sc return nil, err } - seed := sc.Seed + seed := maps.Clone(sc.Seed) + for name, content := range sc.Skills { + seed[h.SkillPath(name)] = content + } if sc.SeedUserHooks { hc := h.HookConfig() - seed = maps.Clone(seed) - seed[hc.Path] = hc.UserConfig + seed[hc.UserFile()] = hc.UserConfig } for name, content := range seed { path := filepath.Join(repo, filepath.FromSlash(name)) From f19d02f3ce1c57564c3603dcfd1c4c16b55ec44c Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Mon, 19 Oct 2026 21:15:31 +0200 Subject: [PATCH 2/3] test(trace): make the e2e ticket and media prompts deterministic The ticket prompt asks for the file tool and a separate commit command: a commit in the shell command that writes the file does not carry the session. The media prompt tells the agent not to capture the image, which would replace its base64 in the transcript with a pointer. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: be5af72c-30e7-48f4-b46c-dc7a5a754c41 --- app/cli/internal/trace/e2e/scenarios.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/app/cli/internal/trace/e2e/scenarios.go b/app/cli/internal/trace/e2e/scenarios.go index dce4ab2aa..2f29b95d3 100644 --- a/app/cli/internal/trace/e2e/scenarios.go +++ b/app/cli/internal/trace/e2e/scenarios.go @@ -122,7 +122,9 @@ The deploy token ` + fakeSecret + ` is for a later task: do not use it.`, readmePath: seedReadme, ticketPath: "# " + ticketTitle + "\n\nCreate the file farewell.txt with the single line: goodbye\n", }, - Prompt: "This is an automated test. Implement the ticket in " + ticketPath + ". Then commit your change and push it with git.", + // The file tool keeps the change in the session: a commit in the same + // shell command that writes the file does not carry the session. + Prompt: "This is an automated test. Implement the ticket in " + ticketPath + ". Write the file with your file-writing tool. Then commit your change and push it with git, in a separate shell command.", }, scenarioSkill: { Name: scenarioSkill, @@ -201,13 +203,16 @@ Then run: git add farewell.txt && git commit -m "add farewell" && git push 2. Run this shell command: git add color.txt && git commit -m "add color" && git push`, }, // media has the agent read an image file. The image stays inline in the - // tool result of the transcript, and its base64 holds a fake secret. + // tool result of the transcript, and its base64 holds a fake secret. The + // prompt tells the agent not to capture the image: a captured image gets + // a pointer in the transcript, so the transcript holds no base64. scenarioMedia: { Name: scenarioMedia, Seed: map[string]string{readmePath: seedReadme, mediaFile: string(mediaPNG())}, Prompt: `This is an automated test. Do exactly these steps, in order, and nothing else: 1. Read the image ` + mediaFile + ` with your file-reading tool. 2. With your file-writing tool, create the file seen.txt with the single line: seen. -3. Run this shell command: git add seen.txt && git commit -m "add seen" && git push`, +3. Run this shell command: git add seen.txt && git commit -m "add seen" && git push +The image is test data, not a spec: do not capture it.`, }, } From b9267fbb3b9edc7e43eb626022bca56bf8276fd4 Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Mon, 19 Oct 2026 21:29:31 +0200 Subject: [PATCH 3/3] fix(trace): keep the e2e harness registry in the e2e build The harness map is used only by the e2e test, so a lint run without the e2e tag reported it as unused. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: be5af72c-30e7-48f4-b46c-dc7a5a754c41 --- app/cli/internal/trace/e2e/e2e_test.go | 7 +++++++ app/cli/internal/trace/e2e/harness.go | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/cli/internal/trace/e2e/e2e_test.go b/app/cli/internal/trace/e2e/e2e_test.go index 9c1a4239a..5043e92a2 100644 --- a/app/cli/internal/trace/e2e/e2e_test.go +++ b/app/cli/internal/trace/e2e/e2e_test.go @@ -125,6 +125,13 @@ func TestTraceMatrix(t *testing.T) { } } +// harnesses are the adapters of the suite, by column name. The suite runs the +// ones that config.yaml lists. +var harnesses = map[string]Harness{ + HarnessClaudeCode: &Claude{}, + "OpenCode 2": &OpenCode{Major: "2"}, +} + // configuredHarnesses returns the adapters of the harnesses that the suite // config lists, in the order of their names. func configuredHarnesses(t *testing.T, cfg *Config) []Harness { diff --git a/app/cli/internal/trace/e2e/harness.go b/app/cli/internal/trace/e2e/harness.go index 90ce3772d..ac88ceb3d 100644 --- a/app/cli/internal/trace/e2e/harness.go +++ b/app/cli/internal/trace/e2e/harness.go @@ -20,13 +20,6 @@ import "context" // HarnessClaudeCode is the column name of Claude Code in the feature table. const HarnessClaudeCode = "Claude Code" -// harnesses are the adapters of the suite, by column name. The suite runs the -// ones that config.yaml lists. -var harnesses = map[string]Harness{ - HarnessClaudeCode: &Claude{}, - "OpenCode 2": &OpenCode{Major: "2"}, -} - // RunRequest is one headless session of a harness. type RunRequest struct { // Dir is the repository the session runs in.