From f1cd1d79e4a13605fd401e4885d9f685a5b421ab Mon Sep 17 00:00:00 2001 From: "Jose I. Paris" Date: Tue, 20 Oct 2026 16:02:31 +0200 Subject: [PATCH] feat(trace): e2e adapter for Codex Add a Codex column to the Chainloop Trace end-to-end suite. Trace has no Codex provider yet, so each Codex cell is No and the suite skips it. Assisted-by: Claude Code Signed-off-by: Jose I. Paris Chainloop-Trace-Sessions: 552fa994-c621-4f9c-8150-21b0c9e51bb8 --- app/cli/internal/trace/README.md | 98 ++++++------ app/cli/internal/trace/e2e/codex.go | 195 +++++++++++++++++++++++ app/cli/internal/trace/e2e/codex_test.go | 74 +++++++++ app/cli/internal/trace/e2e/config.yaml | 6 + app/cli/internal/trace/e2e/e2e_test.go | 1 + 5 files changed, 326 insertions(+), 48 deletions(-) create mode 100644 app/cli/internal/trace/e2e/codex.go create mode 100644 app/cli/internal/trace/e2e/codex_test.go diff --git a/app/cli/internal/trace/README.md b/app/cli/internal/trace/README.md index 61bae0344..9b4e3ebfa 100644 --- a/app/cli/internal/trace/README.md +++ b/app/cli/internal/trace/README.md @@ -10,15 +10,17 @@ Claude Code has the most complete support. Trace does not support Pi yet (spec issue-3520): `chainloop trace init --pi` and `trace run --pi` do not exist, so no Pi session is recorded. Each Pi cell is **No** for this reason, and the "Why not Yes" lists do not repeat it. When Pi support is added, set the Pi cells to **Unknown** and run the end-to-end suite to find the values. +Trace does not support Codex yet: `chainloop trace init --codex` and `trace run --codex` do not exist, so no Codex session is recorded. Each Codex cell is **No** for this reason, and the "Why not Yes" lists do not repeat it. When Codex support is added, set the Codex cells to **Unknown** and run the end-to-end suite to find the values. + ## Setup How Trace is added to a repository and to an agent. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | -|---------|---------------|-------------|--------|--------------|------------|---| -| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Unknown | Unknown | Yes | No | -| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Unknown | Unknown | Yes | No | -| More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | No | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Hook installation and deletion | `chainloop trace init` installs the hooks of the selected agent. A second init changes nothing. `chainloop trace uninstall` deletes only the Chainloop hooks and keeps the other hooks of the agent. | Yes | Unknown | Unknown | Yes | No | No | +| Trace run wrapper | `chainloop trace run -- ` gives the agent temporary hooks for one session. When the command exits, the configuration of the agent is the same as before. | Yes | Unknown | Unknown | Yes | No | No | +| More than one agent in one repository | Two agents initialized in the same repository record their sessions separately, and each commit names the sessions of the agent that made it. | Unknown | Unknown | Unknown | Unknown | No | No | Why not Yes: @@ -29,13 +31,13 @@ Why not Yes: What the session evidence holds about the conversation. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | -|---------|---------------|-------------|--------|--------------|------------|---| -| Transcript | The evidence holds the conversation (`raw_session`): the messages and the tool calls of the agent. | Yes | Yes | Partial | Partial | No | -| Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | No | -| Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | No | -| Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | No | -| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Yes | No | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Transcript | The evidence holds the conversation (`raw_session`): the messages and the tool calls of the agent. | Yes | Yes | Partial | Partial | No | No | +| Token usage and cost | The evidence holds the tokens that the session used and their cost in USD. | Yes | No | Yes | Yes | No | No | +| Model detection | The evidence names the primary model and the other models that the session used. | Yes | Partial | Yes | Yes | No | No | +| Subagents | The evidence holds the conversations of the subagents, and their token usage, as part of the session. | Yes | No | No | No | No | No | +| Missed session start | A session whose session-start hook did not fire is still recorded, from its first tool hook. | Yes | Unknown | Unknown | Yes | No | No | Why not Yes: @@ -50,16 +52,16 @@ Why not Yes: Which lines of a commit the AI wrote. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | -|---------|---------------|-------------|--------|--------------|------------|---| -| Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | -| Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | -| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence does not report them yet (spec issue-3639). | Yes | Partial | Yes | Yes | No | -| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | -| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the sum of the removals of the session's commits (spec issue-3639). | Partial | Partial | Partial | Partial | No | -| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | -| Line ranges | The line ranges of each file are positions in the file at the commit that the push sends. They cover the lines of the session that are still there, and each range says if its lines are AI or human. The AI ranges hold as many lines as the net AI lines added, and the human ranges as many lines as the net human lines added. A file that the pushed commit no longer holds has no ranges. A range with no attribution is an AI range at edit time. Older CLIs write only these ranges. This CLI writes them for a file that it cannot blame at the pushed commit, and for a file with edits from before line fingerprints (spec issue-3639, line ranges). | Partial | Partial | Partial | Partial | No | -| Renames and rewrites | The AI/human split survives a file rename, an amend, a rebase, a squash, and a cherry-pick. The lines match the trace state by content, and blame follows a renamed file to its lines from before the rename (spec issue-3639). | Partial | Partial | Partial | Partial | No | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Line attribution | Each changed line of a commit is attributed to the AI or to a human, from a snapshot of each file before and after the agent edits it. An added line is AI when its content matches a line that an agent edit of the session wrote. A line that a human changed is human, also when a later agent edit keeps it. A whitespace-only change keeps the owner of the line. A removed line is AI only when an agent edit removed it. In a commit of more than one session, a line that only another session wrote or removed counts for neither side in this session's evidence (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | +| Partial commits | A commit that holds only part of the AI lines of a file credits the session with those lines. The other AI lines stay pending, and the next commit with them is credited to the session too (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | +| Generated line counts | The trace state keeps the lines that each agent edit added and removed, also the lines that never reach a commit. The evidence does not report them yet (spec issue-3639). | Yes | Partial | Yes | Yes | No | No | +| Shell changes attribution | Changes that the agent makes with shell commands (for example `sed`) are attributed to the AI too, not only changes made with the file tools. | Partial | Partial | Partial | Partial | No | No | +| Net contribution | The added lines of a session are the lines that its commits added and that are still in the commit that the push sends. A line that a later commit replaced does not count, and each line counts one time. The removed lines are the sum of the removals of the session's commits (spec issue-3639). | Partial | Partial | Partial | Partial | No | No | +| Session updates | A later commit that replaces lines of a session sends the evidence of the session again at the next push, with the new numbers. A commit in no session that replaces lines of a session joins it: the commit list of the session holds the commit, and its replacing lines count as human (spec issue-3639). | Partial | Partial | Partial | Partial | No | No | +| Line ranges | The line ranges of each file are positions in the file at the commit that the push sends. They cover the lines of the session that are still there, and each range says if its lines are AI or human. The AI ranges hold as many lines as the net AI lines added, and the human ranges as many lines as the net human lines added. A file that the pushed commit no longer holds has no ranges. A range with no attribution is an AI range at edit time. Older CLIs write only these ranges. This CLI writes them for a file that it cannot blame at the pushed commit, and for a file with edits from before line fingerprints (spec issue-3639, line ranges). | Partial | Partial | Partial | Partial | No | No | +| Renames and rewrites | The AI/human split survives a file rename, an amend, a rebase, a squash, and a cherry-pick. The lines match the trace state by content, and blame follows a renamed file to its lines from before the rename (spec issue-3639). | Partial | Partial | Partial | Partial | No | No | Why not Yes: @@ -74,11 +76,11 @@ Why not Yes: What the user sees in the agent. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | -|---------|---------------|-------------|--------|--------------|------------|---| -| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | No | -| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | No | -| Credentials warning | At session start, in control-plane mode, the user is told when the control plane does not accept the CLI credentials (for example, an expired token), so the push will not upload the session. The hook uses the same check as `chainloop trace init`. An unreachable control plane, or a check that takes more than 2 seconds, shows no warning. | Yes | Partial | Partial | Partial | No | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Welcome message | At session start, the user sees a message that tells that Chainloop records the session, and where the evidence goes: the Chainloop project, or the on-disk path in local mode (spec issue-3582). | Yes | No | No | No | No | No | +| Session link after push | After a `git push` that the agent runs, the user sees a link to the session in Chainloop. | Yes | No | No | No | No | No | +| Credentials warning | At session start, in control-plane mode, the user is told when the control plane does not accept the CLI credentials (for example, an expired token), so the push will not upload the session. The hook uses the same check as `chainloop trace init`. An unreachable control plane, or a check that takes more than 2 seconds, shows no warning. | Yes | Partial | Partial | Partial | No | No | Why not Yes: @@ -91,15 +93,15 @@ Why not Yes: What the session was asked to build, and which skills it used. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | -|---------|---------------|-------------|--------|--------------|------------|---| -| Spec capture instruction | At session start, the agent is told to capture the specs of the session (tickets, documents, plans, images) in the spec folder. A plan is captured only when it comes from an external source or from the plan mode of the agent: an agreement in the chat is not a plan. A pasted image is not captured, also when the agent has a file path for it. The agent tells the user in one line when it captures a file. | Yes | Yes | Yes | Yes | No | -| Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | No | -| Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | No | -| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Unknown | Unknown | Yes | No | -| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | -| Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | No | -| Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | No | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Spec capture instruction | At session start, the agent is told to capture the specs of the session (tickets, documents, plans, images) in the spec folder. A plan is captured only when it comes from an external source or from the plan mode of the agent: an agreement in the chat is not a plan. A pasted image is not captured, also when the agent has a file path for it. The agent tells the user in one line when it captures a file. | Yes | Yes | Yes | Yes | No | No | +| Spec capture reminder | At each user prompt, the agent is reminded to capture new or changed specs. | Yes | No | Yes | Yes | No | No | +| Local spec sources read at push | A capture whose source is a local file holds only a header and a placeholder. Each push reads the file and records its current content, so the evidence follows the edits of the file. Only a regular text file of up to 1 MiB is read. Otherwise the push keeps the body that the agent wrote, or drops the capture with a warning when there is no body (spec issue-3561). | Yes | Yes | Yes | Yes | No | No | +| Spec source role, title, and description | Each captured spec source holds the role, the title, and the description that the agent wrote in its header (spec issue-3531). | Yes | Unknown | Unknown | Yes | No | No | +| Skills tracking | The evidence lists the skills that the session used, with a copy of each skill as it ran. | Yes | No | Partial | Partial | No | No | +| Spec source pointers | Exact copies of spec sources in the transcript (full file reads and file writes) are replaced with a pointer to the spec material, so the evidence does not hold the same content two or three times (spec issue-3556). This includes the copies of pasted images and spec sources in the `api-request-blob` records of a Claude Code transcript (issue 3654). | Yes | No | No | No | No | No | +| Pasted image pointers | Each distinct image that the user pasted into the session, including images in prompts sent while the agent was busy, is stored once as an image spec entry, from the data in the transcript, with the title "Pasted image N" and no role. A repeated paste of the same image adds no entry. The inline image in the transcript is replaced with a pointer to that material. An image whose data cannot be decoded, or that the push could not store, stays inline, and the push continues. When the agent captured the same image, its entry is kept and no second entry is added (spec issue-3569). | Yes | No | No | No | No | No | Why not Yes: @@ -114,11 +116,11 @@ Why not Yes: How the evidence is protected before it leaves the machine. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | -|---------|---------------|-------------|--------|--------------|------------|---| -| Secret redaction | Secrets in the session evidence and in the spec materials are replaced with a placeholder before upload. | Yes | Yes | Yes | Yes | No | -| Media skipped by redaction | The secret redaction does not scan base64 media (images, PDFs) in the transcript, so it is faster and does not break the media. | Yes | Unknown | Not applicable | Not applicable | No | -| Allow markers in redaction | A secret on a line that carries an inline `gitleaks:allow` or `betterleaks:allow` marker, such as a test fixture the agent read, is not redacted. The marker covers only its own line. | Yes | Yes | Yes | Yes | No | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Secret redaction | Secrets in the session evidence and in the spec materials are replaced with a placeholder before upload. | Yes | Yes | Yes | Yes | No | No | +| Media skipped by redaction | The secret redaction does not scan base64 media (images, PDFs) in the transcript, so it is faster and does not break the media. | Yes | Unknown | Not applicable | Not applicable | No | No | +| Allow markers in redaction | A secret on a line that carries an inline `gitleaks:allow` or `betterleaks:allow` marker, such as a test fixture the agent read, is not redacted. The marker covers only its own line. | Yes | Yes | Yes | Yes | No | No | Why not Yes: @@ -129,12 +131,12 @@ Why not Yes: How the sessions are linked to the commits and attested. -| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | -|---------|---------------|-------------|--------|--------------|------------|---| -| Commit trailer | Each commit gets a `Chainloop-Trace-Sessions` trailer with the sessions that contributed to it. | Yes | Yes | Yes | Yes | No | -| Attestation at push | In the default control-plane storage mode, the `pre-push` hook attests the sessions of the pushed commits to the control plane. | Yes | Yes | Yes | Yes | No | -| Local storage at push | In a repository with `traceStorage: LOCAL` in `.chainloop.yml`, the `pre-push` hook runs the same assembly as an attestation. It writes the session evidence set to the `local-sessions` folder of the trace directory and sends no attestation. It needs no control plane, login, or network. The stored set is equal to the upload of a push, except for the signature. A `statement.json` file indexes the set: the in-toto statement that a push would sign, with the `chainloop.dev/attestation/v0.2` predicate. It has no envelope and no signature, so it is not an attestation. `trace init --local` sets up the mode. `trace run --local` stores one wrapped session the same way, with no `.chainloop.yml` setting. Redaction is on by default. Set `CHAINLOOP_TRACE_NO_REDACT` to turn it off, for a trusted local store only (spec issue-3582). | Yes | Yes | Yes | Yes | No | -| Shell commands in another checkout | A session can work in a checkout other than the one it started in through shell commands only, for example in a fresh clone. The checkout is found from the command text: the target of `cd` or `pushd`, the path of `git -C`, and the folder that `git clone` or `gh repo clone` creates. The git hooks are installed there and the session is recorded there, the same as for a file tool edit in that checkout. A clone gets the git hooks only after the command that created it, so a commit in that same command does not carry the session. For OpenCode, a plugin installed by an earlier version does not send the command text: install the hooks again to update it. | Yes | Partial | Yes | Yes | No | +| Feature | What it means | Claude Code | Cursor | OpenCode 1.x | OpenCode 2 | Pi | Codex | +|---------|---------------|-------------|--------|--------------|------------|---|---| +| Commit trailer | Each commit gets a `Chainloop-Trace-Sessions` trailer with the sessions that contributed to it. | Yes | Yes | Yes | Yes | No | No | +| Attestation at push | In the default control-plane storage mode, the `pre-push` hook attests the sessions of the pushed commits to the control plane. | Yes | Yes | Yes | Yes | No | No | +| Local storage at push | In a repository with `traceStorage: LOCAL` in `.chainloop.yml`, the `pre-push` hook runs the same assembly as an attestation. It writes the session evidence set to the `local-sessions` folder of the trace directory and sends no attestation. It needs no control plane, login, or network. The stored set is equal to the upload of a push, except for the signature. A `statement.json` file indexes the set: the in-toto statement that a push would sign, with the `chainloop.dev/attestation/v0.2` predicate. It has no envelope and no signature, so it is not an attestation. `trace init --local` sets up the mode. `trace run --local` stores one wrapped session the same way, with no `.chainloop.yml` setting. Redaction is on by default. Set `CHAINLOOP_TRACE_NO_REDACT` to turn it off, for a trusted local store only (spec issue-3582). | Yes | Yes | Yes | Yes | No | No | +| Shell commands in another checkout | A session can work in a checkout other than the one it started in through shell commands only, for example in a fresh clone. The checkout is found from the command text: the target of `cd` or `pushd`, the path of `git -C`, and the folder that `git clone` or `gh repo clone` creates. The git hooks are installed there and the session is recorded there, the same as for a file tool edit in that checkout. A clone gets the git hooks only after the command that created it, so a commit in that same command does not carry the session. For OpenCode, a plugin installed by an earlier version does not send the command text: install the hooks again to update it. | Yes | Partial | Yes | Yes | No | No | ### Switching storage modes @@ -162,7 +164,7 @@ All agents share the git hooks `commit-msg`, `post-commit` and `pre-push`. ## End-to-end tests -The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code, OpenCode 2, and Pi columns. The Pi cells are **No**, so the suite skips them until Pi support is added. OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace, because OpenCode takes its project folder from `PWD`. Pi runs with `pi -p --mode json --approve`, with its configuration and session folders next to the workspace. The OpenCode 2 and Pi models in `e2e/config.yaml` are OpenRouter models, so the run needs `OPENROUTER_API_KEY`. +The `e2e/` package checks the tables of this page against real agents (spec issue-3635). It runs each agent in headless mode with real models, in throwaway repositories with Trace in local storage mode. For now, it runs the Claude Code, OpenCode 2, Pi, and Codex columns. The Pi and Codex cells are **No**, so the suite skips them until Pi and Codex support is added. OpenCode runs on a private server (`opencode run --standalone`) with `PWD` set to the workspace, because OpenCode takes its project folder from `PWD`. Pi runs with `pi -p --mode json --approve`, with its configuration and session folders next to the workspace. Codex runs with `codex exec --json`, with its `CODEX_HOME` folder next to the workspace, no sandbox, and no hook trust prompt. The OpenCode 2, Pi, and Codex models in `e2e/config.yaml` are OpenRouter models, so the run needs `OPENROUTER_API_KEY`. A Codex model with no `openrouter/` prefix runs on the default Codex provider, which needs `CODEX_API_KEY`. A feature row maps to its checks by the feature name. When you rename a feature in a table, rename it in `e2e/checks.go` too. The suite tests a cell marked **Yes**, **Partial**, or **Unknown** only when the row has a registered check. It reports the other cells as follows: diff --git a/app/cli/internal/trace/e2e/codex.go b/app/cli/internal/trace/e2e/codex.go new file mode 100644 index 000000000..727f2da75 --- /dev/null +++ b/app/cli/internal/trace/e2e/codex.go @@ -0,0 +1,195 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" +) + +const ( + // codexHooksPath is the project file where Codex reads its hooks. Trace + // has no Codex provider yet, so no Chainloop hook is in it. + codexHooksPath = ".codex/hooks.json" + // codexOpenRouterPrefix selects the OpenRouter provider for a model of + // the suite config. + codexOpenRouterPrefix = "openrouter/" + // codexOpenRouterProvider defines the OpenRouter provider, which reads + // its key from OPENROUTER_API_KEY. + codexOpenRouterProvider = `model_providers.openrouter={name="OpenRouter",base_url="https://openrouter.ai/api/v1",env_key="OPENROUTER_API_KEY"}` +) + +var codexVersionRe = regexp.MustCompile(`\d+\.\d+\.\d+`) + +// Codex is the harness adapter of the OpenAI Codex CLI. +type Codex struct { + // Bin is the codex executable. Empty means "codex" from PATH. + Bin string +} + +func (c *Codex) bin() string { + if c.Bin != "" { + return c.Bin + } + return "codex" +} + +// Column implements Harness. +func (c *Codex) Column() string { return "Codex" } + +// TraceFlag implements Harness. +func (c *Codex) TraceFlag() string { return "--codex" } + +// Version implements Harness. +func (c *Codex) Version(ctx context.Context) (string, error) { + out, err := exec.CommandContext(ctx, c.bin(), "--version").Output() //nolint:gosec // the harness binary under test // nosemgrep + if err != nil { + return "", fmt.Errorf("codex --version: %w", err) + } + return parseCodexVersion(out) +} + +// parseCodexVersion reads the output of `codex --version`, for example +// "codex-cli 0.163.0". +func parseCodexVersion(out []byte) (string, error) { + v := codexVersionRe.FindString(string(out)) + if v == "" { + return "", fmt.Errorf("codex --version: no version in %q", strings.TrimSpace(string(out))) + } + return v, nil +} + +// codexModelArgs returns the arguments that select the model. A model with +// the "openrouter/" prefix runs on OpenRouter. Any other model runs on the +// default provider of Codex, which reads its key from CODEX_API_KEY. +func codexModelArgs(model string) []string { + name, ok := strings.CutPrefix(model, codexOpenRouterPrefix) + if !ok { + return []string{flagModel, model} + } + return []string{ + flagModel, name, + "-c", "model_provider=openrouter", + "-c", codexOpenRouterProvider, + } +} + +// Run implements Harness. It runs `codex exec` with the JSON output. +// +// The configuration folder of Codex (CODEX_HOME) is next to the workspace, so +// the configuration, the hooks, the skills and the sessions of the user who +// runs the suite do not load or mix with the sessions of the suite. The +// sandbox and the approvals are off, because the workspace is a throwaway +// repository. The project hooks run with no persisted trust. +// --ignore-user-config is not set, because it also turns off the project +// hooks. +// +// Trace has no Codex provider yet, so no hook message is reported. +func (c *Codex) Run(ctx context.Context, req RunRequest) (*RunOutput, error) { + base := filepath.Dir(req.Dir) + codexHome := filepath.Join(base, "codex-home") + if err := os.MkdirAll(codexHome, 0o700); err != nil { + return nil, err + } + + args := []string{ + "exec", "--json", + "--dangerously-bypass-approvals-and-sandbox", + "--dangerously-bypass-hook-trust", + "-C", req.Dir, + } + args = append(args, codexModelArgs(req.Model)...) + + // A pasted image goes in as an attached file, from outside the + // repository, so that the agent cannot commit it. + if len(req.Images) > 0 { + dir, err := os.MkdirTemp("", "chainloop-trace-e2e-") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + for i, img := range req.Images { + path := filepath.Join(dir, fmt.Sprintf("pasted-%d.png", i+1)) + if err := os.WriteFile(path, img, 0o600); err != nil { + return nil, err + } + args = append(args, "--image", path) + } + } + // The prompt follows "--", so that the list of images ends before it. + args = append(args, "--", req.Prompt) + + name := c.bin() + if len(req.Wrapper) > 0 { + name = req.Wrapper[0] + args = append(append(append([]string{}, req.Wrapper[1:]...), c.bin()), args...) + } + + cmd := exec.CommandContext(ctx, name, args...) //nolint:gosec // the harness binary under test, with the prompt of a scenario // nosemgrep + cmd.Dir = req.Dir + cmd.Env = append(slices.Clone(req.Env), "CODEX_HOME="+codexHome) + var stderr bytes.Buffer + cmd.Stderr = &stderr + + raw, err := cmd.Output() + if err != nil { + return &RunOutput{Raw: raw}, fmt.Errorf("codex exec: %w: %s", err, strings.TrimSpace(stderr.String())) + } + + return &RunOutput{Raw: raw}, nil +} + +// HookConfig implements Harness. The hooks of the user and the Chainloop +// hooks share one file. +func (c *Codex) HookConfig() HookConfig { + return HookConfig{ + Path: codexHooksPath, + ChainloopMarker: "chainloop trace hook codex", + UserConfig: `{ + "hooks": { + "Stop": [ + {"hooks": [{"type": "command", "command": "echo e2e-user-hook"}]} + ] + } +} +`, + UserMarker: userHookMarker, + } +} + +// errCodexSessionStart is the error of DisableSessionStart until Trace has a +// Codex provider. +var errCodexSessionStart = errors.New("codex: the suite cannot remove the session-start hook of Trace, because Trace has no Codex provider yet") + +// DisableSessionStart implements Harness. Trace installs no Codex hooks yet, +// so there is no hook to remove. +func (c *Codex) DisableSessionStart(string) error { + return errCodexSessionStart +} + +// SkillPath implements Harness. Codex finds project skills in +// .agents/skills. +func (c *Codex) SkillPath(name string) string { + return ".agents/skills/" + name + "/SKILL.md" +} diff --git a/app/cli/internal/trace/e2e/codex_test.go b/app/cli/internal/trace/e2e/codex_test.go new file mode 100644 index 000000000..f0e79e989 --- /dev/null +++ b/app/cli/internal/trace/e2e/codex_test.go @@ -0,0 +1,74 @@ +// +// Copyright 2026 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseCodexVersion(t *testing.T) { + cases := []struct { + name string + out string + want string + wantErr bool + }{ + {name: "release", out: "codex-cli 0.163.0\n", want: "0.163.0"}, + {name: "no version", out: "codex-cli\n", wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := parseCodexVersion([]byte(tc.out)) + if tc.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestCodexModelArgs(t *testing.T) { + cases := []struct { + name string + model string + want []string + }{ + { + name: "default provider", + model: "gpt-5.5-mini", + want: []string{flagModel, "gpt-5.5-mini"}, + }, + { + name: "openrouter", + model: "openrouter/openai/gpt-5.5-mini", + want: []string{ + flagModel, "openai/gpt-5.5-mini", + "-c", "model_provider=openrouter", + "-c", codexOpenRouterProvider, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, codexModelArgs(tc.model)) + }) + } +} diff --git a/app/cli/internal/trace/e2e/config.yaml b/app/cli/internal/trace/e2e/config.yaml index ebf721981..86171d5ca 100644 --- a/app/cli/internal/trace/e2e/config.yaml +++ b/app/cli/internal/trace/e2e/config.yaml @@ -18,6 +18,12 @@ harnesses: version: "1.3.0" models: - openrouter/anthropic/claude-haiku-5.5 + # A model with the openrouter/ prefix runs on OpenRouter. Any other model + # runs on the default Codex provider and needs CODEX_API_KEY. + Codex: + version: "0.163.0" + models: + - openrouter/openai/gpt-5-mini # Runs and pass threshold of a cell whose feature depends on the model # following an instruction (R-006, D-004). A cell of any other feature runs diff --git a/app/cli/internal/trace/e2e/e2e_test.go b/app/cli/internal/trace/e2e/e2e_test.go index 599e5d6db..7d57caa48 100644 --- a/app/cli/internal/trace/e2e/e2e_test.go +++ b/app/cli/internal/trace/e2e/e2e_test.go @@ -131,6 +131,7 @@ var harnesses = map[string]Harness{ HarnessClaudeCode: &Claude{}, "OpenCode 2": &OpenCode{Major: "2"}, "Pi": &Pi{}, + "Codex": &Codex{}, } // configuredHarnesses returns the adapters of the harnesses that the suite