diff --git a/.gitignore b/.gitignore index 8d042e0..7336555 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ logs/ # Experiments (generated at runtime) experiments/ +# Benchmark task definitions downloaded by sforge fetch-tasks +/tasks/ + # IDE .idea/ .vscode/ diff --git a/README.md b/README.md index 6bcc671..aef1459 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,7 @@ SFORGE_AGENT_API_KEY="sk-xxx" \ **Step-by-step examples:** - [Single task on local Docker](examples/single-task-docker/) — run one task end-to-end with Docker - [All tasks on Kubernetes](examples/all-tasks-k8s/) — run the full suite on a K8s cluster with the **official leaderboard setting** +- [E2B backend](https://bytedance-seed.github.io/EdgeBench/en/configuration/container-backends#e2b-backend) — run Work and Judge environments on E2B without operating a cluster > [!IMPORTANT] > - **Official setting** — leaderboard numbers use the [official experiment YAMLs](examples/all-tasks-k8s/) unchanged, including the time budget, stop hook, auto-eval, submission cooldowns, and hardware resource limits. diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 3272ff8..8c2b0e6 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -36,6 +36,7 @@ export default defineConfig({ text: 'Examples', items: [ { text: 'Single Task (Docker)', link: '/en/examples/single-task-docker' }, + { text: 'Single Task (E2B)', link: '/en/examples/single-task-e2b' }, { text: 'All Tasks (Kubernetes)', link: '/en/examples/all-tasks-k8s' }, ], }, @@ -105,6 +106,7 @@ export default defineConfig({ text: '示例', items: [ { text: '单任务运行 (Docker)', link: '/zh/examples/single-task-docker' }, + { text: '单任务运行 (E2B)', link: '/zh/examples/single-task-e2b' }, { text: '全部任务 (Kubernetes)', link: '/zh/examples/all-tasks-k8s' }, ], }, diff --git a/docs/en/configuration/container-backends.md b/docs/en/configuration/container-backends.md index 8476fe5..50933a6 100644 --- a/docs/en/configuration/container-backends.md +++ b/docs/en/configuration/container-backends.md @@ -4,7 +4,9 @@ title: Container Backends # Container Backends -SForge uses a container backend to create work containers and judge containers. The default backend is local Docker. Switch to the `k8s` backend when you want SForge workloads to run in a Kubernetes cluster. +SForge uses a container backend to create work and judge environments. The +default backend is local Docker. Use `k8s` for an existing Kubernetes cluster, +or `e2b` for E2B-hosted environments. ## Choosing a Backend @@ -12,8 +14,11 @@ SForge uses a container backend to create work containers and judge containers. |---------|----------|-------------------| | `docker` | Local development, task debugging, single-machine evaluation, small experiments | A working local Docker daemon | | `k8s` | Shared clusters, high-concurrency batch evaluation, running Work/Judge pods in a cluster | `kubectl` access, images pullable by the cluster, and a Judge URL reachable from pods | +| `e2b` | Remote Work/Judge Sandboxes without operating a cluster | E2B API access, E2B Templates, and a routable Judge Server; see [E2B Backend](#e2b-backend) | If you are trying a task locally, use the default Docker backend first. The Kubernetes backend is intended for team environments that already have a cluster and container registry. +The E2B backend has a one-time Template preparation step before a new or +changed task image can run. ::: warning Docker backend does not scale to large batches Each task runs a work container plus ephemeral judge containers, each with its own CPU/memory limits. Running many tasks concurrently on one host (roughly **20+**) causes severe resource contention even on a high-end server. For large batch runs, use the `k8s` backend. @@ -29,15 +34,13 @@ sforge run \ --task ad_placement_optimization \ --agent claude-code \ --backend k8s \ - --judge-url http://10.0.0.12:8080 --backend k8s + --judge-url http://10.0.0.12:8080 # Override through environment variables export SFORGE_BACKEND=k8s sforge run \ --task ad_placement_optimization \ - --agent claude-code \ - --backend k8s \ - --judge-url http://10.0.0.12:8080 + --agent claude-code ``` Experiment YAML can also set the default backend: @@ -62,9 +65,7 @@ sforge serve export SFORGE_AGENT_API_KEY="sk-..." sforge run \ --task ad_placement_optimization \ - --agent claude-code \ - --backend k8s \ - --judge-url http://10.0.0.12:8080 + --agent claude-code ``` The Docker backend: @@ -222,13 +223,113 @@ Notes: - Your kubeconfig must be allowed to create and delete `NetworkPolicy` resources. - DNS, egress gateways, and policy implementations differ between clusters, so validate isolation with a small task first. +## E2B Backend + +The E2B backend runs Work, Judge, and Game environments as E2B Sandboxes. +The Judge Server itself runs on a host you control, exactly as with the other +backends: start `sforge serve` on a machine the Sandboxes can reach and pass +its address via `--judge-url`. + +### Docker Images and E2B Templates + +A Docker image and an E2B Template are different runtime objects: E2B +Sandboxes start from Templates, not from Docker image references. Official +E2B Templates for the published EdgeBench tasks are pre-built and published +by the SForge maintainers, so no template preparation step is required. +Set `SFORGE_E2B_TEMPLATE_NAMESPACE` to the namespace that publishes the +Templates. `sforge run --backend e2b` then derives each task image's public +Template reference automatically (`edgebench.work.foo_bar:abc123` -> +`edgebench/edgebench-work-foo-bar:abc123`). Set `SFORGE_E2B_TEMPLATE_MAP` +only to override this mapping, for example to run self-built Templates for a +modified task. + +### Run a Task + +Start the Judge Server on a host with a publicly routable address (agents +inside E2B Sandboxes submit to it over the internet): + +```bash +uv sync --extra e2b +export E2B_API_KEY=... +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench +export SFORGE_ADMIN_SECRET=... # same value on the serve and run hosts + +sforge serve --host 0.0.0.0 --port 8080 +``` + +Then run the agent, pointing `--judge-url` at that host: + +```bash +export E2B_API_KEY=... +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench +export SFORGE_ADMIN_SECRET=... # same value as on the serve host +export SFORGE_AGENT_API_KEY=... +export SFORGE_AGENT_API_BASE_URL=... +export SFORGE_AGENT_MODEL=... + +sforge run \ + --backend e2b \ + --task ad_placement_optimization \ + --agent claude-code \ + --judge-url http://YOUR_HOST:8080 +``` + +The Judge Server needs `E2B_API_KEY` too: registrations from an e2b run make +it create Judge and Game Sandboxes on E2B (mirroring how a k8s run makes it +schedule judge pods). Work Sandboxes submit to the Judge Server's public URL; +the Judge Server reaches Game Sandboxes through the E2B service gateway. + +Each E2B evaluation is scoped to one task. Supplying multiple task IDs only +batch-schedules several independent evaluations; it does not combine tasks into +one Work or Judge environment. The validation described here invoked one task +per `sforge run` command. + +### Plan Limits and Run Sizing + +E2B plan limits are part of the runtime contract: + +- **Resources**: Template CPU and memory are fixed at build time; over-limit + requests fail at Template creation. Rebuild with supported values, since + `--work-*` and `--judge-*` flags cannot resize an existing Template. +- **Sandbox lifetime**: Keep the agent timeout well below the team's maximum + Sandbox lifetime, which must also cover agent install, archive extraction, + Judge evaluation, result collection, and cleanup. At cleanup SForge caps the + remaining lifetime at 60 seconds and deletes the Sandbox, regardless of the + configured TTL. +- **Evaluator timeout**: Judge Sandboxes share the same limit, so a task whose + `judge.eval_timeout` exceeds it may not finish. Prefer a plan with a longer + lifetime over lowering the timeout. +- **Concurrency**: Tasks run fully in parallel. Budget Work, Judge, and Game + Sandboxes for all selected tasks, plus overlap during evaluation and cleanup. +- **Final score**: As on other backends, the score is the best completed agent + or auto-eval submission. `final_archive.tar.gz` is a recovery snapshot, not + an implicit submission at timeout. For short runs, allow at least one + auto-eval to finish, or have the agent call `sforge-submit` before the end. + +### Required User Input + +| Input | When required | +| --- | --- | +| `E2B_API_KEY` | E2B runs, on both the `sforge run` host and the Judge Server host | +| A publicly routable `--judge-url` | All E2B runs | +| Agent API key, base URL, and model | Agent runs | + +Host `HTTP_PROXY` and `HTTPS_PROXY` variables are not copied into E2B Work, +Judge, or Game Sandboxes. A remote-reachable proxy is forwarded only when the +user explicitly configures `SFORGE_HTTP_PROXY` or `SFORGE_HTTPS_PROXY`; isolated +tasks remove proxy variables entirely. See [Network Isolation](/en/features/network-isolation). + ## Troubleshooting -| Symptom | Likely cause | Fix | -|---------|--------------|-----| -| `kubectl cluster-info failed` | Wrong kubeconfig, unreachable cluster, or wrong namespace context | Check `kubectl -n cluster-info`; set `SFORGE_K8S_KUBECONFIG` and `SFORGE_K8S_NAMESPACE` if needed | -| Pod never becomes Running | Image pull failure, scheduling failure, or insufficient resources | Run `kubectl -n describe pod ` and check events | -| Pod cannot pull image | Image was not pushed to a registry reachable by the cluster, or registry credentials are missing | Run `sforge push`; verify `SFORGE_K8S_IMAGE_REGISTRY`; configure image pull secrets for the namespace | -| Work pod cannot submit evaluations | `--judge-url` is not reachable from pods, or the Judge server is not listening externally | Start `sforge serve` with `--host 0.0.0.0` and pass a pod-reachable IP/Service URL with `--judge-url` | -| `--disable-internet` has no effect | CNI does not enforce NetworkPolicy, or permissions are insufficient | Confirm NetworkPolicy support and verify the current identity can create NetworkPolicies | -| Pod stays Pending after setting a node selector | No nodes match the selector | Check `SFORGE_K8S_NODE_SELECTOR` and node labels | +| Backend | Symptom | Likely cause | Fix | +| --- | --- | --- | --- | +| Kubernetes | `kubectl cluster-info failed` | Wrong kubeconfig, unreachable cluster, or wrong namespace context | Check `kubectl -n cluster-info`; set `SFORGE_K8S_KUBECONFIG` and `SFORGE_K8S_NAMESPACE` if needed | +| Kubernetes | Pod never becomes Running | Image pull failure, scheduling failure, or insufficient resources | Run `kubectl -n describe pod ` and check events | +| Kubernetes | Pod cannot pull an image | The image is unavailable to cluster nodes, or registry credentials are missing | Run `sforge push`; verify `SFORGE_K8S_IMAGE_REGISTRY`; configure image pull secrets for the namespace | +| Kubernetes | Work cannot reach Judge | The Judge URL is not pod-reachable, or the server is not listening externally | Start `sforge serve` with `--host 0.0.0.0` and pass a pod-reachable IP or Service URL | +| Kubernetes | `--disable-internet` has no effect | CNI does not enforce NetworkPolicy, or permissions are insufficient | Confirm NetworkPolicy support and permissions | +| Kubernetes | Pod remains Pending with a node selector | No nodes match the selector | Check `SFORGE_K8S_NODE_SELECTOR` and node labels | +| E2B | Template not found | The Template namespace is missing or incorrect, the task image version has no published official Template, or the task uses a modified image | Check `SFORGE_E2B_TEMPLATE_NAMESPACE` and the task image version, or point `SFORGE_E2B_TEMPLATE_MAP` at a self-built Template | +| E2B | Agent-side Judge requests intermittently time out | The Work Sandbox could not reach the Judge Server over the internet | Check that the Judge host and port are publicly reachable, then retry the agent submission | +| E2B | A run ends at the configured task timeout with `Sandbox not found` | The team-level maximum Sandbox lifetime was reached before final extraction and cleanup | Shorten the agent timeout, reduce multi-wave batch duration, or use a plan with a longer Sandbox lifetime | +| E2B | A completed run retains a Sandbox for more than 60 seconds | Both the provider-side termination deadline and the immediate deletion request failed | Inspect the lifecycle cleanup error in the run or Judge Server log | diff --git a/docs/en/configuration/environment-variables.md b/docs/en/configuration/environment-variables.md index c4db0d1..135d009 100644 --- a/docs/en/configuration/environment-variables.md +++ b/docs/en/configuration/environment-variables.md @@ -43,9 +43,10 @@ Used when running agents via `sforge run`. These are injected as container envir | Variable | Purpose | |----------|---------| -| `SFORGE_AGENT_API_KEY` | API key passed to the selected agent using that agent's expected env var (`ANTHROPIC_AUTH_TOKEN` or `CODEX_API_KEY`) | +| `SFORGE_AGENT_API_KEY` | API key passed to the selected agent using that agent's expected env var (`ANTHROPIC_AUTH_TOKEN` for Claude Code, `CODEX_API_KEY` for Codex, `ANTHROPIC_API_KEY` for OpenCode) | | `SFORGE_AGENT_API_BASE_URL` | API base URL override passed through the selected agent's base-url env var when it has one | | `SFORGE_AGENT_MODEL` | Model override for the agent | +| `SFORGE_AGENT_EFFORT` | Reasoning effort (`low`, `medium`, `high`, or `max`) | | `SFORGE_AGENT_TIMEOUT` | Agent timeout in seconds | | `SFORGE_AGENT_EXTRA_ENV` | Extra env vars for agent container, format: `"KEY1=VAL1,KEY2=VAL2"` | | `SFORGE_HTTP_PROXY` | HTTP proxy injected into the agent container at run time. Not recommended for normal use; incompatible with `--disable-internet`. | @@ -59,6 +60,11 @@ Used when running agents via `sforge run`. These are injected as container envir Directly injecting `SFORGE_HTTP_PROXY` / `SFORGE_HTTPS_PROXY` into the agent container gives that container proxy-mediated network access. This is useful only for exceptional run-time dependency downloads and is **not recommended** for LLM API access. It is also incompatible with network isolation (`--disable-internet`). If the agent needs to call an LLM API through a corporate proxy while network isolation is enabled, run `sforge proxy` on the host and point `SFORGE_AGENT_API_BASE_URL` to `http://host.docker.internal:` instead. + +For E2B, generic host `HTTP_PROXY` / `HTTPS_PROXY` variables are not forwarded +to Work Sandboxes. Set the `SFORGE_` variants only when the proxy is reachable +from E2B; isolated E2B runs always remove proxy variables and use the native +host allowlist directly. ::: ## Judge Variables @@ -66,6 +72,7 @@ It is also incompatible with network isolation (`--disable-internet`). If the ag | Variable | Purpose | |----------|---------| | `SFORGE_JUDGE_EXTRA_ENV` | Extra env vars for judge containers, format: `"KEY1=VAL1,KEY2=VAL2"` | +| `SFORGE_ADMIN_SECRET` | Trusted-host credential for Judge session registration and auto-eval. If omitted, SForge creates a mode-`0600` secret under `SFORGE_LOG_DIR`. Never expose it to Work containers. | ## Resource Limit Variables @@ -80,11 +87,14 @@ Judge CPU/memory limits are currently set through CLI flags (`--judge-cpu-limit` | Variable | Default | Purpose | |----------|---------|---------| -| `SFORGE_BACKEND` | `docker` | Container backend: `docker` or `k8s` | +| `SFORGE_BACKEND` | `docker` | Container backend: `docker`, `k8s`, or official `e2b` | | `SFORGE_K8S_NAMESPACE` | `default` | Kubernetes namespace | | `SFORGE_K8S_IMAGE_REGISTRY` | --- | Container registry for K8s image pulls | | `SFORGE_K8S_KUBECONFIG` | --- | Path to kubeconfig file | | `SFORGE_K8S_NODE_SELECTOR` | --- | Node selector for K8s pods, format: `"key1=val1,key2=val2"` | +| `SFORGE_E2B_TEMPLATE_MAP` | --- | Explicit E2B image-key to Template mapping, overriding the derived name. Inline `"image_key=template,..."` pairs, or a path ending in `.json` holding a JSON object | +| `SFORGE_E2B_TEMPLATE_NAMESPACE` | --- | Namespace for published E2B Templates (for example, `edgebench`) | +| `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL in seconds | ## Path Variables diff --git a/docs/en/configuration/experiments.md b/docs/en/configuration/experiments.md index 5f0c9c9..97a45b2 100644 --- a/docs/en/configuration/experiments.md +++ b/docs/en/configuration/experiments.md @@ -54,8 +54,9 @@ Default settings applied to all tasks unless overridden per-task. | Field | Type | Default | Description | |-------|------|---------|-------------| -| `agent` | `string` | --- | Agent name (`claude-code`, `codex`) | +| `agent` | `string` | --- | Agent name (`claude-code`, `codex`, `opencode`, or a pinned variant such as `claude-code-2.1.214`) | | `model` | `string` | --- | Model override (overrides `model.model` for specific tasks if set per-task) | +| `effort` | `string` | --- | Reasoning effort (`low`, `medium`, `high`, or `max`); mapped to the selected Agent when supported | | `timeout` | `int` | --- | Agent timeout in seconds | | `eval_interval` | `int` | --- | Auto-eval interval in seconds | | `disable_stop_hook` | `bool` | `false` | Disable the agent stop hook | @@ -63,7 +64,7 @@ Default settings applied to all tasks unless overridden per-task. | `disable_auto_resume` | `bool` | `false` | Disable auto-resume on abnormal agent exit | | `internet` | `bool` | --- | Enable/disable internet access | | `extra_env` | `dict` | --- | Extra environment variables injected into the agent container | -| `backend` | `string` | --- | Container backend (`docker` or `k8s`) | +| `backend` | `string` | --- | Container backend (`docker`, `k8s`, or `e2b`) | | `judge_url` | `string` | --- | Judge server URL override | | `max_submissions` | `int` | --- | Maximum number of agent submissions per run | | `submission_cooldown` | `int` | --- | Minimum seconds between agent submissions | diff --git a/docs/en/examples/single-task-e2b.md b/docs/en/examples/single-task-e2b.md new file mode 100644 index 0000000..24546e0 --- /dev/null +++ b/docs/en/examples/single-task-e2b.md @@ -0,0 +1,173 @@ +--- +title: "Single Task (E2B)" +--- + +# Single Task (E2B) + +Run one EdgeBench task (`ad_placement_optimization`) in [E2B](https://e2b.dev) cloud Sandboxes. No local Docker daemon or Kubernetes cluster is needed; the only thing you host yourself is the Judge Server. + +::: tip When to use E2B +E2B is the easiest way to run tasks when you cannot run Docker locally. Work, Judge, and Game environments are all created as E2B Sandboxes from pre-built official Templates. Concurrency and Sandbox lifetime are bounded by your E2B plan; see [Plan Limits and Run Sizing](/en/configuration/container-backends#plan-limits-and-run-sizing) before scaling up. +::: + +::: warning Cost +Frontier-model runs are expensive — one task over the official 12-hour budget can cost hundreds to over a thousand USD in API usage, plus E2B Sandbox time. Start with a short `--timeout` (this example uses 2 hours) to gauge your burn rate before scaling up. +::: + +## Prerequisites + +| Requirement | Check | +|-------------|-------| +| E2B account and API key | `E2B_API_KEY` from the E2B dashboard | +| A host with a **publicly routable** address for the Judge Server | Sandboxes submit to it over the internet | +| Python >= 3.10 | `python --version` | + +> **Note:** The agent timeout must fit within your E2B plan's maximum Sandbox +> lifetime, including time for evaluation and cleanup. Check your plan before +> running long tasks. + +## Using Claude Code with Anthropic API + +### 1. Install SForge with the E2B extra + +```bash +pip install "sforge[e2b]" +``` + +### 2. Fetch task definitions + +```bash +sforge fetch-tasks edgebench +``` + +Downloads the EdgeBench task JSONs and `BENCHMARK.yaml` into `./tasks/`. +Verify with: + +```bash +sforge list +``` + +### 3. Configure E2B + +E2B Sandboxes start from Templates, not Docker images, so there is no +`sforge pull` step. Official Templates for every published EdgeBench task are +already built; SForge derives the Template name from each task's image +reference. Set the following on **both** the Judge Server host and the run +host: + +```bash +export E2B_API_KEY="e2b_xxxx" +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench +``` + +If the Judge Server and `sforge run` are on **different machines**, they also +need a shared admin secret. The Judge Server only accepts registrations whose +secret matches its own, and each side otherwise generates its own random one +under `SFORGE_LOG_DIR/.judge-admin-secret`. Export the same value on both hosts: + +```bash +export SFORGE_ADMIN_SECRET="$(openssl rand -base64 32)" # same value on both hosts +``` + +When both run on the same machine with the same log directory, they share the +generated file automatically and nothing needs to be set. Never pass the secret +to the agent. + +### 4. Start the judge server + +On the publicly reachable host: + +```bash +sforge serve --host 0.0.0.0 --port 8080 +``` + +The Judge Server needs `E2B_API_KEY` too: when a run registers, it creates +the Judge and Game Sandboxes on E2B, evaluates each submitted archive in an +ephemeral Judge Sandbox, and returns scores. Make sure port 8080 is open to +the internet. + +### 5. Run the agent + +On the run host (can be the same machine), point `--judge-url` at the Judge +Server's public address: + +```bash +SFORGE_AGENT_API_KEY="sk-ant-xxxx" \ +sforge run --backend e2b \ + --task ad_placement_optimization --agent claude-code \ + --model claude-opus-4-8[1m] \ + --timeout 7200 \ + --judge-url http://YOUR_PUBLIC_HOST:8080 \ + --run-id ad-placement-optimization-e2b-001 +``` + +This launches Claude Opus 4.8 in an E2B Work Sandbox for 2 hours. +You will see the agent's work output streamed to stdout in real time. + +Each `sforge run` with `--backend e2b` evaluates tasks independently; passing +several task IDs schedules several separate Sandbox sets. + +### 6. View results + +You can view the progress in real time via the built-in web UI: + +```bash +sforge visualizer +# Open http://127.0.0.1:8000 +``` + +Or inspect files directly: + +```bash +ls logs/runs/*/ad_placement_optimization/ +cat logs/runs/*/ad_placement_optimization/final_result.json +``` + +Sandboxes are deleted automatically when the run finishes. If one lingers for +more than a minute, see [Troubleshooting](/en/configuration/container-backends#troubleshooting). + + +## Using a Third-Party Model + +The model-routing and context-window settings are identical to the Docker +example; only the backend flags change. Follow +[Using a Third-Party Model](/en/examples/single-task-docker#using-a-third-party-model) +and add `--backend e2b --judge-url http://YOUR_PUBLIC_HOST:8080` to the +`sforge run` command, for example: + +```bash +export SFORGE_AGENT_API_KEY="your-deepseek-key" +export SFORGE_AGENT_API_BASE_URL="https://api.deepseek.com/anthropic" +export SFORGE_CLAUDE_CACHE_OPT=1 +export SFORGE_AGENT_EXTRA_ENV="ANTHROPIC_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_OPUS_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_SONNET_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_HAIKU_MODEL=deepseek-v4-pro[1m],CLAUDE_CODE_SUBAGENT_MODEL=deepseek-v4-pro[1m]" + +sforge run --backend e2b \ + --task ad_placement_optimization --agent claude-code \ + --model deepseek-v4-pro[1m] \ + --timeout 7200 \ + --judge-url http://YOUR_PUBLIC_HOST:8080 \ + --run-id ad-placement-deepseek-e2b-001 +``` + + +## Network Isolation + +The `internet` field in each task JSON and the `--disable-internet` / +`--enable-internet` flags work the same as on Docker. On E2B, SForge applies +the allowlist through E2B's native `update_network` API instead of host +iptables, so nothing is left behind on your machine. E2B rules are host-level +rather than port-level: allowing a hostname allows all of its ports. + +Host `HTTP_PROXY` / `HTTPS_PROXY` variables are **not** forwarded into +Sandboxes. If the model API must go through a proxy, it has to be reachable +from the internet and set explicitly via `SFORGE_HTTP_PROXY` / +`SFORGE_HTTPS_PROXY`. See [Network Isolation](/en/features/network-isolation#e2b-backend). + + +## LLM-Graded Tasks + +LLM-graded tasks such as `college_english_exam_bank` need grading credentials +in the Judge environment, exactly as on Docker. Set `SFORGE_JUDGE_EXTRA_ENV` +**before starting the judge server**; see +[LLM-Graded Tasks](/en/examples/single-task-docker#llm-graded-tasks) for the +variables. diff --git a/docs/en/features/network-isolation.md b/docs/en/features/network-isolation.md index 6fd39c1..374fbf2 100644 --- a/docs/en/features/network-isolation.md +++ b/docs/en/features/network-isolation.md @@ -25,6 +25,7 @@ The implementation of `--disable-internet` depends on the container backend: |---------|----------------| | Docker | Host-side iptables/ip6tables rules | | Kubernetes | Kubernetes NetworkPolicy | +| E2B | E2B `update_network` allow/deny rules | ## Allowed Endpoints @@ -97,6 +98,16 @@ sforge run \ Whether NetworkPolicy actually blocks traffic depends on the cluster CNI. Some clusters allow NetworkPolicy objects to be created but do not enforce them. Validate isolation with a small task before relying on it. ::: +## E2B Backend + +The E2B backend applies the allowlist with the official `update_network` API. +E2B rules operate at host level rather than port level, so allowing a hostname +allows all ports on that host. The E2B API currently accepts the +IPv4 all-traffic selector (`0.0.0.0/0`) and rejects `::/0`; SForge therefore +uses the provider-supported selector rather than sending an invalid IPv6 rule. +The deny rule remains active until the Sandbox is deleted. Generic host proxy +variables are not forwarded to E2B Sandboxes. + ## Using with Proxy: `sforge proxy` Directly injecting `SFORGE_HTTP_PROXY` / `SFORGE_HTTPS_PROXY` into the Work container is incompatible with network isolation. If the AI API must be reached through a corporate proxy, run `sforge proxy` on the host and allow the Work container to reach only that local proxy. diff --git a/docs/en/guide/agents.md b/docs/en/guide/agents.md index 2fc0acd..59e1a8b 100644 --- a/docs/en/guide/agents.md +++ b/docs/en/guide/agents.md @@ -8,14 +8,46 @@ SForge manages different Code Agents through a plugin-style agent registry. To r |-------|----------|-------------|-----------|---------------|-----------|-------------| | Claude Code | `claude-code` | `ANTHROPIC_AUTH_TOKEN` | `ANTHROPIC_MODEL` | -- | Yes | Yes | | Codex | `codex` | `CODEX_API_KEY` | `CODEX_MODEL` | -- | Yes | Yes | +| OpenCode | `opencode` | `ANTHROPIC_API_KEY` | -- | -- | No | Yes | Use the `--agent` flag to select an Agent: ```bash sforge run --task ad_placement_optimization --agent claude-code sforge run --task ad_placement_optimization --agent codex +sforge run --task ad_placement_optimization --agent OpenCode ``` +::: details Pinned versions +Each agent installs a fixed release inside the Work container: + +| CLI name | Installed release | +|----------|-------------------| +| `claude-code` | Claude Code 2.1.159 | +| `claude-code-2.1.214` | Claude Code 2.1.214 | +| `codex` | Codex 0.130.0 | +| `codex-0.145.0` | Codex 0.145.0 | +| `opencode` | OpenCode 1.18.2 | + +`claude-code-2.1.214` and `codex-0.145.0` are newer pins kept as separate +agents so existing results stay comparable. Claude 5-family models +(Fable/Mythos) require `claude-code-2.1.214`. +::: + +::: details OpenCode notes +- Models are addressed as `provider/model`; a bare model name is treated as + `anthropic/`. +- The default model, custom base URL (`SFORGE_AGENT_API_BASE_URL`), and + permissions are injected via the `OPENCODE_CONFIG_CONTENT` environment + variable. No config file is written into the container. +- `SFORGE_AGENT_API_BASE_URL` follows the Claude Code convention (no `/v1`); + SForge appends `/v1` for OpenCode's AI SDK client. +- When the task runs without internet, `webfetch` and `websearch` are denied so + the model does not waste turns on blocked requests. +- OpenCode has no Stop-hook mechanism; long runs rely on the harness + auto-resume loop (`opencode run --continue`). +::: + ## Agent Configuration ### API Key @@ -44,6 +76,17 @@ Override the model using the `--model` CLI flag or the `SFORGE_AGENT_MODEL` envi sforge run --task ad_placement_optimization --agent claude-code --model claude-opus-4-8 ``` +### Reasoning Effort + +Use `--effort` or `SFORGE_AGENT_EFFORT` to select `low`, `medium`, `high`, or +`max`. If neither is set, the Agent keeps its own default. + +| Agent | Native setting | Notes | +|-------|----------------|-------| +| Claude Code | `CLAUDE_CODE_EFFORT_LEVEL` | Uses the selected value directly | +| Codex | `model_reasoning_effort` | Maps `max` to Codex's `xhigh` value | +| OpenCode | `reasoningEffort` | Applies to non-Anthropic providers; Anthropic providers retain their own thinking configuration | + ### Extra Environment Variables Pass additional environment variables to the agent container using `SFORGE_AGENT_EXTRA_ENV`: @@ -83,6 +126,7 @@ When an Agent such as Claude Code decides the task is complete and tries to exit |-------|-----------|-------------| | `claude-code` | Claude Code Stop Hook | Registered via `.claude/settings.json` | | `codex` | Codex Stop Hook | Registered via `/etc/codex/hooks.json` | +| `opencode` | -- | No hook mechanism; early exits are covered by auto-resume | ### Disabling the Stop Hook @@ -110,6 +154,7 @@ Auto-resume handles **abnormal agent exits** (API disconnects, transient errors, |-------|------------------| | `claude-code` | `claude --continue -p "Continue working."` | | `codex` | `codex exec resume --last "Continue working."` | +| `opencode` | `opencode run --continue "Continue working."` | ### Safety Guards diff --git a/docs/en/guide/getting-started.md b/docs/en/guide/getting-started.md index 26c5147..c81664f 100644 --- a/docs/en/guide/getting-started.md +++ b/docs/en/guide/getting-started.md @@ -34,6 +34,8 @@ Install the released package: pip install sforge ``` +To use the [E2B backend](/en/configuration/container-backends#e2b-backend), include the `e2b` extra: `pip install "sforge[e2b]"`. + Or install from source: ```bash diff --git a/docs/en/reference/cli.md b/docs/en/reference/cli.md index b62c05c..b3d8172 100644 --- a/docs/en/reference/cli.md +++ b/docs/en/reference/cli.md @@ -114,7 +114,9 @@ The server exposes the [Judge HTTP API](/en/reference/judge-api) and handles bot ## sforge run -Run an agent on one or more tasks. This is the primary command for evaluation. +Run an agent on one task, or batch-schedule independent evaluations for several +tasks. Each task always has its own Work environment, Judge session, results, +and cleanup lifecycle. ### Basic usage @@ -122,7 +124,7 @@ Run an agent on one or more tasks. This is the primary command for evaluation. # Standard agent mode sforge run --task ad_placement_optimization --agent claude-code -# Multiple tasks in parallel +# Batch-schedule independent task runs sforge run --task ad_placement_optimization gitlet rookiedb --agent claude-code # Experiment config mode @@ -133,15 +135,16 @@ sforge run --experiment experiment.yaml | Flag | Default | Description | |------|---------|-------------| -| `--task` | required* | One or more task IDs (space-separated). Multiple tasks run fully in parallel. | -| `--agent` | required* | Agent name (e.g., `claude-code`, `codex`). Required unless `--experiment` is specified. | +| `--task` | required* | One or more task IDs (space-separated). Multiple IDs batch-schedule independent task runs; they are not combined into one evaluation. | +| `--agent` | required* | Agent name: `claude-code`, `codex`, `opencode`, or a pinned variant (`claude-code-2.1.214`, `codex-0.145.0`). Required unless `--experiment` is specified. | | `--experiment` | --- | Path to experiment YAML config file. If `--task` is omitted, all YAML tasks run; if `--task` is provided, only that subset runs with the experiment settings. | | `--model` | --- | Model override (e.g., `claude-opus-4-8`) | +| `--effort` | Agent default | Reasoning effort (`low`, `medium`, `high`, or `max`). SForge maps the value to the selected Agent when supported. | | `--timeout` | `3600` | Agent timeout in seconds | | `--eval-interval` | `300` | Auto-eval daemon interval in seconds | | `--run-id` | random | Run identifier for tracking and log organization | | `--judge-url` | `http://host.docker.internal:8080` | Judge server URL as seen from inside the container | -| `--backend` | `docker` | Container backend (`docker` or `k8s`) | +| `--backend` | `docker` | Container backend (`docker`, `k8s`, or `e2b`) | | `--stagger` | --- | Spread task launches evenly over N seconds (e.g., `--stagger 300`) | | `--max-submissions` | --- | Maximum number of agent submissions per run | | `--submission-cooldown` | --- | Minimum seconds between agent submissions | @@ -152,7 +155,7 @@ sforge run --experiment experiment.yaml | `--disable-stop-hook` | `false` | Disable the stop hook (allow agent to exit normally) | | `--disable-auto-eval` | `false` | Disable the background auto-evaluation daemon | | `--disable-auto-resume` | `false` | Disable auto-resume on abnormal agent exit | -| `--disable-internet` | `false` | Force network isolation (only judge server + API allowed). Requires `sudo` for iptables. Mutually exclusive with `--enable-internet`. | +| `--disable-internet` | `false` | Force network isolation (only judge server + API allowed). On the Docker backend this requires passwordless `sudo` for iptables; k8s and e2b isolate inside the cluster/provider. Mutually exclusive with `--enable-internet`. | | `--enable-internet` | `false` | Force full internet access (overrides per-task `internet: false` setting). Mutually exclusive with `--disable-internet`. | ::: warning Requirement @@ -192,7 +195,7 @@ sforge eval --task ad_placement_optimization --archive - < solution.tar.gz | `--run-id` | Custom run ID for log organization | | `--timeout` | Evaluation timeout in seconds | | `--json` | Also output the full JSON report | -| `--backend` | Container backend (`docker` or `k8s`) | +| `--backend` | Container backend (`docker`, `k8s`, or `e2b`) | | `--judge-cpu-limit` | Number of CPUs for judge container | | `--judge-mem-limit` | Memory limit for judge container (e.g., `'4g'`) | diff --git a/docs/zh/configuration/container-backends.md b/docs/zh/configuration/container-backends.md index 5986bac..999db01 100644 --- a/docs/zh/configuration/container-backends.md +++ b/docs/zh/configuration/container-backends.md @@ -4,7 +4,8 @@ title: 容器后端 # 容器后端 -SForge 通过容器后端创建 Work 容器和 Judge 容器。默认后端是本机 Docker；当需要把运行负载放到 Kubernetes 集群中时，可以切换到 `k8s` 后端。 +SForge 通过容器后端创建 Work 和 Judge 环境。默认后端是本机 Docker；已有 Kubernetes +集群时可以使用 `k8s`，需要 E2B 托管远端运行环境时可以使用 `e2b`。 ## 如何选择后端 @@ -12,8 +13,10 @@ SForge 通过容器后端创建 Work 容器和 Judge 容器。默认后端是本 |------|----------|----------| | `docker` | 本地开发、调试任务、单机评测、小规模实验 | 本机 Docker daemon 可用 | | `k8s` | 共享集群、批量并发评测、希望让 Work/Judge Pod 在集群中运行 | `kubectl` 可访问集群，镜像在集群中可拉取，Judge URL 可从 Pod 访问 | +| `e2b` | 不自行维护集群或 Judge broker 的远端 Work/Judge 运行 | E2B API 与 E2B Template；详见 [E2B 后端](#e2b-后端) | 如果只是本地试跑任务，优先使用默认的 Docker 后端。Kubernetes 后端更适合已经有集群和镜像仓库的团队环境。 +E2B 后端在首次运行新增或发生变化的任务镜像前，需要先准备 Template。 ::: warning Docker 后端不适合大批量运行 每个任务会占用一个 Work 容器外加临时 Judge 容器，各自有独立的 CPU/内存限额。单机并发运行大量任务（约 **20 个以上**）即使是高性能服务器也会出现严重的资源争抢。大批量运行请使用 `k8s` 后端。 @@ -29,15 +32,13 @@ sforge run \ --task ad_placement_optimization \ --agent claude-code \ --backend k8s \ - --judge-url http://10.0.0.12:8080 --backend k8s + --judge-url http://10.0.0.12:8080 # 环境变量覆盖 export SFORGE_BACKEND=k8s sforge run \ --task ad_placement_optimization \ - --agent claude-code \ - --backend k8s \ - --judge-url http://10.0.0.12:8080 + --agent claude-code ``` 实验配置中也可以设置默认后端： @@ -51,7 +52,7 @@ tasks: ad_placement_optimization: {} ``` -## Docker backend +## Docker 后端 Docker 是默认后端，不需要显式配置： @@ -62,9 +63,7 @@ sforge serve export SFORGE_AGENT_API_KEY="sk-..." sforge run \ --task ad_placement_optimization \ - --agent claude-code \ - --backend k8s \ - --judge-url http://10.0.0.12:8080 + --agent claude-code ``` Docker 后端会： @@ -86,7 +85,7 @@ sforge run \ --judge-mem-limit 4g ``` -## Kubernetes backend +## Kubernetes 后端 Kubernetes 后端通过 `kubectl` 创建 Pod。每个 SForge 容器会对应到集群中的一个 Pod，容器名为 `work`，Pod 使用标签 `app=sforge` 和 `sforge-pod=`。 @@ -222,13 +221,102 @@ sforge run \ - 当前 kubeconfig 需要有创建和删除 `NetworkPolicy` 的权限。 - 不同集群的默认 DNS、出口网关和策略实现可能不同，建议先用小任务验证隔离效果。 +## E2B 后端 + +E2B 后端使用 E2B Sandbox 运行 Work、Judge 和 Game 环境。Judge Server 本身与其他后端 +一样部署在你自己的主机上：在 Sandbox 可达的机器上启动 `sforge serve`，并通过 +`--judge-url` 传入其地址。 + +### Docker 镜像与 E2B Template + +Docker 镜像和 E2B Template 是两种不同的运行对象：E2B Sandbox 从 Template 启动， +不能直接传入 Docker image 引用。已发布 EdgeBench 任务的官方 E2B Template 由 +SForge 维护者预先构建并发布，用户不需要执行任何 Template 准备步骤。 +设置 `SFORGE_E2B_TEMPLATE_NAMESPACE` 指定发布 Template 的 namespace。 +`sforge run --backend e2b` 随后会自动推导每个任务镜像对应的公开 Template 引用 +（`edgebench.work.foo_bar:abc123` -> +`edgebench/edgebench-work-foo-bar:abc123`）。只有需要覆盖该映射时 +（例如为修改过的任务运行自建 Template）才设置 `SFORGE_E2B_TEMPLATE_MAP`。 + +### 运行任务 + +在有公网可达地址的主机上启动 Judge Server（E2B Sandbox 里的 Agent 通过公网向它提交）： + +```bash +uv sync --extra e2b +export E2B_API_KEY=... +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench +export SFORGE_ADMIN_SECRET=... # serve 和 run 两台主机需使用同一个值 + +sforge serve --host 0.0.0.0 --port 8080 +``` + +然后运行 Agent，`--judge-url` 指向该主机： + +```bash +export E2B_API_KEY=... +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench +export SFORGE_ADMIN_SECRET=... # 与 serve 主机一致 +export SFORGE_AGENT_API_KEY=... +export SFORGE_AGENT_API_BASE_URL=... +export SFORGE_AGENT_MODEL=... + +sforge run \ + --backend e2b \ + --task ad_placement_optimization \ + --agent claude-code \ + --judge-url http://YOUR_HOST:8080 +``` + +Judge Server 主机同样需要 `E2B_API_KEY`：e2b 运行注册后，Judge Server 会在 E2B 上 +创建 Judge 和 Game Sandbox（与 k8s 运行让它调度 judge pod 的机制对称）。Work Sandbox +向 Judge Server 的公网地址提交；Judge Server 通过 E2B service gateway 访问 Game +Sandbox。 + +每个 E2B 评测实例只对应一个 task。传入多个 task ID 时，CLI 只是批量调度多个相互独立的 +评测，不会把多个 task 合并进同一个 Work 或 Judge 环境。本文验证采用每条 +`sforge run` 命令运行一个 task 的方式。 + +### 套餐限制与运行规模 + +E2B 套餐限制是运行契约的一部分： + +- **资源规格**：Template 的 CPU 和内存在构建时固化，超限会在创建 Template 时被拒绝。 + 请按 team 支持的规格重建；`--work-*` 和 `--judge-*` 参数不能调整已有 Template。 +- **Sandbox 生命周期**：Agent timeout 应明显短于套餐的最大 Sandbox 生命周期，后者还要覆盖 + Agent 安装、归档提取、Judge 评测、读取结果和清理。清理时 SForge 会把剩余生命周期收紧到 + 60 秒内并删除 Sandbox，不受配置的 TTL 影响。 +- **评测超时**：Judge Sandbox 受同样限制，任务的 `judge.eval_timeout` 若超过该限制则可能 + 无法完成。应换更长生命周期的套餐，而不是缩短超时。 +- **并发用量**：任务全并行运行，需按所选任务数为 Work、Judge 和 Game Sandbox 预留容量， + 并为评测与清理的重叠阶段留出余量。 +- **最终分数**：与其他后端一致，取已完成的 agent 或 auto-eval submission 中的最佳结果。 + `final_archive.tar.gz` 是恢复快照，timeout 时不会隐式提交。短任务应保证至少一次 auto-eval + 能完成，或让 Agent 在结束前执行 `sforge-submit`。 + +### 用户需要提供的内容 + +| 输入 | 需要场景 | +| --- | --- | +| `E2B_API_KEY` | E2B 运行（`sforge run` 主机和 Judge Server 主机都需要） | +| 公网可达的 `--judge-url` | 所有 E2B 运行 | +| Agent API key、base URL 和模型 | 运行 Agent | + +宿主机的 `HTTP_PROXY` 和 `HTTPS_PROXY` 不会复制到 E2B Work、Judge 或 Game Sandbox。 +只有用户显式配置 `SFORGE_HTTP_PROXY` 或 `SFORGE_HTTPS_PROXY` 时，远端可访问的代理才会 +被转发；网络隔离任务会移除全部代理变量。详见[网络隔离](/zh/features/network-isolation)。 + ## 常见问题 -| 现象 | 可能原因 | 处理方式 | -|------|----------|----------| -| `kubectl cluster-info failed` | kubeconfig 不正确、集群不可达或 namespace 参数有误 | 检查 `kubectl -n cluster-info`；必要时设置 `SFORGE_K8S_KUBECONFIG` 和 `SFORGE_K8S_NAMESPACE` | -| Pod 一直无法 Running | 镜像拉取失败、调度失败或资源不足 | 使用 `kubectl -n describe pod ` 查看事件 | -| Pod 拉不到镜像 | 镜像没有推送到集群可访问的 registry，或 registry 凭证未配置 | 先 `sforge push`；确认 `SFORGE_K8S_IMAGE_REGISTRY` 正确；为 namespace 配置 image pull secret | -| Work Pod 无法提交评测 | `--judge-url` 不是 Pod 可访问地址，或 Judge server 没有监听外部地址 | 用 `--host 0.0.0.0` 启动 `sforge serve`，并通过 `--judge-url` 设置 Pod 可访问的 IP/Service URL | -| `--disable-internet` 没有效果 | 集群 CNI 不支持 NetworkPolicy 或策略权限不足 | 确认 CNI 支持 NetworkPolicy，并检查当前身份是否能创建 NetworkPolicy | -| node selector 后 Pod Pending | 没有节点匹配 selector | 检查 `SFORGE_K8S_NODE_SELECTOR` 和节点标签 | +| 后端 | 现象 | 可能原因 | 处理方式 | +| --- | --- | --- | --- | +| Kubernetes | `kubectl cluster-info failed` | kubeconfig 不正确、集群不可达或 namespace 参数有误 | 检查 `kubectl -n cluster-info`；必要时设置 `SFORGE_K8S_KUBECONFIG` 和 `SFORGE_K8S_NAMESPACE` | +| Kubernetes | Pod 一直无法 Running | 镜像拉取失败、调度失败或资源不足 | 使用 `kubectl -n describe pod ` 查看事件 | +| Kubernetes | Pod 拉不到镜像 | 镜像对集群节点不可见，或 registry 凭证未配置 | 先 `sforge push`；确认 `SFORGE_K8S_IMAGE_REGISTRY`；为 namespace 配置 image pull secret | +| Kubernetes | Work Pod 无法提交评测 | Judge URL 对 Pod 不可达，或服务没有监听外部地址 | 用 `--host 0.0.0.0` 启动 `sforge serve`，并设置 Pod 可访问的 IP 或 Service URL | +| Kubernetes | `--disable-internet` 没有效果 | CNI 不执行 NetworkPolicy，或权限不足 | 确认 CNI 支持 NetworkPolicy，并检查当前身份的权限 | +| Kubernetes | 配置 node selector 后 Pod Pending | 没有节点匹配 selector | 检查 `SFORGE_K8S_NODE_SELECTOR` 和节点标签 | +| E2B | 找不到 Template | Template namespace 缺失或错误、该任务镜像版本没有已发布的官方 Template，或任务使用了修改过的镜像 | 检查 `SFORGE_E2B_TEMPLATE_NAMESPACE` 和任务镜像版本，或用 `SFORGE_E2B_TEMPLATE_MAP` 指向自建 Template | +| E2B | Agent 侧 Judge 请求偶发超时 | Work Sandbox 无法通过公网访问 Judge Server | 确认 Judge 主机和端口公网可达后重试提交 | +| E2B | task timeout 到达时出现 `Sandbox not found` | 在最终归档和清理前达到了 team 级 Sandbox 最大生命周期 | 缩短 Agent timeout、减少多波次批次总时长，或使用支持更长 Sandbox 生命周期的套餐 | +| E2B | 运行结束超过 60 秒后仍有 Sandbox | 服务端终止期限和即时删除请求均未生效 | 检查运行日志或 Judge Server 日志中的生命周期清理错误 | diff --git a/docs/zh/configuration/environment-variables.md b/docs/zh/configuration/environment-variables.md index 1a7fff6..56872bb 100644 --- a/docs/zh/configuration/environment-variables.md +++ b/docs/zh/configuration/environment-variables.md @@ -43,9 +43,10 @@ SForge 大部分配置可通过 `SFORGE_*` 环境变量注入；仅 CLI/YAML 支 | 变量 | 用途 | |------|------| -| `SFORGE_AGENT_API_KEY` | 传递给所选 Agent 的 API 密钥，会映射为该 Agent 需要的环境变量（`ANTHROPIC_AUTH_TOKEN` 或 `CODEX_API_KEY`） | +| `SFORGE_AGENT_API_KEY` | 传递给所选 Agent 的 API 密钥，会映射为该 Agent 需要的环境变量（Claude Code 为 `ANTHROPIC_AUTH_TOKEN`，Codex 为 `CODEX_API_KEY`，OpenCode 为 `ANTHROPIC_API_KEY`） | | `SFORGE_AGENT_API_BASE_URL` | API Base URL 覆盖，会传递给所选 Agent 支持的 base-url 环境变量 | | `SFORGE_AGENT_MODEL` | Agent 模型覆盖 | +| `SFORGE_AGENT_EFFORT` | Agent 推理强度（`low`、`medium`、`high` 或 `max`） | | `SFORGE_AGENT_TIMEOUT` | Agent 超时时间（秒） | | `SFORGE_AGENT_EXTRA_ENV` | Agent 容器额外环境变量，格式：`"KEY1=VAL1,KEY2=VAL2"` | | `SFORGE_HTTP_PROXY` | 运行阶段注入 Agent 容器的 HTTP 代理。不推荐常规使用；不兼容 `--disable-internet`。 | @@ -59,6 +60,10 @@ SForge 大部分配置可通过 `SFORGE_*` 环境变量注入；仅 CLI/YAML 支 把 `SFORGE_HTTP_PROXY` / `SFORGE_HTTPS_PROXY` 直接注入 Agent 容器，会让容器通过代理获得网络访问能力。这个方式只适合极少数运行时下载依赖的场景，**不推荐**用于 LLM API 访问。 它也不兼容网络隔离模式（`--disable-internet`）。如果需要在网络隔离下通过企业代理访问 LLM API，请在宿主机运行 `sforge proxy`，并把 `SFORGE_AGENT_API_BASE_URL` 指向 `http://host.docker.internal:`。 + +对于 E2B，宿主机通用的 `HTTP_PROXY` / `HTTPS_PROXY` 不会自动传入 Work Sandbox。只有 +显式设置 `SFORGE_` 代理变量且远端能够访问该代理时才应转发；隔离模式始终移除代理并由 +E2B 原生域名白名单直接放行。 ::: ## Judge 变量 @@ -66,6 +71,7 @@ SForge 大部分配置可通过 `SFORGE_*` 环境变量注入；仅 CLI/YAML 支 | 变量 | 用途 | |------|------| | `SFORGE_JUDGE_EXTRA_ENV` | Judge 容器额外环境变量，格式：`"KEY1=VAL1,KEY2=VAL2"` | +| `SFORGE_ADMIN_SECRET` | 可信 Host 用于注册 Judge 会话和 auto-eval 的凭证；未设置时在 `SFORGE_LOG_DIR` 下生成权限为 `0600` 的 secret 文件，不得注入 Work 容器。 | ## 资源限制变量 @@ -80,11 +86,14 @@ Judge CPU/内存限制目前通过 CLI 参数（`--judge-cpu-limit`、`--judge-m | 变量 | 默认值 | 用途 | |------|--------|------| -| `SFORGE_BACKEND` | `docker` | 容器后端：`docker` 或 `k8s` | +| `SFORGE_BACKEND` | `docker` | 容器后端：`docker`、`k8s` 或官方 `e2b` | | `SFORGE_K8S_NAMESPACE` | `default` | Kubernetes 命名空间 | | `SFORGE_K8S_IMAGE_REGISTRY` | --- | K8s 镜像拉取使用的容器镜像仓库 | | `SFORGE_K8S_KUBECONFIG` | --- | kubeconfig 文件路径 | | `SFORGE_K8S_NODE_SELECTOR` | --- | K8s Pod 的节点选择器，格式：`"key1=val1,key2=val2"` | +| `SFORGE_E2B_TEMPLATE_MAP` | --- | E2B image key 到 Template 的显式映射，覆盖自动推导的名称。支持内联 `"image_key=template,..."`，或以 `.json` 结尾的 JSON 对象文件路径 | +| `SFORGE_E2B_TEMPLATE_NAMESPACE` | --- | 已发布 E2B Template 的 namespace（例如 `edgebench`） | +| `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL（秒） | ## 路径变量 diff --git a/docs/zh/configuration/experiments.md b/docs/zh/configuration/experiments.md index b74318c..dbb2870 100644 --- a/docs/zh/configuration/experiments.md +++ b/docs/zh/configuration/experiments.md @@ -54,8 +54,9 @@ tasks: | 字段 | 类型 | 默认值 | 说明 | |------|------|--------|------| -| `agent` | `string` | --- | Agent 名称（`claude-code`、`codex`） | +| `agent` | `string` | --- | Agent 名称（`claude-code`、`codex`、`opencode`，或 `claude-code-2.1.214` 等固定版本变体） | | `model` | `string` | --- | 模型覆盖（在逐任务设置中可覆盖 `model.model`） | +| `effort` | `string` | --- | 推理强度（`low`、`medium`、`high` 或 `max`），在所选 Agent 支持时映射到其原生配置 | | `timeout` | `int` | --- | Agent 超时时间（秒） | | `eval_interval` | `int` | --- | 自动评测间隔（秒） | | `disable_stop_hook` | `bool` | `false` | 禁用 Agent Stop Hook | @@ -63,7 +64,7 @@ tasks: | `disable_auto_resume` | `bool` | `false` | 禁用 Agent 异常退出时的自动恢复 | | `internet` | `bool` | --- | 启用/禁用网络访问 | | `extra_env` | `dict` | --- | 注入 Agent 容器的额外环境变量 | -| `backend` | `string` | --- | 容器后端（`docker` 或 `k8s`） | +| `backend` | `string` | --- | 容器后端（`docker`、`k8s` 或 `e2b`） | | `judge_url` | `string` | --- | Judge 服务器 URL 覆盖 | | `max_submissions` | `int` | --- | 每次运行中 Agent 最大提交次数 | | `submission_cooldown` | `int` | --- | Agent 两次提交之间的最小间隔（秒） | diff --git a/docs/zh/examples/single-task-e2b.md b/docs/zh/examples/single-task-e2b.md new file mode 100644 index 0000000..b93bbc1 --- /dev/null +++ b/docs/zh/examples/single-task-e2b.md @@ -0,0 +1,158 @@ +--- +title: "单任务运行 (E2B)" +--- + +# 单任务运行 (E2B) + +在 [E2B](https://e2b.dev) 云端 Sandbox 中运行一个 EdgeBench 任务（`ad_placement_optimization`）。不需要本地 Docker daemon 或 Kubernetes 集群，唯一需要自己部署的是 Judge Server。 + +::: tip 适用场景 +本地没法跑 Docker 时，E2B 是最省事的方式。Work、Judge 和 Game 环境都会从官方预构建的 Template 创建为 E2B Sandbox。并发数和 Sandbox 生命周期受你的 E2B 套餐限制，扩大规模前请先看[套餐限制与运行规模](/zh/configuration/container-backends#套餐限制与运行规模)。 +::: + +::: warning 费用 +前沿模型的运行费用很高——单个任务跑满官方 12 小时预算，API 费用可达数百甚至上千美元，另外还有 E2B Sandbox 时长费用。建议先用较短的 `--timeout`（本示例为 2 小时）试跑，摸清费用水平后再扩大规模。 +::: + +## 前提条件 + +| 要求 | 验证方式 | +|------|----------| +| E2B 账号和 API key | 在 E2B 控制台获取 `E2B_API_KEY` | +| 一台有**公网可达**地址的主机运行 Judge Server | Sandbox 通过公网向它提交 | +| Python >= 3.10 | `python --version` | + +> **注意：** Agent timeout 必须落在 E2B 套餐允许的最大 Sandbox 生命周期之内， +> 且要给评测和清理留出时间。跑长任务前先确认套餐上限。 + +## 使用 Claude Code + Anthropic API + +### 1. 安装带 E2B 依赖的 SForge + +```bash +pip install "sforge[e2b]" +``` + +### 2. 获取任务定义 + +```bash +sforge fetch-tasks edgebench +``` + +将 EdgeBench 任务 JSON 和 `BENCHMARK.yaml` 下载到 `./tasks/`。验证： + +```bash +sforge list +``` + +### 3. 配置 E2B + +E2B Sandbox 从 Template 启动而不是 Docker 镜像，所以没有 `sforge pull` 这一步。 +所有已发布 EdgeBench 任务的官方 Template 已经构建好，SForge 会根据任务镜像引用 +自动推导 Template 名称。在 Judge Server 主机和运行主机上**都**设置： + +```bash +export E2B_API_KEY="e2b_xxxx" +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench +``` + +如果 Judge Server 和 `sforge run` **不在同一台机器**，两边还需要一个共享的 admin +secret。Judge Server 只接受与自己一致的 secret，而未设置时两边会各自在 +`SFORGE_LOG_DIR/.judge-admin-secret` 下随机生成，值对不上。在两台主机上导出同一个值： + +```bash +export SFORGE_ADMIN_SECRET="$(openssl rand -base64 32)" # 两台主机使用同一个值 +``` + +同一台机器、同一个 log 目录下运行时，两边自动共用生成的文件，不需要设置。 +不要把该 secret 传给 Agent。 + +### 4. 启动 Judge 服务器 + +在公网可达的主机上： + +```bash +sforge serve --host 0.0.0.0 --port 8080 +``` + +Judge Server 同样需要 `E2B_API_KEY`：运行注册后，它会在 E2B 上创建 Judge 和 +Game Sandbox，在临时 Judge Sandbox 中评测每个提交的压缩包并返回分数。 +确保 8080 端口对公网开放。 + +### 5. 运行 Agent + +在运行主机上（可以和 Judge Server 同一台机器），`--judge-url` 指向 Judge Server +的公网地址： + +```bash +SFORGE_AGENT_API_KEY="sk-ant-xxxx" \ +sforge run --backend e2b \ + --task ad_placement_optimization --agent claude-code \ + --model claude-opus-4-8[1m] \ + --timeout 7200 \ + --judge-url http://YOUR_PUBLIC_HOST:8080 \ + --run-id ad-placement-optimization-e2b-001 +``` + +这会在 E2B Work Sandbox 中启动 Claude Opus 4.8 处理该任务，持续 2 小时。 +你会在 stdout 中实时看到 Agent 的工作输出。 + +`--backend e2b` 下每个任务独立评测；传入多个 task ID 只是调度多套互不相关的 Sandbox。 + +### 6. 查看结果 + +通过内置 Web UI 实时查看评分进展： + +```bash +sforge visualizer +# 打开 http://127.0.0.1:8000 +``` + +也可以直接查看文件： + +```bash +ls logs/runs/*/ad_placement_optimization/ +cat logs/runs/*/ad_placement_optimization/final_result.json +``` + +运行结束后 Sandbox 会自动删除。如果超过一分钟仍有残留，见[常见问题](/zh/configuration/container-backends#常见问题)。 + + +## 使用第三方模型 + +模型路由和上下文窗口的配置与 Docker 示例完全相同，只需改后端参数。按照 +[使用第三方模型](/zh/examples/single-task-docker#使用第三方模型)配置，并在 +`sforge run` 命令中加上 `--backend e2b --judge-url http://YOUR_PUBLIC_HOST:8080`，例如： + +```bash +export SFORGE_AGENT_API_KEY="your-deepseek-key" +export SFORGE_AGENT_API_BASE_URL="https://api.deepseek.com/anthropic" +export SFORGE_CLAUDE_CACHE_OPT=1 +export SFORGE_AGENT_EXTRA_ENV="ANTHROPIC_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_OPUS_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_SONNET_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_HAIKU_MODEL=deepseek-v4-pro[1m],CLAUDE_CODE_SUBAGENT_MODEL=deepseek-v4-pro[1m]" + +sforge run --backend e2b \ + --task ad_placement_optimization --agent claude-code \ + --model deepseek-v4-pro[1m] \ + --timeout 7200 \ + --judge-url http://YOUR_PUBLIC_HOST:8080 \ + --run-id ad-placement-deepseek-e2b-001 +``` + + +## 网络隔离 + +任务 JSON 中的 `internet` 字段以及 `--disable-internet` / `--enable-internet` +参数与 Docker 下用法一致。E2B 上 SForge 通过 E2B 原生的 `update_network` API +下发白名单，而不是宿主机 iptables，因此本机不会留下任何残留。E2B 规则是主机级 +而非端口级：放行一个域名即放行它的所有端口。 + +宿主机的 `HTTP_PROXY` / `HTTPS_PROXY` **不会**转发进 Sandbox。如果模型 API +必须走代理，该代理需要公网可达，并通过 `SFORGE_HTTP_PROXY` / +`SFORGE_HTTPS_PROXY` 显式指定。详见[网络隔离](/zh/features/network-isolation#e2b-backend)。 + + +## LLM 评分任务 + +`college_english_exam_bank` 这类 LLM 评分任务需要在 Judge 环境中提供评分用凭证， +与 Docker 下相同。**在启动 Judge 服务器前**设置 `SFORGE_JUDGE_EXTRA_ENV`，变量 +说明见 [LLM 评分任务](/zh/examples/single-task-docker#llm-评分任务)。 diff --git a/docs/zh/features/network-isolation.md b/docs/zh/features/network-isolation.md index 8b84225..00b6ef1 100644 --- a/docs/zh/features/network-isolation.md +++ b/docs/zh/features/network-isolation.md @@ -25,6 +25,7 @@ sforge run --task --agent claude-code --enable-internet |---------|----------| | Docker | 宿主机 iptables/ip6tables 规则 | | Kubernetes | Kubernetes NetworkPolicy | +| E2B | E2B `update_network` allow/deny 规则 | ## 允许访问的端点 @@ -97,6 +98,14 @@ sforge run \ NetworkPolicy 是否真正生效取决于集群 CNI。部分集群即使成功创建 NetworkPolicy，也可能不会实际拦截流量。首次使用时建议用小任务验证隔离效果。 ::: +## E2B backend + +E2B backend 通过官方 `update_network` API 下发白名单。E2B 规则按 Host +生效，不支持端口粒度；允许某个域名意味着允许访问该域名的所有端口。E2B API +当前接受 IPv4 全流量选择器（`0.0.0.0/0`），但会拒绝 `::/0`，因此 SForge 使用平台 +支持的选择器，不发送无效的 IPv6 规则。deny 规则会一直保留到 Sandbox 被删除，宿主机 +通用代理变量也不会自动传入 E2B Sandbox。 + ## 配合代理使用：`sforge proxy` 直接把 `SFORGE_HTTP_PROXY` / `SFORGE_HTTPS_PROXY` 注入 Work 容器与网络隔离不兼容。如果 AI API 必须通过企业代理访问，推荐在宿主机上启动 `sforge proxy`，然后只允许 Work 容器访问这个本地代理。 diff --git a/docs/zh/guide/agents.md b/docs/zh/guide/agents.md index a42f69c..f5ba000 100644 --- a/docs/zh/guide/agents.md +++ b/docs/zh/guide/agents.md @@ -4,20 +4,47 @@ SForge 通过插件式的 Agent 注册表管理不同的 Code Agent。运行 Edg ## Agent 注册表 -SForge 内置了 2 种 Agent： +SForge 内置了以下 Agent： | Agent 名称 | CLI 名称 | API Key 环境变量 | 模型环境变量 | 默认模型 | Stop Hook | Auto-Resume | |-----------|----------|-----------------|-------------|---------|-----------|-------------| | Claude Code | `claude-code` | `ANTHROPIC_AUTH_TOKEN` | `ANTHROPIC_MODEL` | — | 支持 | 支持 | | Codex | `codex` | `CODEX_API_KEY` | `CODEX_MODEL` | — | 支持 | 支持 | +| OpenCode | `opencode` | `ANTHROPIC_API_KEY` | — | — | 不支持 | 支持 | 使用 `--agent` 参数指定要运行的 Agent： ```bash sforge run --task ad_placement_optimization --agent claude-code sforge run --task ad_placement_optimization --agent codex +sforge run --task ad_placement_optimization --agent OpenCode ``` +::: details 固定版本 +每个 Agent 在 Work 容器内安装固定版本： + +| CLI 名称 | 安装版本 | +|----------|----------| +| `claude-code` | Claude Code 2.1.159 | +| `claude-code-2.1.214` | Claude Code 2.1.214 | +| `codex` | Codex 0.130.0 | +| `codex-0.145.0` | Codex 0.145.0 | +| `opencode` | OpenCode 1.18.2 | + +`claude-code-2.1.214` 和 `codex-0.145.0` 是更新的固定版本，作为独立 Agent 保留以保证 +既有结果可比。Claude 5 系列模型（Fable/Mythos）需要使用 `claude-code-2.1.214`。 +::: + +::: details OpenCode 说明 +- 模型名格式为 `provider/model`，裸模型名会视为 `anthropic/`。 +- 默认模型、自定义 base URL（`SFORGE_AGENT_API_BASE_URL`）和权限均通过 + `OPENCODE_CONFIG_CONTENT` 环境变量注入，容器内不写配置文件。 +- `SFORGE_AGENT_API_BASE_URL` 沿用 Claude Code 约定（不带 `/v1`），SForge 会为 + OpenCode 的 AI SDK 客户端自动补上 `/v1`。 +- 任务断网时会禁用 `webfetch` 和 `websearch`，避免模型在被阻断的请求上浪费轮次。 +- OpenCode 无 Stop Hook 机制，长时运行依赖自动恢复循环（`opencode run --continue`）。 +::: + ## Agent 配置 ### API Key @@ -46,6 +73,17 @@ sforge run --task ad_placement_optimization --agent claude-code sforge run --task ad_placement_optimization --agent claude-code --model claude-opus-4-8 ``` +### 推理强度 + +通过 `--effort` 或 `SFORGE_AGENT_EFFORT` 选择 `low`、`medium`、`high` 或 +`max`。两者均未设置时，Agent 保持自身默认配置。 + +| Agent | 原生配置 | 说明 | +|-------|----------|------| +| Claude Code | `CLAUDE_CODE_EFFORT_LEVEL` | 直接使用所选值 | +| Codex | `model_reasoning_effort` | 将 `max` 映射为 Codex 的 `xhigh` | +| OpenCode | `reasoningEffort` | 对非 Anthropic provider 生效；Anthropic provider 保持自身 thinking 配置 | + ### 额外环境变量 通过 `SFORGE_AGENT_EXTRA_ENV` 向 Agent 容器注入额外的环境变量（逗号分隔的 `KEY=VALUE` 对）： @@ -85,6 +123,7 @@ Stop Hook 是 SForge 的一个重要机制，用于阻止 Agent 提前退出。 |-------|----------|------| | `claude-code` | Claude Code Stop Hook | 通过 `.claude/settings.json` 注册 | | `codex` | Codex Stop Hook | 通过 `/etc/codex/hooks.json` 注册 | +| `opencode` | — | 无 hook 机制，提前退出由自动恢复兜底 | ### 禁用 Stop Hook @@ -112,6 +151,7 @@ Auto-Resume 处理的是 **Agent 异常退出** 的情况（如 API 断连、瞬 |-------|---------| | `claude-code` | `claude --continue -p "Continue working."` | | `codex` | `codex exec resume --last "Continue working."` | +| `opencode` | `opencode run --continue "Continue working."` | ### 安全保护 diff --git a/docs/zh/guide/getting-started.md b/docs/zh/guide/getting-started.md index 1e4739d..922d6be 100644 --- a/docs/zh/guide/getting-started.md +++ b/docs/zh/guide/getting-started.md @@ -34,6 +34,8 @@ Docker backend 只适合小批量任务。在单机上并发运行大量任务 pip install sforge ``` +如需使用 [E2B 后端](/zh/configuration/container-backends#e2b-后端)，请带上 `e2b` extra：`pip install "sforge[e2b]"`。 + 也可以从源码安装： ```bash diff --git a/docs/zh/reference/cli.md b/docs/zh/reference/cli.md index c379220..cecf27b 100644 --- a/docs/zh/reference/cli.md +++ b/docs/zh/reference/cli.md @@ -114,7 +114,8 @@ sforge serve --host 0.0.0.0 --port 9090 ## sforge run -在一个或多个任务上运行 Agent。这是评测的核心命令。 +在一个 task 上运行 Agent，或批量调度多个相互独立的 task 评测。每个 task 始终拥有各自的 +Work 环境、Judge session、结果和清理生命周期。 ### 基本用法 @@ -122,7 +123,7 @@ sforge serve --host 0.0.0.0 --port 9090 # 标准 Agent 模式 sforge run --task ad_placement_optimization --agent claude-code -# 多任务并行运行 +# 批量调度多个独立 task sforge run --task ad_placement_optimization gitlet rookiedb --agent claude-code # 实验配置模式 @@ -133,15 +134,16 @@ sforge run --experiment experiment.yaml | 选项 | 默认值 | 说明 | |------|--------|------| -| `--task` | 必填* | 一个或多个任务 ID（空格分隔）。多任务时完全并行运行。 | -| `--agent` | 必填* | Agent 名称（如 `claude-code`、`codex`）。除非指定 `--experiment`，否则必填。 | +| `--task` | 必填* | 一个或多个任务 ID（空格分隔）。传入多个 ID 时，CLI 批量调度多个独立 task，不会把它们合并为一次评测。 | +| `--agent` | 必填* | Agent 名称：`claude-code`、`codex`、`opencode`，或固定版本变体（`claude-code-2.1.214`、`codex-0.145.0`）。除非指定 `--experiment`，否则必填。 | | `--experiment` | --- | 实验 YAML 配置文件路径。如果未指定 `--task`，运行 YAML 中的全部任务；如果指定了 `--task`，则只运行该子集并套用实验配置。 | | `--model` | --- | 模型覆盖（如 `claude-opus-4-8`） | +| `--effort` | Agent 默认值 | 推理强度（`low`、`medium`、`high` 或 `max`）。SForge 会在所选 Agent 支持时映射到其原生配置。 | | `--timeout` | `3600` | Agent 超时时间（秒） | | `--eval-interval` | `300` | 自动评测守护进程的间隔时间（秒） | | `--run-id` | 随机生成 | 运行标识符，用于跟踪和日志组织 | | `--judge-url` | `http://host.docker.internal:8080` | 容器内部看到的 Judge 服务器 URL | -| `--backend` | `docker` | 容器后端（`docker` 或 `k8s`） | +| `--backend` | `docker` | 容器后端（`docker`、`k8s` 或 `e2b`） | | `--stagger` | --- | 将任务启动均匀分散在 N 秒内（如 `--stagger 300`） | | `--max-submissions` | --- | 每次运行的最大 Agent 提交次数 | | `--submission-cooldown` | --- | Agent 两次提交之间的最小间隔（秒） | @@ -152,7 +154,7 @@ sforge run --experiment experiment.yaml | `--disable-stop-hook` | `false` | 禁用 stop hook（允许 Agent 正常退出） | | `--disable-auto-eval` | `false` | 禁用后台自动评测守护进程 | | `--disable-auto-resume` | `false` | 禁用 Agent 异常退出时的自动恢复 | -| `--disable-internet` | `false` | 强制网络隔离（仅允许 Judge 服务器 + API 访问）。需要 `sudo` 权限来配置 iptables。与 `--enable-internet` 互斥。 | +| `--disable-internet` | `false` | 强制网络隔离（仅允许 Judge 服务器 + API 访问）。Docker 后端需要免密 `sudo` 配置 iptables；k8s 和 e2b 在集群/服务商侧隔离，不需要。与 `--enable-internet` 互斥。 | | `--enable-internet` | `false` | 强制开启完整网络访问（覆盖任务的 `internet: false` 设置）。与 `--disable-internet` 互斥。 | ::: warning 必填参数 @@ -192,7 +194,7 @@ sforge eval --task ad_placement_optimization --archive - < solution.tar.gz | `--run-id` | 自定义运行 ID，用于日志组织 | | `--timeout` | 评测超时时间（秒） | | `--json` | 同时输出完整的 JSON 报告 | -| `--backend` | 容器后端（`docker` 或 `k8s`） | +| `--backend` | 容器后端（`docker`、`k8s` 或 `e2b`） | | `--judge-cpu-limit` | Judge 容器的 CPU 数量限制 | | `--judge-mem-limit` | Judge 容器的内存限制（如 `'4g'`） | diff --git a/pyproject.toml b/pyproject.toml index f197ec8..43dd387 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,9 @@ dependencies = [ "huggingface-hub>=0.20", ] +[project.optional-dependencies] +e2b = ["e2b>=2.46.0,<3"] + [project.scripts] sforge = "sforge.cli:main" diff --git a/sforge/cli.py b/sforge/cli.py index ad46d50..1713933 100644 --- a/sforge/cli.py +++ b/sforge/cli.py @@ -20,27 +20,26 @@ import concurrent.futures import copy import json +import os import signal import sys import threading import time from pathlib import Path +from urllib.parse import urlparse import docker +from sforge.harness.benchmark import load_benchmark from sforge.harness.config import SForgeConfig, create_backend_from_config, load_config from sforge.harness.constants import DEFAULT_EVAL_INTERVAL from sforge.harness.docker_build import ( build_all_images, - build_work_image, - build_judge_image, pull_all_images, push_all_images, ) -from sforge.harness.docker_utils import cleanup_container from sforge.harness.run_evaluation import judge_submission -from sforge.harness.benchmark import load_benchmark -from sforge.harness.task_spec import TaskSpec, make_task_spec, load_all_tasks +from sforge.harness.task_spec import TaskSpec, load_all_tasks, make_task_spec def _resolve_task(args, config: SForgeConfig) -> TaskSpec: @@ -252,6 +251,10 @@ def _run_single_task( from sforge.harness.agent import create_agent from sforge.harness.run_agent import run_agent + # CLI --effort wins over experiment/env values already on the config. + if getattr(args, "effort", None): + config.agent_effort = args.effort + agent = create_agent(args.agent, config) effective_timeout = args.timeout or config.agent_timeout or agent.timeout disable_stop_hook = getattr(args, "disable_stop_hook", False) @@ -273,6 +276,8 @@ def _run_single_task( print(f" Timeout: {effective_timeout}s") if args.model or config.agent_model or agent.default_model: print(f" Model: {args.model or config.agent_model or agent.default_model}") + if config.agent_effort: + print(f" Effort: {config.agent_effort}") if not task_spec.game_mode: eval_status = f"{effective_eval_interval}s" if not disable_auto_eval and effective_eval_interval > 0 else "disabled" print(f" Auto-eval: {eval_status}") @@ -340,6 +345,7 @@ def _run_single_task( "task": task_spec.task_id, "run_id": run_id, "model": args.model or config.agent_model or agent.default_model, + "effort": config.agent_effort, **result.to_dict(), } (run_log_dir / "final_result.json").write_text( @@ -408,6 +414,9 @@ def _apply_experiment_overrides( if task_args.model is None and merged.model is not None: task_args.model = merged.model + if getattr(task_args, "effort", None) is None and merged.effort is not None: + task_config.agent_effort = merged.effort + if task_args.timeout is None and merged.timeout is not None: task_args.timeout = merged.timeout @@ -481,6 +490,7 @@ def _effective_config_dict( "task_id": task_spec.task_id, "agent": agent_name, "model": model, + "effort": getattr(args, "effort", None) or config.agent_effort, "timeout": timeout, "eval_interval": eval_interval, "disable_stop_hook": getattr(args, "disable_stop_hook", False), @@ -498,9 +508,19 @@ def _effective_config_dict( if config.agent_api_base_url: d["api_base_url"] = config.agent_api_base_url if config.agent_api_key: - d["api_key"] = config.agent_api_key[:8] + "..." + d["api_key"] = "" if config.agent_extra_env: - d["extra_env"] = config.agent_extra_env + sensitive_markers = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL") + d["extra_env"] = { + key: ( + "" + if any(marker in key.upper() for marker in sensitive_markers) + else value + ) + for key, value in config.agent_extra_env.items() + } + if config.e2b_template_namespace: + d["e2b_template_namespace"] = config.e2b_template_namespace return d @@ -520,7 +540,6 @@ def cmd_run(args): # Apply experiment-level backend/judge_url defaults before creating backend if experiment: - from sforge.harness.experiment import resolve_task_overrides exp_defaults = experiment.defaults if exp_defaults.backend and base_config.backend == "docker" and getattr(args, "backend", None) is None: base_config.backend = exp_defaults.backend @@ -536,8 +555,6 @@ def cmd_run(args): print("Error: --task or --experiment is required") sys.exit(1) - backend = create_backend_from_config(base_config) - run_id = args.run_id or uuid.uuid4().hex[:12] multi = len(task_specs) > 1 verbose = not args.silent and not multi @@ -575,6 +592,28 @@ def cmd_run(args): } (run_root / "run_config.json").write_text(json.dumps(unified, indent=2, ensure_ascii=False)) + backend_names = {task_config.backend for _, task_config, _ in task_runs} + if len(backend_names) != 1: + raise ValueError( + "A run must use one container backend; resolved backends: " + + ", ".join(sorted(backend_names)) + ) + base_config.backend = backend_names.pop() + backend = create_backend_from_config(base_config) + if backend.backend_name == "e2b": + unroutable = [ + ta.judge_url for _, _, ta in task_runs + if urlparse(ta.judge_url).hostname == "host.docker.internal" + ] + if unroutable: + print( + "Error: the e2b backend needs a Judge Server the sandboxes " + "can reach. Start `sforge serve` on a routable host and pass " + "its address via --judge-url.", + file=sys.stderr, + ) + sys.exit(1) + # Resolve stagger: CLI flag wins over experiment YAML stagger = args.stagger if stagger is None and experiment and experiment.stagger: @@ -743,9 +782,10 @@ def cmd_serve(args): """Start the judge HTTP server.""" config = _make_config(args) - from sforge.harness.judge_server import create_app import uvicorn + from sforge.harness.judge_server import create_app + app = create_app(config) print(f"Starting SForge judge server on port {args.port}") uvicorn.run(app, host=args.host, port=args.port) @@ -787,9 +827,10 @@ def cmd_proxy(args): def cmd_visualizer(args): """Start the run-results visualizer web UI.""" - from sforge.visualizer.server import create_app as create_viz_app import uvicorn + from sforge.visualizer.server import create_app as create_viz_app + config = _make_config(args) runs_dir = Path(args.runs_dir).resolve() tasks_dir = Path(args.tasks_dir).resolve() if args.tasks_dir else config.tasks_dir @@ -842,6 +883,7 @@ def _pad(s: str, width: int) -> str: def cmd_fetch_tasks(args): """Download benchmark task definitions from HuggingFace Hub.""" from huggingface_hub import snapshot_download + from sforge.harness.constants import BENCHMARK_REGISTRY, DEFAULT_BENCHMARK benchmark = args.benchmark or DEFAULT_BENCHMARK @@ -931,15 +973,20 @@ def main(): # run p_run = subparsers.add_parser("run", help="Run an agent on one or more tasks") - p_run.add_argument("--backend", choices=["docker", "k8s"], default=None, + p_run.add_argument("--backend", choices=["docker", "k8s", "e2b"], default=None, help="Container backend (default from SFORGE_BACKEND or 'docker')") p_run.add_argument("--task", default=None, nargs="+", help="One or more task IDs (e.g. --task ahc056 ahc057). " "Multiple tasks are run fully in parallel.") p_run.add_argument("--experiment", default=None, help="Path to experiment YAML config file (model config + per-task overrides)") - p_run.add_argument("--agent", default=None, help="Agent name (claude-code, aider, codex)") + p_run.add_argument("--agent", default=None, help="Agent name (claude-code, codex, opencode)") p_run.add_argument("--model", default=None, help="Model override") + p_run.add_argument("--effort", default=None, + choices=["low", "medium", "high", "max"], + help="Reasoning effort; each agent translates to its native " + "mechanism (claude-code env, codex/opencode config). " + "Default: agent's own default") p_run.add_argument("--timeout", type=int, default=None, help="Agent timeout in seconds") p_run.add_argument("--eval-interval", type=int, default=None, help=f"Auto-eval interval in seconds (default {DEFAULT_EVAL_INTERVAL})") p_run.add_argument("--disable-auto-eval", action="store_true", default=False, @@ -980,7 +1027,7 @@ def main(): # eval p_eval = subparsers.add_parser("eval", help="Evaluate an archive") - p_eval.add_argument("--backend", choices=["docker", "k8s"], default=None, + p_eval.add_argument("--backend", choices=["docker", "k8s", "e2b"], default=None, help="Container backend (default from SFORGE_BACKEND or 'docker')") p_eval.add_argument("--task", required=True, help="Task ID") p_eval.add_argument("--archive", required=True, help="Path to .tar.gz archive (or - for stdin)") diff --git a/sforge/harness/agent/claude_code.py b/sforge/harness/agent/claude_code.py index 00c3640..4bb50a8 100644 --- a/sforge/harness/agent/claude_code.py +++ b/sforge/harness/agent/claude_code.py @@ -55,6 +55,9 @@ def augment_env(self, env: dict[str, str], model: str | None) -> None: env["CLAUDE_CODE_ATTRIBUTION_HEADER"] = "0" if not self._config.agent_api_base_url and self._config.agent_api_key: env["ANTHROPIC_API_KEY"] = self._config.agent_api_key + # setdefault so an explicit extra_env value stays authoritative. + if self._config.agent_effort: + env.setdefault("CLAUDE_CODE_EFFORT_LEVEL", self._config.agent_effort) def format_run_cmd( self, @@ -110,9 +113,28 @@ def install_stop_hook( local_settings, PurePosixPath("/home/agent/.claude/settings.json"), ) + result = backend.exec_run( + handle, "chown -R agent:agent /home/agent/.claude", user="root", + ) + if result.exit_code != 0: + raise RuntimeError( + f"Failed to set Claude Code config ownership: {result.output}" + ) logger.info("Configured Claude Code settings with hooks") +class ClaudeCode2_1_214Agent(ClaudeCodeAgent): + """Claude Code pinned to 2.1.214 — required by Claude 5-family models + (Fable/Mythos); the 2.1.159 pin predates them. Kept as a separate agent + so existing claude-code results stay comparable.""" + + name = "claude-code-2.1.214" + install_cmds = [ + ClaudeCodeAgent.install_cmds[0], + "sudo -E npm install -g @anthropic-ai/claude-code@2.1.214", + ] + + # --------------------------------------------------------------------------- # Script generators (private to this module) # --------------------------------------------------------------------------- diff --git a/sforge/harness/agent/codex.py b/sforge/harness/agent/codex.py index 2ae7d67..dc21008 100644 --- a/sforge/harness/agent/codex.py +++ b/sforge/harness/agent/codex.py @@ -70,6 +70,14 @@ def format_run_cmd( prompt_path, model=model, cwd=cwd, internet=internet, resume=resume, ) + effort = self._config.agent_effort + if effort: + # codex vocabulary tops out at "xhigh" where ours says "max". + effort = {"max": "xhigh"}.get(effort, effort) + cmd = cmd.replace( + "codex exec", f'codex exec -c model_reasoning_effort="{effort}"', 1, + ) + if not internet: cmd = cmd.replace( "codex exec", 'codex exec -c web_search="disabled"', 1, @@ -144,6 +152,18 @@ def install_stop_hook( logger.info("Configured Codex hooks") +class Codex0_145_0Agent(CodexAgent): + """Codex pinned to 0.145.0. Kept as a separate agent so existing + codex (0.130.0) results stay comparable.""" + + name = "codex-0.145.0" + install_cmds = [ + CodexAgent.install_cmds[0], + "sudo -E npm install -g @openai/codex@0.145.0", + CodexAgent.install_cmds[2], + ] + + # --------------------------------------------------------------------------- # Script generators (private to this module) # --------------------------------------------------------------------------- diff --git a/sforge/harness/agent/factory.py b/sforge/harness/agent/factory.py index 712178c..7c63fe4 100644 --- a/sforge/harness/agent/factory.py +++ b/sforge/harness/agent/factory.py @@ -19,8 +19,9 @@ from sforge.harness.agent.base import Agent from sforge.harness.config import SForgeConfig -from sforge.harness.agent.claude_code import ClaudeCodeAgent -from sforge.harness.agent.codex import CodexAgent +from sforge.harness.agent.claude_code import ClaudeCode2_1_214Agent, ClaudeCodeAgent +from sforge.harness.agent.codex import Codex0_145_0Agent, CodexAgent +from sforge.harness.agent.opencode import OpenCodeAgent # --------------------------------------------------------------------------- # Registry: name → agent class @@ -28,7 +29,10 @@ _REGISTRY: dict[str, type[Agent]] = { "claude-code": ClaudeCodeAgent, + "claude-code-2.1.214": ClaudeCode2_1_214Agent, "codex": CodexAgent, + "codex-0.145.0": Codex0_145_0Agent, + "opencode": OpenCodeAgent, } diff --git a/sforge/harness/agent/opencode.py b/sforge/harness/agent/opencode.py new file mode 100644 index 0000000..52fa38c --- /dev/null +++ b/sforge/harness/agent/opencode.py @@ -0,0 +1,149 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""opencode agent — env-based config, model flag, permission handling. + +opencode (npm ``opencode-ai``) differs from Claude Code in two ways that +matter to the harness: + +* All configuration (default model, provider base URL, permissions) is + passed via the ``OPENCODE_CONFIG_CONTENT`` env var instead of CLI flags + or config files, so no container-side config install step is needed. +* There is no Stop-hook mechanism, so early exits are handled entirely by + the harness auto-resume loop (``opencode run --continue``). +""" + +from __future__ import annotations + +import json +import shlex + +from sforge.harness.agent.base import Agent + +# Permission set used when the task runs without internet: web tools are +# denied outright so the model doesn't waste turns on blocked requests. +# ``deny`` rules are enforced even under ``--auto``. +_NO_INTERNET_PERMISSION = json.dumps( + {"edit": "allow", "bash": "allow", "webfetch": "deny", "websearch": "deny"}, + separators=(",", ":"), +) + + +class OpenCodeAgent(Agent): + + name = "opencode" + install_cmds = [ + "sudo -E bash -c 'NODE_MIRROR=${SFORGE_NODEJS_MIRROR_URL:-https://nodejs.org/dist} && curl -fsSL $NODE_MIRROR/v20.18.0/node-v20.18.0-linux-x64.tar.xz | tar -xJ -C /usr/local --strip-components=1'", + "sudo -E npm install -g opencode-ai@1.18.2", + # npm's postinstall runs as root with HOME=/home/agent and leaves + # root-owned ~/.local etc. that opencode then cannot write to + "sudo chown -R agent:agent /home/agent", + ] + run_cmd = 'opencode run --format json --auto "$(cat {prompt_file})"' + resume_cmd = 'opencode run --continue --format json --auto "Continue working."' + api_key_env = "ANTHROPIC_API_KEY" + # opencode reads no base-URL env var; the base URL is embedded into + # OPENCODE_CONFIG_CONTENT in augment_env(). default_api_base_url is + # still needed for the network-isolation allowlist. + default_api_base_url = "https://api.anthropic.com" + + def augment_env(self, env: dict[str, str], model: str | None) -> None: + config = self._config + qualified = _qualify_model( + model or config.agent_model or self.default_model + ) + + cfg: dict = { + "permission": { + "edit": "allow", + "bash": "allow", + "webfetch": "allow", + }, + } + if qualified: + # small_model too, or title generation falls back to an + # Anthropic haiku model that third-party endpoints reject. + cfg["model"] = qualified + cfg["small_model"] = qualified + + provider_id, _, model_id = (qualified or "anthropic/").partition("/") + provider_cfg: dict = {} + if config.agent_api_base_url: + provider_cfg["options"] = { + "baseURL": _sdk_base_url(config.agent_api_base_url), + } + if model_id: + # Declare the model so opencode accepts ids unknown to models.dev. + model_cfg: dict = {} + # Anthropic-style providers have no reasoningEffort, only a + # thinking budgetTokens knob; the effort->tokens mapping is a + # policy decision we haven't made, so effort applies to + # OpenAI-style providers only for now. + effort = config.agent_effort + if effort and provider_id != "anthropic": + model_cfg["options"] = { + "reasoningEffort": {"max": "xhigh"}.get(effort, effort), + } + provider_cfg["models"] = {model_id: model_cfg} + if provider_cfg: + cfg["provider"] = {provider_id: provider_cfg} + + env.setdefault("OPENCODE_CONFIG_CONTENT", json.dumps(cfg)) + env.setdefault("OPENCODE_DISABLE_AUTOUPDATE", "1") + env.setdefault("OPENCODE_DISABLE_LSP_DOWNLOAD", "1") + + def format_run_cmd( + self, + prompt_path: str, + *, + model: str | None = None, + cwd: str = "", + internet: bool = True, + resume: bool = False, + ) -> str: + cmd = super().format_run_cmd( + prompt_path, model=model, cwd=cwd, internet=internet, resume=resume, + ) + + qualified = _qualify_model( + model or self._config.agent_model or self.default_model + ) + if qualified: + cmd += f" --model {shlex.quote(qualified)}" + + if not internet: + # OPENCODE_PERMISSION overrides the config permissions; + # OPENCODE_DISABLE_MODELS_FETCH stops the models.dev refresh + # from stalling behind the network isolation. + cmd = ( + f"OPENCODE_PERMISSION={shlex.quote(_NO_INTERNET_PERMISSION)}" + " OPENCODE_DISABLE_MODELS_FETCH=1 " + cmd + ) + + return cmd + + +def _qualify_model(model: str | None) -> str | None: + """opencode expects ``provider/model``; default bare names to anthropic.""" + if not model: + return None + return model if "/" in model else f"anthropic/{model}" + + +def _sdk_base_url(base_url: str) -> str: + """SForge base URLs follow the Claude Code convention (no ``/v1``), + while the AI SDK expects the versioned root it appends ``/messages`` to. + """ + base = base_url.rstrip("/") + return base if base.endswith("/v1") else f"{base}/v1" diff --git a/sforge/harness/backend/__init__.py b/sforge/harness/backend/__init__.py index a869252..43dc231 100644 --- a/sforge/harness/backend/__init__.py +++ b/sforge/harness/backend/__init__.py @@ -23,6 +23,7 @@ ContainerHandle, ExecResult, NetworkIsolationStrategy, + ServiceEndpoint, StreamingExecResult, ) from sforge.harness.backend.factory import create_backend @@ -32,6 +33,7 @@ "ContainerHandle", "ExecResult", "NetworkIsolationStrategy", + "ServiceEndpoint", "StreamingExecResult", "create_backend", ] diff --git a/sforge/harness/backend/base.py b/sforge/harness/backend/base.py index 12aa9c1..6b307fb 100644 --- a/sforge/harness/backend/base.py +++ b/sforge/harness/backend/base.py @@ -42,6 +42,14 @@ class StreamingExecResult: elapsed_seconds: float = 0.0 +@dataclass(frozen=True) +class ServiceEndpoint: + """A routable service URL and provider gateway credentials.""" + + url: str + headers: dict[str, str] = field(default_factory=dict) + + class ContainerHandle(abc.ABC): """Opaque handle to a running container or pod.""" @@ -184,6 +192,14 @@ def exec_run_with_exit_code( def get_container_ip(self, handle: ContainerHandle) -> str: ... + def get_service_endpoint( + self, handle: ContainerHandle, port: int, + ) -> ServiceEndpoint: + """Return the URL plus headers needed to cross the provider gateway.""" + return ServiceEndpoint( + url=f"http://{self.get_container_ip(handle)}:{port}", + ) + @abc.abstractmethod def get_container_gateway_ip(self, handle: ContainerHandle) -> str | None: ... diff --git a/sforge/harness/backend/e2b_backend.py b/sforge/harness/backend/e2b_backend.py new file mode 100644 index 0000000..69f5bbf --- /dev/null +++ b/sforge/harness/backend/e2b_backend.py @@ -0,0 +1,1103 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""E2B backend: runs work/judge containers on official E2B. + +Targets native E2B semantics (https://e2b.dev): + +- endpoint/credentials via the standard e2b env vars + (``E2B_API_KEY``, ``E2B_API_URL`` / ``E2B_DOMAIN``); +- public Template namespace via ``SFORGE_E2B_TEMPLATE_NAMESPACE`` and optional + image-key overrides via ``SFORGE_E2B_TEMPLATE_MAP`` + (see :meth:`E2BBackend.resolve_template`). + +Semantics notes vs the docker backend: + +- Commands run through the sandbox's envd daemon via ``/bin/bash -l -c``, + so string commands get full shell interpretation (docker exec splits + them into argv without a shell). All harness call sites are compatible. +- ``copy_from_container`` preserves the docker contract of returning a + tar archive containing the file. +- stdout/stderr arrive as separate streams; chunks are interleaved in + arrival order, which is close to but not guaranteed identical to the + container's write order. +- cpu/mem limits and cap_drop are template-level concepts in e2b and are + ignored at container creation (logged once). +""" + +from __future__ import annotations + +import io +import json +import logging +import os +import re +import shlex +import tarfile +import threading +import time +import uuid +from collections.abc import Callable +from pathlib import Path, PurePosixPath + +from sforge.harness.backend.base import ( + ContainerBackend, + ContainerHandle, + ExecResult, + NetworkIsolationStrategy, + ServiceEndpoint, + StreamingExecResult, +) + +logger = logging.getLogger(__name__) + +# Sandbox lifetime lease. The renewal thread extends every live sandbox back +# to the full TTL at TTL/3 intervals, so a crashed harness leaks a sandbox for +# at most one TTL. +DEFAULT_SANDBOX_TTL = 3600 + +# Upper bound for the blocking, non-streaming exec_run path. The interface +# carries no per-call timeout there and every caller only runs short +# housekeeping commands (chmod, mkdir, tar list/extract, chown, kill), so a +# generous ceiling turns a hung remote stream into a failure instead of a +# forever-block. The streaming exec paths manage their own deadlines and the +# detached path stays unbounded on purpose so background servers keep running. +DEFAULT_EXEC_TIMEOUT = 600 + +# Files API calls are idempotent at the paths used by the harness. Uploads use +# a bounded request deadline; downloads stream so progress can continue for +# large archives while the SDK still aborts an idle stream. Retry transient +# transport failures, but surface deterministic provider errors immediately. +FILE_TRANSFER_ATTEMPTS = 3 +FILE_TRANSFER_REQUEST_TIMEOUT = 300 +SANDBOX_CREATE_ATTEMPTS = 3 +SANDBOX_CLEANUP_ATTEMPTS = 3 +SANDBOX_LIFECYCLE_REQUEST_TIMEOUT = 10 +SANDBOX_CLEANUP_GRACE_PERIOD = 60 + +_METADATA_NAME_KEY = "sforge_name" +_E2B_NAME_PATTERN = re.compile(r"[a-z0-9_-]{1,128}") + + +class E2BSandboxHandle(ContainerHandle): + """Wraps an e2b Sandbox object.""" + + def __init__( + self, + sandbox, + name: str, + default_user: str | None = None, + ) -> None: + self._sandbox = sandbox + self._name = name + self.default_user = default_user + self._lease_lock = threading.Lock() + self._closing = False + + @property + def id(self) -> str: + return self._sandbox.sandbox_id + + @property + def name(self) -> str: + return self._name + + @property + def ip_address(self) -> str | None: + return None + + @property + def raw(self): + return self._sandbox + + +class E2BNetworkIsolation(NetworkIsolationStrategy): + """Egress whitelist via the e2b network API. + + Translates the harness's ``AllowedEndpoint`` list into + ``update_network(allow_out=, deny_out=[0.0.0.0/0])`` on the running + sandbox. Allow rules take precedence over deny in e2b, which gives exactly + the whitelist-plus-default-drop semantics supported by the official API. + Port granularity is not supported by e2b selectors: allowing an endpoint + allows all ports on that host. + """ + + def __init__(self, sandbox, allowed_endpoints: list, log: logging.Logger) -> None: + self._sandbox = sandbox + self._endpoints = allowed_endpoints + self._logger = log + + def apply(self) -> None: + hosts: list[str] = [] + for ep in self._endpoints: + host = getattr(ep, "hostname", None) or getattr(ep, "ip", None) + if host and host not in hosts: + hosts.append(host) + try: + self._sandbox.update_network( + {"allow_out": hosts, "deny_out": ["0.0.0.0/0"]} + ) + except Exception as exc: + raise RuntimeError( + "e2b network isolation failed: the provider rejected " + f"update_network ({exc}). Run with --enable-internet to skip " + "isolation, or use an internet-enabled template." + ) from exc + self._logger.info( + "e2b network isolation applied: allow=%s deny=all (host-level, " + "ports are not filtered)", + hosts, + ) + + def cleanup(self) -> None: + # E2B Sandboxes are never reused. Keep the deny rules in place until + # deletion so a transient kill failure cannot leave an unrestricted + # workload alive for the remainder of its TTL. + return None + + +class E2BBackend(ContainerBackend): + """Container backend for the official E2B Sandbox API.""" + + def __init__( + self, + template_map: dict[str, str] | None = None, + sandbox_ttl: int | None = None, + *, + template_namespace: str | None = None, + sandbox_cls=None, + template_cls=None, + ) -> None: + if sandbox_cls is None: + try: + from e2b import Sandbox as sandbox_cls + except ImportError as exc: + raise RuntimeError( + "The e2b backend requires the 'e2b' package. " + "Install with: uv sync --extra e2b (or pip install e2b)" + ) from exc + self._sandbox_cls = sandbox_cls + if template_cls is None: + try: + from e2b import Template as template_cls + except ImportError: + template_cls = None + self._template_cls = template_cls + self._template_map = ( + dict(template_map) if template_map else template_map_from_env() + ) + self._template_namespace = ( + template_namespace + if template_namespace is not None + else os.environ.get("SFORGE_E2B_TEMPLATE_NAMESPACE", "") + ).strip() or None + if ( + self._template_namespace is not None + and not _E2B_NAME_PATTERN.fullmatch(self._template_namespace) + ): + raise ValueError( + "E2B Template namespace must contain only lowercase letters, " + "numbers, dashes, and underscores, and be at most 128 characters" + ) + self._sandbox_ttl = ( + int(os.environ.get("SFORGE_E2B_SANDBOX_TTL", DEFAULT_SANDBOX_TTL)) + if sandbox_ttl is None + else sandbox_ttl + ) + if self._sandbox_ttl <= 0: + raise ValueError("E2B Sandbox TTL must be greater than zero") + self._handles: dict[str, E2BSandboxHandle] = {} + self._handles_lock = threading.Lock() + self._renewal_thread: threading.Thread | None = None + self._renewal_wakeup = threading.Event() + self._warned_limits = False + + @property + def backend_name(self) -> str: + return "e2b" + + # --- Template resolution --- + + def resolve_template(self, image_key: str) -> str: + """Map an image key to an e2b template reference. + + Explicit entries in ``SFORGE_E2B_TEMPLATE_MAP`` win (matched with and + without the image tag); otherwise the repo part is normalized to the + template-name charset, the image tag carries over as the Template tag, + and ``SFORGE_E2B_TEMPLATE_NAMESPACE`` is prepended when configured + (``edgebench.work.foo_bar:abc123`` -> + ``edgebench/edgebench-work-foo-bar:abc123``). + """ + if image_key in self._template_map: + return self._template_map[image_key] + repo, sep, tag = image_key.partition(":") + if repo in self._template_map: + return self._template_map[repo] + sep + tag + return e2b_template_reference(image_key, self._template_namespace) + + # --- Lifecycle --- + + def create_container( + self, + image: str, + name: str, + *, + command: str = "tail -f /dev/null", + environment: dict[str, str] | None = None, + extra_hosts: dict[str, str] | None = None, + cap_drop: list[str] | None = None, + cpu_limit: int | None = None, + mem_limit: str | None = None, + user: str | None = None, + annotations: dict[str, str] | None = None, + ) -> E2BSandboxHandle: + if (cpu_limit or mem_limit or cap_drop) and not self._warned_limits: + self._warned_limits = True + logger.warning( + "e2b backend: cpu/mem limits and cap_drop are fixed at " + "template build time; per-container values are ignored" + ) + + template = self.resolve_template(image) + envs = {k: str(v) for k, v in (environment or {}).items() if v is not None} + metadata = {str(key): str(value) for key, value in (annotations or {}).items()} + metadata[_METADATA_NAME_KEY] = name + create_kwargs = { + "template": template, + "envs": envs, + "metadata": metadata, + "timeout": self._sandbox_ttl, + "secure": True, + "network": {"allow_public_traffic": False}, + "lifecycle": {"on_timeout": "kill"}, + } + sandbox = self._create_sandbox(name, create_kwargs) + handle = E2BSandboxHandle(sandbox, name, default_user=user) + with self._handles_lock: + self._handles[handle.id] = handle + self._ensure_renewal_thread() + try: + # host.docker.internal has no meaning outside the docker backend and + # would shadow real DNS if written into /etc/hosts. + hosts = { + h: ip + for h, ip in (extra_hosts or {}).items() + if h != "host.docker.internal" and ip != "host-gateway" + } + if hosts: + lines = "".join(f"{ip} {h}\\n" for h, ip in hosts.items()) + sandbox.commands.run( + f"printf %s {shlex.quote(lines)} >> /etc/hosts", + user="root", + timeout=30, + ) + + if command and command != "tail -f /dev/null": + sandbox.commands.run( + command, + background=True, + user=user, + timeout=0, + envs=envs or None, + ) + except BaseException: + try: + self.cleanup_container(handle, logger) + except Exception: + logger.exception( + "Failed to clean up E2B Sandbox %s after initialization error", + handle.id, + ) + raise + return handle + + def _create_sandbox(self, name: str, create_kwargs: dict): + """Create a Sandbox, reconciling an ambiguous API timeout by name. + + A create request may reach E2B even when the client times out before + receiving its response. The unique SForge name in metadata lets us + reconnect to that Sandbox instead of leaking it and creating another. + """ + error = None + for attempt in range(1, SANDBOX_CREATE_ATTEMPTS + 1): + try: + return self._sandbox_cls.create(**create_kwargs) + except Exception as exc: + error = exc + if not _is_retryable_transport_error(exc): + raise + try: + matches = list(self._iter_sandboxes_by_name(name)) + except RuntimeError as lookup_exc: + logger.warning( + "Could not reconcile timed-out E2B Sandbox creation " + "for %s (attempt %d/%d): %s", + name, attempt, SANDBOX_CREATE_ATTEMPTS, lookup_exc, + ) + else: + if matches: + sandbox_id = matches[0].sandbox_id + logger.warning( + "E2B Sandbox creation response was lost; " + "reconnecting to %s (%s)", name, sandbox_id, + ) + return self._sandbox_cls.connect( + sandbox_id, timeout=self._sandbox_ttl, + ) + if attempt < SANDBOX_CREATE_ATTEMPTS: + logger.warning( + "E2B Sandbox creation failed for %s " + "(attempt %d/%d): %s", + name, attempt, SANDBOX_CREATE_ATTEMPTS, exc, + ) + time.sleep(attempt) + raise RuntimeError( + f"Failed to create E2B Sandbox {name!r} after " + f"{SANDBOX_CREATE_ATTEMPTS} attempts: {error}" + ) from error + + def start_container(self, handle: ContainerHandle) -> None: + self._raw(handle) + # E2B Sandboxes are running as soon as create() returns. + + def cleanup_container( + self, + handle: ContainerHandle | None, + logger_: logging.Logger | None = None, + ) -> None: + if handle is None: + return + log = logger_ or logger + sandbox = self._raw(handle) + # Cleanup is a terminal lease transition, independent of the runtime + # TTL selected by the user. Remove the handle from lease renewal first, + # then shorten its provider-side deadline before attempting deletion. + # If E2B temporarily rejects kill(), on_timeout=kill still reclaims the + # Sandbox within the cleanup grace period. + with self._handles_lock: + self._handles.pop(handle.id, None) + self._renewal_wakeup.set() + cleanup_deadline_armed = False + cleanup_timeout = min( + self._sandbox_ttl, SANDBOX_CLEANUP_GRACE_PERIOD, + ) + with handle._lease_lock: + handle._closing = True + for attempt in range(1, SANDBOX_CLEANUP_ATTEMPTS + 1): + try: + sandbox.set_timeout( + cleanup_timeout, + request_timeout=SANDBOX_LIFECYCLE_REQUEST_TIMEOUT, + ) + cleanup_deadline_armed = True + break + except Exception as exc: + if attempt == SANDBOX_CLEANUP_ATTEMPTS: + log.warning( + "Could not shorten sandbox %s cleanup deadline " + "after %d attempts: %s", + handle.id, attempt, exc, + ) + else: + time.sleep(attempt) + log.info(f"Killing sandbox {handle.name} ({handle.id})...") + for attempt in range(1, SANDBOX_CLEANUP_ATTEMPTS + 1): + try: + killed = sandbox.kill( + request_timeout=SANDBOX_LIFECYCLE_REQUEST_TIMEOUT, + ) + if killed: + log.info(f"Sandbox {handle.name} killed.") + else: + log.info(f"Sandbox {handle.name} was already gone.") + return + except Exception as exc: + if attempt == SANDBOX_CLEANUP_ATTEMPTS: + if cleanup_deadline_armed: + log.warning( + "Failed to kill sandbox %s after %d attempts; " + "E2B will reclaim it within %ds: %s", + handle.id, attempt, + cleanup_timeout, exc, + ) + else: + message = ( + "Failed to arm cleanup deadline and kill sandbox " + "%s after %d attempts; original TTL remains the " + "only safeguard: %s" + ) % (handle.id, attempt, exc) + log.error(message) + raise RuntimeError(message) from exc + return + time.sleep(attempt) + + def container_exists(self, name: str) -> bool: + return any(True for _ in self._iter_sandboxes_by_name(name)) + + def remove_container_by_name(self, name: str) -> None: + for info in self._iter_sandboxes_by_name(name): + for attempt in range(1, 4): + try: + self._sandbox_cls.kill(info.sandbox_id) + with self._handles_lock: + self._handles.pop(info.sandbox_id, None) + self._renewal_wakeup.set() + break + except Exception as exc: + if attempt == 3: + raise RuntimeError( + f"Failed to remove stale E2B Sandbox " + f"{info.sandbox_id}: {exc}" + ) from exc + time.sleep(attempt) + + def _iter_sandboxes_by_name(self, name: str): + from e2b import SandboxQuery # local: keeps module import-light + + query = SandboxQuery(metadata={_METADATA_NAME_KEY: name}) + try: + paginator = self._sandbox_cls.list(query=query) + while paginator.has_next: + for info in paginator.next_items(): + if (info.metadata or {}).get(_METADATA_NAME_KEY) == name: + yield info + except Exception as exc: + raise RuntimeError( + f"Could not list E2B Sandboxes named {name!r}: {exc}" + ) from exc + + # --- Image --- + + def image_exists(self, image_key: str) -> bool: + if self._template_cls is None: + return False + template = self.resolve_template(image_key) + if _is_namespaced_template_reference(template): + logger.debug( + "E2B cannot preflight public Template %r across namespaces; " + "deferring validation to Sandbox.create()", + template, + ) + return True + try: + # Check the complete name:tag alias. Sandbox.create remains the + # authoritative launchability check performed later in the flow. + return bool(self._template_cls.exists(template)) + except Exception as exc: + raise RuntimeError( + f"Could not verify E2B template {template!r}: {exc}" + ) from exc + + # --- File transfer --- + + def copy_to_container( + self, + handle: ContainerHandle, + src: Path, + dst: PurePosixPath, + ) -> None: + if str(dst.parent) == ".": + raise ValueError(f"Destination parent directory cannot be empty: {dst}") + sandbox = self._raw(handle) + # Harness file transfers include privileged destinations such as + # /usr/local/bin and /etc. Match the effective daemon-level behavior + # of Docker's put_archive by writing as root; callers set ownership + # explicitly when a user-owned destination requires it. + run_user = "root" + sandbox.commands.run( + f"mkdir -p {shlex.quote(str(dst.parent))}", + user=run_user, + timeout=60, + ) + + if src.is_dir(): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + tar.add(src, arcname=dst.name) + tmp = f"/tmp/.sforge-copy-{uuid.uuid4().hex}.tar" + self._retry_file_transfer( + f"upload {src}", + lambda: sandbox.files.write( + tmp, buf.getvalue(), user=run_user, + request_timeout=FILE_TRANSFER_REQUEST_TIMEOUT, + ), + ) + sandbox.commands.run( + f"tar xf {tmp} -C {shlex.quote(str(dst.parent))}; " + f"status=$?; rm -f {tmp}; exit $status", + user=run_user, + timeout=300, + ) + else: + data = src.read_bytes() + self._retry_file_transfer( + f"upload {src}", + lambda: sandbox.files.write( + str(dst), data, user=run_user, + request_timeout=FILE_TRANSFER_REQUEST_TIMEOUT, + ), + ) + + def copy_from_container( + self, + handle: ContainerHandle, + src: PurePosixPath, + *, + shutdown_event: threading.Event | None = None, + ) -> bytes: + sandbox = self._raw(handle) + archive = f"/tmp/.sforge-download-{uuid.uuid4().hex}.tar" + command = ( + f"tar cf {shlex.quote(archive)} " + f"-C {shlex.quote(str(src.parent))} {shlex.quote(src.name)}" + ) + try: + result = sandbox.commands.run(command, user="root", timeout=300) + if result.exit_code != 0: + raise RuntimeError( + f"Failed to archive {src}: {result.stderr or result.stdout}" + ) + def download() -> bytes: + if shutdown_event is not None and shutdown_event.is_set(): + raise InterruptedError(f"Download of {src} was cancelled") + reader = sandbox.files.read( + archive, format="stream", user="root", + ) + try: + chunks = [] + for chunk in reader: + if ( + shutdown_event is not None + and shutdown_event.is_set() + ): + raise InterruptedError( + f"Download of {src} was cancelled" + ) + chunks.append(chunk) + return b"".join(chunks) + finally: + close = getattr(reader, "close", None) + if close is not None: + close() + + return self._retry_file_transfer(f"download {src}", download) + finally: + try: + sandbox.files.remove(archive, user="root") + except Exception: + logger.debug( + "Could not remove E2B download archive %s", + archive, + exc_info=True, + ) + + def write_to_container( + self, + handle: ContainerHandle, + data: str, + dst: PurePosixPath, + ) -> None: + sandbox = self._raw(handle) + self._retry_file_transfer( + f"write {dst}", + lambda: sandbox.files.write( + str(dst), data, user="root", + request_timeout=FILE_TRANSFER_REQUEST_TIMEOUT, + ), + ) + + # --- Exec --- + + def exec_run( + self, + handle: ContainerHandle, + cmd: str | list[str], + *, + user: str | None = None, + workdir: str | None = None, + environment: dict[str, str] | None = None, + detach: bool = False, + ) -> ExecResult: + sandbox = self._raw(handle) + run_user = user or self._default_user(handle) + cmd_str = _as_shell(cmd) + envs = environment or None + + if detach: + sandbox.commands.run( + cmd_str, + background=True, + user=run_user, + cwd=workdir, + envs=envs, + timeout=0, + ) + return ExecResult(output="", exit_code=0) + + chunks: list[str] = [] + try: + result = sandbox.commands.run( + cmd_str, + user=run_user, + cwd=workdir, + envs=envs, + timeout=DEFAULT_EXEC_TIMEOUT, + on_stdout=chunks.append, + on_stderr=chunks.append, + ) + exit_code = result.exit_code + except Exception as exc: + exit_code = _exit_code_of(exc) + if exit_code is None: + raise + return ExecResult(output="".join(chunks), exit_code=exit_code) + + def exec_run_with_timeout( + self, + handle: ContainerHandle, + cmd: str | list[str], + timeout: int | None = 60, + *, + log_file: Path | None = None, + user: str | None = None, + workdir: str | None = None, + environment: dict[str, str] | None = None, + stream_to_stdout: bool = False, + shutdown_event: threading.Event | None = None, + log_append: bool = False, + on_chunk: Callable[[bytes], None] | None = None, + ) -> StreamingExecResult: + output, exit_code, timed_out, elapsed = self._exec_stream( + handle, + cmd, + timeout, + log_file=log_file, + user=user, + workdir=workdir, + environment=environment, + stream_to_stdout=stream_to_stdout, + shutdown_event=shutdown_event, + log_append=log_append, + on_chunk=on_chunk, + ) + return StreamingExecResult( + output=output, + exit_code=exit_code, + timed_out=timed_out, + elapsed_seconds=elapsed, + ) + + def exec_run_with_exit_code( + self, + handle: ContainerHandle, + cmd: str | list[str], + timeout: int | None = 60, + *, + user: str | None = None, + workdir: str | None = None, + environment: dict[str, str] | None = None, + ) -> StreamingExecResult: + output, exit_code, timed_out, elapsed = self._exec_stream( + handle, + cmd, + timeout, + user=user, + workdir=workdir, + environment=environment, + ) + return StreamingExecResult( + output=output, + exit_code=exit_code, + timed_out=timed_out, + elapsed_seconds=elapsed, + ) + + def _exec_stream( + self, + handle: ContainerHandle, + cmd: str | list[str], + timeout: int | None, + *, + log_file: Path | None = None, + user: str | None = None, + workdir: str | None = None, + environment: dict[str, str] | None = None, + stream_to_stdout: bool = False, + shutdown_event: threading.Event | None = None, + log_append: bool = False, + on_chunk: Callable[[bytes], None] | None = None, + ) -> tuple[str, int, bool, float]: + import sys + + sandbox = self._raw(handle) + run_user = user or self._default_user(handle) + chunks: list[str] = [] + exit_code: int | None = None + exception: Exception | None = None + timed_out = False + log_fh = None + + try: + if log_file: + log_file.parent.mkdir(parents=True, exist_ok=True) + log_fh = open(log_file, "ab" if log_append else "wb") + + def emit(text: str) -> None: + chunks.append(text) + raw = text.encode(errors="replace") + if log_fh: + log_fh.write(raw) + log_fh.flush() + if stream_to_stdout: + sys.stdout.buffer.write(raw) + sys.stdout.buffer.flush() + if on_chunk is not None: + try: + on_chunk(raw) + except Exception: + logger.debug( + "E2B output callback failed", exc_info=True, + ) + + # The wait stream can be interrupted by request timeouts or + # ordinary network failures while agent runs last hours. The + # command therefore runs in the background — it survives a dropped + # stream — and its exit code is persisted in the sandbox so it is + # recoverable even if the process finishes while we are detached. + rc_file = f"/tmp/.sforge-rc-{uuid.uuid4().hex}" + wrapped = ( + f"{{ {_as_shell(cmd)}\n}}; __ec=$?; echo $__ec > {rc_file}; exit $__ec" + ) + proc = sandbox.commands.run( + wrapped, + background=True, + user=run_user, + cwd=workdir, + envs=environment or None, + timeout=0, + ) + + def read_rc_file() -> int | None: + try: + result = sandbox.commands.run( + f"cat {rc_file}", + user="root", + timeout=30, + ) + return int(result.stdout.strip()) + except Exception: + return None + + def wait_command() -> None: + nonlocal exit_code, exception + attach = proc + reconnects = 0 + while True: + try: + result = attach.wait(on_stdout=emit, on_stderr=emit) + exit_code = result.exit_code + return + except Exception as exc: + code = _exit_code_of(exc) + if code is not None: + exit_code = code + return + # Stream dropped, not a command failure. Reattach to + # the still-running process; output produced while + # detached is lost, the exit code is not. + last_exc = exc + for attempt in range(5): + if shutdown_event and shutdown_event.is_set(): + exception = last_exc + return + try: + attach = sandbox.commands.connect(proc.pid, timeout=0) + break + except Exception: + code = read_rc_file() + if code is not None: + # Command finished while we were detached. + exit_code = code + return + time.sleep(min(2**attempt, 15)) + else: + exception = last_exc + return + reconnects += 1 + logger.info( + "e2b exec stream dropped, reattached to pid %s (reconnect #%d)", + proc.pid, + reconnects, + ) + + start_time = time.time() + thread = threading.Thread(target=wait_command, daemon=True) + thread.start() + + deadline = start_time + timeout if timeout else None + while thread.is_alive(): + remaining = (deadline - time.time()) if deadline else 1.0 + if remaining <= 0: + break + thread.join(min(remaining, 1.0)) + if shutdown_event and shutdown_event.is_set(): + break + + if thread.is_alive(): + timed_out = True + try: + killed = sandbox.commands.kill(proc.pid) + if killed is False: + raise RuntimeError( + f"E2B command {proc.pid} was not found while timing out" + ) + except Exception as exc: + raise RuntimeError( + f"Failed to stop timed-out E2B command {proc.pid}: {exc}" + ) from exc + thread.join(10) + if thread.is_alive(): + raise RuntimeError( + f"Timed-out E2B command {proc.pid} did not terminate" + ) + # The late thread result (e.g. 137 from the kill) must not + # shadow the docker-contract exit code for timeouts. + exit_code = -1 + exception = None + finally: + if log_fh: + log_fh.close() + if "rc_file" in locals(): + try: + sandbox.files.remove(rc_file, user="root") + except Exception: + logger.debug( + "Could not remove E2B command status file %s", + rc_file, + exc_info=True, + ) + + if exception: + raise exception + + elapsed = time.time() - start_time + return ( + "".join(chunks), + exit_code if exit_code is not None else -1, + timed_out, + elapsed, + ) + + # --- Inspection --- + + def get_container_ip(self, handle: ContainerHandle) -> str: + raise NotImplementedError( + "e2b sandboxes have no routable IP; use get_service_endpoint() to " + "reach in-sandbox services through the e2b gateway" + ) + + def get_service_endpoint( + self, + handle: ContainerHandle, + port: int, + ) -> ServiceEndpoint: + sandbox = self._raw(handle) + token = getattr(sandbox, "traffic_access_token", None) + if not token: + raise RuntimeError( + "E2B did not return a traffic access token for the secured service" + ) + return ServiceEndpoint( + url=f"https://{sandbox.get_host(port)}", + headers={"e2b-traffic-access-token": token}, + ) + + def get_container_gateway_ip(self, handle: ContainerHandle) -> str | None: + return None + + # --- Network isolation --- + + def create_network_isolation( + self, + handle: ContainerHandle, + allowed_endpoints: list, + logger_: logging.Logger, + ) -> NetworkIsolationStrategy: + return E2BNetworkIsolation(self._raw(handle), allowed_endpoints, logger_) + + # --- Lease renewal --- + + def _ensure_renewal_thread(self) -> None: + """Start one renewal worker while holding ``_handles_lock``.""" + if self._renewal_thread is not None and self._renewal_thread.is_alive(): + return + self._renewal_wakeup.clear() + self._renewal_thread = threading.Thread( + target=self._renewal_loop, + daemon=True, + name="e2b-lease-renewal", + ) + self._renewal_thread.start() + + def _renewal_loop(self) -> None: + interval = max(self._sandbox_ttl // 3, 1) + while True: + woke_early = self._renewal_wakeup.wait(interval) + with self._handles_lock: + handles = list(self._handles.values()) + if not handles: + self._renewal_thread = None + return + if woke_early: + self._renewal_wakeup.clear() + if woke_early: + continue + for handle in handles: + self._renew_handle(handle) + + def _renew_handle(self, handle: E2BSandboxHandle) -> None: + with handle._lease_lock: + if handle._closing: + return + try: + handle.raw.set_timeout( + self._sandbox_ttl, + request_timeout=SANDBOX_LIFECYCLE_REQUEST_TIMEOUT, + ) + except Exception as exc: + logger.warning( + f"Lease renewal failed for sandbox {handle.id}: {exc}" + ) + + # --- Helpers --- + + @staticmethod + def _default_user(handle: ContainerHandle) -> str | None: + return getattr(handle, "default_user", None) + + @staticmethod + def _raw(handle: ContainerHandle): + if isinstance(handle, E2BSandboxHandle): + return handle.raw + raise TypeError(f"Expected E2BSandboxHandle, got {type(handle).__name__}") + + @staticmethod + def _retry_file_transfer(description: str, operation): + for attempt in range(1, FILE_TRANSFER_ATTEMPTS + 1): + try: + return operation() + except Exception as exc: + if ( + attempt == FILE_TRANSFER_ATTEMPTS + or not _is_retryable_transport_error(exc) + ): + raise + logger.warning( + "E2B file transfer failed while attempting to %s " + "(attempt %d/%d): %s", + description, attempt, FILE_TRANSFER_ATTEMPTS, exc, + ) + time.sleep(attempt) + raise AssertionError("unreachable") + + +def _as_shell(cmd: str | list[str]) -> str: + return shlex.join(cmd) if isinstance(cmd, list) else cmd + + +def e2b_template_reference(image_key: str, namespace: str | None = None) -> str: + repo, separator, tag = image_key.partition(":") + name = repo.lower().replace(".", "-").replace("_", "-") + if not _E2B_NAME_PATTERN.fullmatch(name): + raise ValueError( + f"E2B Template name derived from {image_key!r} is invalid: {name!r}" + ) + if namespace: + if not _E2B_NAME_PATTERN.fullmatch(namespace): + raise ValueError( + "E2B Template namespace must contain only lowercase letters, " + "numbers, dashes, and underscores, and be at most 128 characters" + ) + name = f"{namespace}/{name}" + return name + separator + tag + + +def _is_namespaced_template_reference(template: str) -> bool: + name, _, _tag = template.partition(":") + parts = name.split("/") + return len(parts) == 2 and all(_E2B_NAME_PATTERN.fullmatch(part) for part in parts) + + +def _exit_code_of(exc: Exception) -> int | None: + """Exit code carried by a CommandExitException, duck-typed so fakes and + provider variants work.""" + code = getattr(exc, "exit_code", None) + return code if isinstance(code, int) else None + + +def _is_retryable_transport_error(exc: Exception) -> bool: + """Recognize transport failures without retrying provider rejections.""" + current: BaseException | None = exc + while current is not None: + if type(current).__name__ in { + "ConnectError", + "ConnectTimeout", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "TimeoutException", + "WriteError", + "WriteTimeout", + }: + return True + current = current.__cause__ or current.__context__ + message = str(exc).lower() + return any( + marker in message + for marker in ( + "connection reset", + "operation timed out", + "request timed out", + "temporarily unavailable", + ) + ) + + +def template_map_from_env() -> dict[str, str]: + """Parse SFORGE_E2B_TEMPLATE_MAP: inline ``key=value,key=value`` pairs or + a path to a JSON object file.""" + raw = os.environ.get("SFORGE_E2B_TEMPLATE_MAP", "").strip() + if not raw: + return {} + if raw.endswith(".json"): + try: + return { + str(k): str(v) for k, v in json.loads(Path(raw).read_text()).items() + } + except (OSError, json.JSONDecodeError) as exc: + raise ValueError( + f"Cannot load SFORGE_E2B_TEMPLATE_MAP file {raw!r}: {exc}" + ) from exc + result: dict[str, str] = {} + for item in raw.split(","): + item = item.strip() + if not item: + continue + if "=" not in item: + raise ValueError( + f"Malformed SFORGE_E2B_TEMPLATE_MAP entry {item!r} (expected key=value)" + ) + k, v = item.split("=", 1) + result[k.strip()] = v.strip() + return result diff --git a/sforge/harness/backend/factory.py b/sforge/harness/backend/factory.py index 707d929..7bac954 100644 --- a/sforge/harness/backend/factory.py +++ b/sforge/harness/backend/factory.py @@ -27,6 +27,9 @@ def create_backend( k8s_node_selector: dict[str, str] | None = None, k8s_image_registry: str = "", k8s_kubeconfig: str | None = None, + e2b_template_map: dict[str, str] | None = None, + e2b_template_namespace: str | None = None, + e2b_sandbox_ttl: int | None = None, ) -> ContainerBackend: if name == "docker": from sforge.harness.backend.docker_backend import DockerBackend @@ -41,5 +44,13 @@ def create_backend( image_registry=k8s_image_registry, kubeconfig=k8s_kubeconfig, ) - else: - raise ValueError(f"Unknown backend: {name!r}. Must be 'docker' or 'k8s'.") + elif name == "e2b": + from sforge.harness.backend.e2b_backend import E2BBackend + + return E2BBackend( + template_map=e2b_template_map, + template_namespace=e2b_template_namespace, + sandbox_ttl=e2b_sandbox_ttl, + ) + supported = "docker, k8s, e2b" + raise ValueError(f"Unknown backend: {name!r}. Must be one of: {supported}.") diff --git a/sforge/harness/config.py b/sforge/harness/config.py index c86efd0..7056f87 100644 --- a/sforge/harness/config.py +++ b/sforge/harness/config.py @@ -42,6 +42,12 @@ class SForgeConfig: agent_api_key: str | None = None agent_api_base_url: str | None = None agent_model: str | None = None + # Reasoning effort, canonical vocabulary "low"/"medium"/"high"/"max"; + # each agent translates to its native mechanism (env var, CLI flag or + # config option) and maps aliases (e.g. max -> xhigh). None means the + # agent's own default — existing runs were recorded under that behavior, + # so never default this to an explicit level. + agent_effort: str | None = None agent_timeout: int | None = None agent_extra_env: dict[str, str] = field(default_factory=dict) @@ -69,7 +75,7 @@ class SForgeConfig: # Remote container registry registry: str | None = None - # Container backend: "docker" or "k8s" + # Container backend: "docker", "k8s" or "e2b" backend: str = "docker" # K8s backend settings @@ -77,6 +83,12 @@ class SForgeConfig: k8s_node_selector: dict[str, str] = field(default_factory=dict) k8s_image_registry: str = "" k8s_kubeconfig: str | None = None + + # e2b backend settings (endpoint/credentials come from the standard + # E2B_API_KEY / E2B_API_URL / E2B_DOMAIN env vars read by the e2b SDK) + e2b_template_map: dict[str, str] = field(default_factory=dict) + e2b_template_namespace: str | None = None + e2b_sandbox_ttl: int | None = None # Paths log_dir: Path = field(default_factory=lambda: LOG_DIR) tasks_dir: Path = field(default_factory=lambda: TASKS_DIR) @@ -100,6 +112,7 @@ def load_config(cli_overrides: dict | None = None) -> SForgeConfig: "agent_api_key": ["SFORGE_AGENT_API_KEY"], "agent_api_base_url": ["SFORGE_AGENT_API_BASE_URL"], "agent_model": ["SFORGE_AGENT_MODEL"], + "agent_effort": ["SFORGE_AGENT_EFFORT"], "agent_timeout": ["SFORGE_AGENT_TIMEOUT"], "git_user": ["SFORGE_GIT_USER"], "git_token": ["SFORGE_GIT_TOKEN"], @@ -108,6 +121,7 @@ def load_config(cli_overrides: dict | None = None) -> SForgeConfig: "k8s_namespace": ["SFORGE_K8S_NAMESPACE"], "k8s_image_registry": ["SFORGE_K8S_IMAGE_REGISTRY"], "k8s_kubeconfig": ["SFORGE_K8S_KUBECONFIG"], + "e2b_template_namespace": ["SFORGE_E2B_TEMPLATE_NAMESPACE"], "work_cpu_limit": ["SFORGE_WORK_CPU_LIMIT"], "work_mem_limit": ["SFORGE_WORK_MEM_LIMIT"], "log_dir": ["SFORGE_LOG_DIR"], @@ -168,10 +182,19 @@ def load_config(cli_overrides: dict | None = None) -> SForgeConfig: k, v = item.split("=", 1) config.k8s_node_selector[k.strip()] = v.strip() + # SFORGE_E2B_TEMPLATE_MAP: "image_key=template,..." or a JSON file path + if os.environ.get("SFORGE_E2B_TEMPLATE_MAP"): + from sforge.harness.backend.e2b_backend import template_map_from_env + config.e2b_template_map = template_map_from_env() + e2b_ttl = os.environ.get("SFORGE_E2B_SANDBOX_TTL") + if e2b_ttl: + config.e2b_sandbox_ttl = int(e2b_ttl) # Override from CLI flags if cli_overrides: for k, v in cli_overrides.items(): if v is not None and hasattr(config, k): + if k in ("log_dir", "tasks_dir"): + v = Path(v) setattr(config, k, v) return config @@ -267,6 +290,26 @@ def get_container_env(config: SForgeConfig, include_judge_extra: bool = False) - return env +def sanitize_e2b_proxy_env( + env: dict[str, str], *, internet: bool = True, +) -> None: + """Remove host-only proxies before sending an environment to E2B. + + Generic ``HTTP_PROXY``/``HTTPS_PROXY`` values are commonly inherited from + the machine running SForge and may point at private hosts that a remote + Sandbox cannot reach. Only the explicit SForge variants opt a proxy into + E2B; isolated Work Sandboxes never receive one. + """ + proxy_groups = ( + ("SFORGE_HTTP_PROXY", "http_proxy", "HTTP_PROXY"), + ("SFORGE_HTTPS_PROXY", "https_proxy", "HTTPS_PROXY"), + ) + for explicit_name, *runtime_names in proxy_groups: + if not internet or not os.environ.get(explicit_name): + for name in runtime_names: + env.pop(name, None) + + def get_container_resource_kwargs( config_cpu_limit: int | None, config_mem_limit: str | None, @@ -298,4 +341,7 @@ def create_backend_from_config(config: SForgeConfig, docker_client=None): k8s_node_selector=config.k8s_node_selector, k8s_image_registry=config.k8s_image_registry, k8s_kubeconfig=config.k8s_kubeconfig, + e2b_template_map=config.e2b_template_map or None, + e2b_template_namespace=config.e2b_template_namespace, + e2b_sandbox_ttl=config.e2b_sandbox_ttl, ) diff --git a/sforge/harness/constants.py b/sforge/harness/constants.py index e4c0f1c..77a55bc 100644 --- a/sforge/harness/constants.py +++ b/sforge/harness/constants.py @@ -15,7 +15,10 @@ from __future__ import annotations import os +import secrets +import stat from pathlib import Path + from typing_extensions import NotRequired, TypedDict # --- Paths --- @@ -39,10 +42,35 @@ # --- Defaults --- DEFAULT_EVAL_INTERVAL = 300 # seconds -# --- Admin secret for judge server registration --- -# Only known to host-side processes (run_agent.py, judge_server.py). -# Never injected into agent containers. -ADMIN_SECRET = "sEb3nCh!aDm1n#2026-x9Kp7qW4mZq" + +def get_admin_secret(log_dir: Path | None = None) -> str: + """Load or create the host/controller secret without import side effects.""" + configured = os.environ.get("SFORGE_ADMIN_SECRET") + if configured: + return configured + path = (log_dir or LOG_DIR) / ".judge-admin-secret" + path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_RDWR | os.O_CREAT + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + import fcntl # POSIX-only; imported lazily so Windows can still import the package + + fd = os.open(path, flags, 0o600) + with os.fdopen(fd, "r+") as secret_file: + fcntl.flock(secret_file.fileno(), fcntl.LOCK_EX) + mode = os.fstat(secret_file.fileno()).st_mode + if not stat.S_ISREG(mode): + raise RuntimeError(f"Judge admin secret is not a regular file: {path}") + os.fchmod(secret_file.fileno(), 0o600) + secret = secret_file.read().strip() + if not secret: + secret = secrets.token_urlsafe(32) + secret_file.seek(0) + secret_file.write(secret) + secret_file.truncate() + secret_file.flush() + os.fsync(secret_file.fileno()) + return secret # --- Docker --- DOCKER_USER = "root" diff --git a/sforge/harness/experiment.py b/sforge/harness/experiment.py index 8a5de22..de56cde 100644 --- a/sforge/harness/experiment.py +++ b/sforge/harness/experiment.py @@ -63,6 +63,7 @@ class ModelConfig: class TaskOverrides: agent: str | None = None model: str | None = None + effort: str | None = None timeout: int | None = None eval_interval: int | None = None disable_stop_hook: bool | None = None @@ -126,7 +127,7 @@ def _parse_task_overrides(data: dict | None) -> TaskOverrides: if not data: return TaskOverrides() ovr = TaskOverrides() - for key in ("agent", "model"): + for key in ("agent", "model", "effort"): val = data.get(key) if val is not None: setattr(ovr, key, str(val)) @@ -217,6 +218,9 @@ def resolve_task_overrides( merged = TaskOverrides( agent=task_ovr.agent if task_ovr.agent is not None else defaults.agent, model=task_ovr.model if task_ovr.model is not None else defaults.model, + effort=( + task_ovr.effort if task_ovr.effort is not None else defaults.effort + ), timeout=( task_ovr.timeout if task_ovr.timeout is not None else defaults.timeout ), diff --git a/sforge/harness/game_server_app.py b/sforge/harness/game_server_app.py index 8bc54b7..ba5dc63 100644 --- a/sforge/harness/game_server_app.py +++ b/sforge/harness/game_server_app.py @@ -24,11 +24,14 @@ from __future__ import annotations import argparse +import hmac +import os import sys import threading import uvicorn -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse from pydantic import BaseModel try: @@ -78,6 +81,17 @@ def create_app(rom_path: str) -> FastAPI: app = FastAPI(title="SForge Game Server (Container)") state = GameState(rom_path) + # On backends where this port is exposed through a public gateway (e2b), + # only the judge server knows the per-session token it set in our env. + token = os.environ.get("SFORGE_GAME_TOKEN", "") + if token: + @app.middleware("http") + async def require_token(request: Request, call_next): + got = request.headers.get("x-sforge-game-token", "") + if not hmac.compare_digest(got, token): + return JSONResponse({"detail": "unauthorized"}, status_code=401) + return await call_next(request) + @app.get("/health") def health() -> dict: return {"ok": True} diff --git a/sforge/harness/judge_server.py b/sforge/harness/judge_server.py index 0135830..ad09e91 100644 --- a/sforge/harness/judge_server.py +++ b/sforge/harness/judge_server.py @@ -32,12 +32,18 @@ from pydantic import BaseModel from sforge.harness.backend import ContainerBackend -from sforge.harness.config import SForgeConfig, create_backend_from_config, get_container_env, load_config -from sforge.harness.constants import ADMIN_SECRET +from sforge.harness.benchmark import load_benchmark +from sforge.harness.config import ( + SForgeConfig, + create_backend_from_config, + get_container_env, + load_config, + sanitize_e2b_proxy_env, +) +from sforge.harness.constants import get_admin_secret from sforge.harness.docker_build import BuildImageError, build_judge_image from sforge.harness.run_evaluation import judge_submission from sforge.harness.selection import select_best -from sforge.harness.benchmark import load_benchmark from sforge.harness.task_spec import TaskSpec, load_all_tasks logger = logging.getLogger("sforge.judge_server") @@ -128,6 +134,9 @@ class RegisterRequest(BaseModel): k8s_kubeconfig: str | None = None max_agent_submissions: int | None = None submission_cooldown: int | None = None + e2b_template_map: dict[str, str] | None = None + e2b_template_namespace: str | None = None + e2b_sandbox_ttl: int | None = None class RegisterResponse(BaseModel): @@ -150,6 +159,8 @@ class SubmitResponse(BaseModel): GAME_CONTAINER_READY_TIMEOUT = 30 GAME_CONTAINER_READY_POLL = 0.5 GAME_CONTAINER_PORT = 8000 +GAME_GATEWAY_ATTEMPTS = 3 +GAME_GATEWAY_BACKOFF = 1.0 @dataclass @@ -160,6 +171,7 @@ class GameSessionState: game_num: int container: object # ContainerHandle container_url: str + headers: dict = field(default_factory=dict) max_score: int = 0 peak_score: int = 0 current_score: int = 0 @@ -176,13 +188,14 @@ class GameSessionState: class JudgeState: def __init__(self, config: SForgeConfig): self.config = config + self.admin_secret = get_admin_secret(config.log_dir) self.tasks: dict[str, TaskSpec] = {} self.submissions: dict[str, dict] = {} # submission_id -> {status, report, ...} self.game_sessions: dict[str, GameSessionState] = {} self.run_history: dict[str, list[dict]] = {} # run_id -> [entries] self.tokens: dict[str, dict] = {} # token -> {task_id, run_id, next_agent, next_auto, judge_cpu_limit, judge_mem_limit} - self.run_resource_limits: dict[str, dict] = {} # run_id -> {judge_cpu_limit, judge_mem_limit} - self.run_backends: dict[str, ContainerBackend] = {} # run_id -> backend + self.run_resource_limits: dict[str, dict] = {} # run/task -> limits + self.run_backends: dict[str, ContainerBackend] = {} # run/task -> backend self._game_lock = threading.Lock() self._history_lock = threading.Lock() self._tokens_lock = threading.Lock() @@ -214,12 +227,15 @@ def _destroy_game_session(self, session_id: str) -> None: sess = self.game_sessions.pop(session_id, None) if sess is None: return - self._archive_game_session(sess) try: - backend = self._get_backend(sess.run_id) + backend = self._get_backend(sess.run_id, sess.task_id) backend.cleanup_container(sess.container, logger) except Exception: logger.exception("Error cleaning up game container for session %s", session_id) + with self._game_lock: + self.game_sessions.setdefault(session_id, sess) + return + self._archive_game_session(sess) def _game_log_dir(self, sess: GameSessionState) -> Path: return ( @@ -270,7 +286,10 @@ def register_session(self, task_id: str, run_id: str, k8s_node_selector: dict[str, str] | None = None, k8s_kubeconfig: str | None = None, max_agent_submissions: int | None = None, - submission_cooldown: int | None = None) -> str: + submission_cooldown: int | None = None, + e2b_template_map: dict[str, str] | None = None, + e2b_template_namespace: str | None = None, + e2b_sandbox_ttl: int | None = None) -> str: if task_id not in self.tasks: raise ValueError(f"Unknown task: {task_id}") token = secrets.token_hex(16) @@ -286,25 +305,61 @@ def register_session(self, task_id: str, run_id: str, "submission_cooldown": submission_cooldown, "last_agent_submit_at": 0.0, } - self.run_resource_limits[run_id] = { + run_key = self._run_key(run_id, task_id) + self.run_resource_limits[run_key] = { "judge_cpu_limit": judge_cpu_limit, "judge_mem_limit": judge_mem_limit, } - if backend and backend != self.backend.backend_name: + effective_e2b_template_map = ( + self.config.e2b_template_map + if not e2b_template_map + else e2b_template_map + ) + effective_e2b_template_namespace = ( + self.config.e2b_template_namespace + if not e2b_template_namespace + else e2b_template_namespace + ) + effective_e2b_sandbox_ttl = ( + self.config.e2b_sandbox_ttl + if e2b_sandbox_ttl is None + else e2b_sandbox_ttl + ) + e2b_config_changed = backend == "e2b" and ( + effective_e2b_template_map != self.config.e2b_template_map + or effective_e2b_template_namespace + != self.config.e2b_template_namespace + or effective_e2b_sandbox_ttl != self.config.e2b_sandbox_ttl + ) + if backend and ( + backend != self.backend.backend_name or e2b_config_changed + ): from sforge.harness.backend.factory import create_backend - self.run_backends[run_id] = create_backend( + self.run_backends[run_key] = create_backend( backend, k8s_namespace=k8s_namespace or self.config.k8s_namespace, k8s_node_selector=k8s_node_selector or self.config.k8s_node_selector, k8s_image_registry=k8s_image_registry or self.config.k8s_image_registry, k8s_kubeconfig=k8s_kubeconfig or self.config.k8s_kubeconfig, + e2b_template_map=effective_e2b_template_map, + e2b_template_namespace=effective_e2b_template_namespace, + e2b_sandbox_ttl=effective_e2b_sandbox_ttl, ) return token - def _get_backend(self, run_id: str | None = None) -> ContainerBackend: - if run_id and run_id in self.run_backends: - return self.run_backends[run_id] + @staticmethod + def _run_key(run_id: str, task_id: str) -> str: + return f"{run_id}/{task_id}" + + def _get_backend( + self, run_id: str | None = None, task_id: str | None = None, + ) -> ContainerBackend: + if run_id and task_id: + backend = self.run_backends.get(self._run_key(run_id, task_id)) + if backend is not None: + return backend return self.backend + def resolve_token(self, token: str) -> dict: with self._tokens_lock: info = self.tokens.get(token) @@ -467,7 +522,7 @@ def _grade_worker(self, submission_id: str, task_id: str, archive: bytes, task_spec=task_spec, archive=archive, config=config, - backend=self._get_backend(run_id), + backend=self._get_backend(run_id, task_id), submission_id=submission_id, log_dir=sub_log_dir, ) @@ -514,7 +569,7 @@ def create_game_session(self, run_id: str, task_id: str) -> tuple[GameSessionSta logger.info("Evicting oldest game session %s for %s/%s", sess.session_id, run_id, task_id) self._destroy_game_session(sess.session_id) - backend = self._get_backend(run_id) + backend = self._get_backend(run_id, task_id) if backend.backend_name == "docker": from sforge.harness.backend.docker_backend import DockerBackend @@ -525,8 +580,16 @@ def create_game_session(self, run_id: str, task_id: str) -> tuple[GameSessionSta container_name = f"sforge.game.{task_id}.{session_id}" port = GAME_CONTAINER_PORT env = get_container_env(self.config, include_judge_extra=True) - - rl = self.run_resource_limits.get(run_id, {}) + game_headers = {} + if backend.backend_name == "e2b": + sanitize_e2b_proxy_env(env) + # E2B exposes the service through a public gateway. Protect the + # in-sandbox app as well as the outer Judge route. + game_token = uuid.uuid4().hex + env["SFORGE_GAME_TOKEN"] = game_token + game_headers["X-SForge-Game-Token"] = game_token + + rl = self.run_resource_limits.get(self._run_key(run_id, task_id), {}) cpu = rl.get("judge_cpu_limit") mem = rl.get("judge_mem_limit") # fall back to task defaults @@ -534,76 +597,108 @@ def create_game_session(self, run_id: str, task_id: str) -> tuple[GameSessionSta cpu = task_spec.judge.cpu_limit if mem is None: mem = task_spec.judge.mem_limit - handle = backend.create_container( - task_spec.judge_image_key, - container_name, - environment=env, - cpu_limit=cpu, - mem_limit=mem, - ) - backend.start_container(handle) - logger.info("Game container started: %s (%s)", container_name, handle.id[:12]) - - backend.copy_to_container(handle, GAME_SERVER_APP_PATH, PurePosixPath("/tmp/game_server_app.py")) - - backend.exec_run( - handle, - f"/bin/bash -c '{task_spec.judge.game_server_cmd} &'", - detach=True, - ) + handle = None + committed = False + try: + handle = backend.create_container( + task_spec.judge_image_key, + container_name, + environment=env, + cpu_limit=cpu, + mem_limit=mem, + ) + backend.start_container(handle) + logger.info( + "Game container started: %s (%s)", container_name, handle.id[:12], + ) - container_ip = backend.get_container_ip(handle) - if not container_ip: - backend.cleanup_container(handle, logger) - raise RuntimeError("Cannot determine container IP address") + backend.copy_to_container( + handle, GAME_SERVER_APP_PATH, + PurePosixPath("/tmp/game_server_app.py"), + ) + backend.exec_run( + handle, f"/bin/bash -c '{task_spec.judge.game_server_cmd} &'", + detach=True, + ) - container_url = f"http://{container_ip}:{port}" + endpoint = backend.get_service_endpoint(handle, port) + container_url = endpoint.url + game_headers = {**endpoint.headers, **game_headers} - deadline = time.time() + GAME_CONTAINER_READY_TIMEOUT - while time.time() < deadline: - try: - resp = requests.get(f"{container_url}/health", timeout=2) - if resp.status_code == 200: + deadline = time.time() + GAME_CONTAINER_READY_TIMEOUT + while time.time() < deadline: + try: + resp = requests.get( + f"{container_url}/health", headers=game_headers, timeout=5, + ) + if resp.status_code == 200: + break + except requests.RequestException: + pass + time.sleep(GAME_CONTAINER_READY_POLL) + else: + raise RuntimeError("Game container did not become ready in time") + + gateway_attempts = ( + GAME_GATEWAY_ATTEMPTS if backend.backend_name == "e2b" else 1 + ) + for attempt in range(1, gateway_attempts + 1): + try: + resp = requests.post( + f"{container_url}/new", json={}, headers=game_headers, + timeout=10, + ) + resp.raise_for_status() + data = resp.json() break - except requests.ConnectionError: - pass - time.sleep(GAME_CONTAINER_READY_POLL) - else: - backend.cleanup_container(handle, logger) - raise RuntimeError("Game container did not become ready in time") - - resp = requests.post( - f"{container_url}/new", - json={}, - timeout=10, - ) - resp.raise_for_status() - data = resp.json() - - with self._history_lock: - history_key = f"{run_id}/{task_id}" - archived = sum(1 for e in self.run_history.get(history_key, []) if e.get("type") == "game") - with self._game_lock: - active = sum(1 for s in self.game_sessions.values() if s.run_id == run_id) - game_num = archived + active + 1 - - sess = GameSessionState( - session_id=session_id, - run_id=run_id, - task_id=task_id, - game_num=game_num, - container=handle, - container_url=container_url, - max_score=data.get("max_score", 0), - peak_score=data.get("peak_score", 0), - current_score=data.get("score", 0), - moves=data.get("moves", 0), - done=data.get("done", False), - ) - with self._game_lock: - self.game_sessions[session_id] = sess - - return sess, data + except (requests.RequestException, ValueError) as exc: + if attempt == gateway_attempts: + raise RuntimeError( + "Game gateway failed while creating a session after " + f"{gateway_attempts} attempts: {exc}" + ) from exc + time.sleep(GAME_GATEWAY_BACKOFF * attempt) + + with self._history_lock: + history_key = f"{run_id}/{task_id}" + archived = sum( + 1 for e in self.run_history.get(history_key, []) + if e.get("type") == "game" + ) + with self._game_lock: + active = sum( + 1 for s in self.game_sessions.values() if s.run_id == run_id + ) + game_num = archived + active + 1 + + sess = GameSessionState( + session_id=session_id, + run_id=run_id, + task_id=task_id, + game_num=game_num, + container=handle, + container_url=container_url, + headers=game_headers, + max_score=data.get("max_score", 0), + peak_score=data.get("peak_score", 0), + current_score=data.get("score", 0), + moves=data.get("moves", 0), + done=data.get("done", False), + ) + with self._game_lock: + self.game_sessions[session_id] = sess + committed = True + return sess, data + except Exception as original_error: + if handle is not None and not committed: + try: + backend.cleanup_container(handle, logger) + except Exception as cleanup_error: + raise RuntimeError( + "Game container initialization failed and cleanup also " + f"failed: {cleanup_error}" + ) from original_error + raise def game_step(self, session_id: str, action: str) -> dict: with self._game_lock: @@ -614,6 +709,7 @@ def game_step(self, session_id: str, action: str) -> dict: resp = requests.post( f"{sess.container_url}/step", json={"action": action}, + headers=sess.headers, timeout=10, ) resp.raise_for_status() @@ -651,12 +747,24 @@ def _auto_close_session(self, session_id: str) -> None: sess = self.game_sessions.pop(session_id, None) if sess is None: return - self._archive_game_session(sess) + backend = self._get_backend(sess.run_id, sess.task_id) + if backend.backend_name != "e2b": + self._archive_game_session(sess) + try: + backend.cleanup_container(sess.container, logger) + except Exception: + logger.exception( + "Error auto-closing game session %s", session_id, + ) + return try: - backend = self._get_backend(sess.run_id) backend.cleanup_container(sess.container, logger) except Exception: logger.exception("Error auto-closing game session %s", session_id) + with self._game_lock: + self.game_sessions.setdefault(session_id, sess) + return + self._archive_game_session(sess) logger.info("Auto-closed done game session %s (max_score=%d)", session_id, sess.max_score) def game_status(self, session_id: str) -> dict: @@ -665,7 +773,7 @@ def game_status(self, session_id: str) -> dict: if sess is None: raise KeyError(f"Game session not found: {session_id}") - resp = requests.get(f"{sess.container_url}/status", timeout=5) + resp = requests.get(f"{sess.container_url}/status", headers=sess.headers, timeout=5) resp.raise_for_status() data = resp.json() @@ -685,7 +793,7 @@ def close_game_session(self, session_id: str) -> dict: raise KeyError(f"Game session not found: {session_id}") try: - resp = requests.post(f"{sess.container_url}/close", timeout=10) + resp = requests.post(f"{sess.container_url}/close", headers=sess.headers, timeout=10) resp.raise_for_status() data = resp.json() except Exception: @@ -695,13 +803,24 @@ def close_game_session(self, session_id: str) -> dict: "max_score": sess.max_score, "moves": sess.moves, } - finally: + backend = self._get_backend(sess.run_id, sess.task_id) + if backend.backend_name != "e2b": self._archive_game_session(sess) try: - backend = self._get_backend(sess.run_id) backend.cleanup_container(sess.container, logger) except Exception: - logger.exception("Error cleaning up game container for session %s", session_id) + logger.exception( + "Error cleaning up game container for session %s", + session_id, + ) + return data + try: + backend.cleanup_container(sess.container, logger) + except Exception: + with self._game_lock: + self.game_sessions.setdefault(session_id, sess) + raise + self._archive_game_session(sess) return data @@ -711,18 +830,37 @@ def close_all_game_sessions(self, run_id: str, task_id: str) -> int: (sid, sess) for sid, sess in self.game_sessions.items() if sess.run_id == run_id and sess.task_id == task_id ] - # Phase 1: archive all sessions synchronously (fast — JSON write + dict - # append). This is what later /history calls will read, so it must - # complete before we return. + backend = self._get_backend(run_id, task_id) + if backend.backend_name == "e2b": + with self._game_lock: + for sid, _ in sessions: + self.game_sessions.pop(sid, None) + failures = [] + for sid, sess in sessions: + try: + backend.cleanup_container(sess.container, logger) + except Exception as exc: + logger.exception( + "Error cleaning up game container for session %s", sid, + ) + failures.append((sid, exc)) + with self._game_lock: + self.game_sessions.setdefault(sid, sess) + continue + self._archive_game_session(sess) + if failures: + detail = ", ".join(f"{sid}: {exc}" for sid, exc in failures) + raise RuntimeError(f"Failed to clean up game containers: {detail}") + return len(sessions) + + # Preserve the existing non-blocking Docker/K8s cleanup behavior. containers_to_cleanup = [] for sid, sess in sessions: with self._game_lock: self.game_sessions.pop(sid, None) self._archive_game_session(sess) containers_to_cleanup.append((sid, sess.container)) - # Phase 2: docker cleanup is slow (stop+rm per container). Run it in - # a background thread so we can return to the caller immediately. - backend = self._get_backend(run_id) + def _bg_cleanup(): for sid, container in containers_to_cleanup: try: @@ -761,7 +899,7 @@ def get_result(submission_id: str) -> ResultResponse: @app.post("/api/v1/register") def register(req: RegisterRequest) -> RegisterResponse: """Register a session and get a token for submissions.""" - if req.admin_secret != ADMIN_SECRET: + if req.admin_secret != state.admin_secret: raise HTTPException(status_code=403, detail="Invalid admin secret") try: token = state.register_session( @@ -775,6 +913,9 @@ def register(req: RegisterRequest) -> RegisterResponse: k8s_kubeconfig=req.k8s_kubeconfig, max_agent_submissions=req.max_agent_submissions, submission_cooldown=req.submission_cooldown, + e2b_template_map=req.e2b_template_map, + e2b_template_namespace=req.e2b_template_namespace, + e2b_sandbox_ttl=req.e2b_sandbox_ttl, ) return RegisterResponse(token=token) except ValueError as e: @@ -794,7 +935,7 @@ def submit( """ if kind not in ("agent", "auto"): raise HTTPException(status_code=400, detail="kind must be 'agent' or 'auto'") - if kind == "auto" and admin_secret != ADMIN_SECRET: + if kind == "auto" and admin_secret != state.admin_secret: raise HTTPException(status_code=403, detail="admin_secret required for auto submissions") try: task_id, run_id, round_id, judge_cpu_limit, judge_mem_limit, remaining = state.consume_round(token, kind) @@ -830,7 +971,7 @@ def history(token: str = Query(...), admin_secret: str = Query("")) -> dict: except KeyError: raise HTTPException(status_code=401, detail="Invalid token") full = state.get_run_history(info["run_id"], info["task_id"]) - if admin_secret == ADMIN_SECRET: + if admin_secret == state.admin_secret: return full # Filter to agent-only view: hide auto-eval entries agent_entries = [ diff --git a/sforge/harness/network_isolation.py b/sforge/harness/network_isolation.py index 5a65a3f..b66f7cd 100644 --- a/sforge/harness/network_isolation.py +++ b/sforge/harness/network_isolation.py @@ -278,6 +278,8 @@ def build_allowed_endpoints( api_url: str | None, gateway_ip: str, logger: logging.Logger, + *, + resolve_hostnames: bool = True, ) -> list[AllowedEndpoint]: """Build the whitelist of TCP endpoints from judge + API URLs.""" endpoints: list[AllowedEndpoint] = [] @@ -289,6 +291,10 @@ def build_allowed_endpoints( endpoints.append(AllowedEndpoint(ip=gateway_ip, port=judge_port)) elif is_ip_address(judge_host): endpoints.append(AllowedEndpoint(ip=judge_host, port=judge_port)) + elif not resolve_hostnames: + endpoints.append( + AllowedEndpoint(ip="", port=judge_port, hostname=judge_host) + ) else: for ip in resolve_hostname(judge_host, logger): endpoints.append(AllowedEndpoint(ip=ip, port=judge_port, hostname=judge_host)) @@ -301,6 +307,10 @@ def build_allowed_endpoints( endpoints.append(AllowedEndpoint(ip=gateway_ip, port=api_port)) elif is_ip_address(api_host): endpoints.append(AllowedEndpoint(ip=api_host, port=api_port)) + elif not resolve_hostnames: + endpoints.append( + AllowedEndpoint(ip="", port=api_port, hostname=api_host) + ) else: for ip in resolve_hostname(api_host, logger): endpoints.append( diff --git a/sforge/harness/run_agent.py b/sforge/harness/run_agent.py index af4ff72..270e9c4 100644 --- a/sforge/harness/run_agent.py +++ b/sforge/harness/run_agent.py @@ -30,12 +30,13 @@ import io import json -import signal import shlex +import signal import tarfile as _tarfile import threading import time import traceback +import uuid from dataclasses import dataclass from pathlib import Path, PurePosixPath from urllib.parse import urlparse @@ -44,8 +45,8 @@ from sforge.harness.agent import Agent from sforge.harness.backend import ContainerBackend, ContainerHandle -from sforge.harness.config import SForgeConfig -from sforge.harness.constants import ADMIN_SECRET +from sforge.harness.config import SForgeConfig, sanitize_e2b_proxy_env +from sforge.harness.constants import get_admin_secret from sforge.harness.docker_build import ( close_logger, setup_logger, @@ -135,6 +136,21 @@ def _build_agent_env( return env + +def _e2b_install_env(env: dict[str, str], agent: Agent) -> dict[str, str]: + """Keep only non-secret values required while installing the agent.""" + allowed = { + "http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", + "no_proxy", "NO_PROXY", "SFORGE_NODEJS_MIRROR_URL", + "npm_config_registry", + } + if agent.api_base_env: + allowed.add(agent.api_base_env) + if agent.model_env: + allowed.add(agent.model_env) + return {key: value for key, value in env.items() if key in allowed} + + # --------------------------------------------------------------------------- # Container setup helpers # --------------------------------------------------------------------------- @@ -172,21 +188,66 @@ def _extract_archive_from_container( backend: ContainerBackend, handle: ContainerHandle, task_spec: TaskSpec, + *, + shutdown_event: threading.Event | None = None, ) -> bytes: """Extract the current submission archive from a running work container.""" - patch_dir = task_spec.cwd - submit_paths = " ".join(task_spec.submit_paths) - excludes = " ".join(f"--exclude={e}" for e in task_spec.submit_exclude) - tar_cmd = ( - f"cd {patch_dir} && tar czf /tmp/final.tar.gz " - f"--exclude=.git {excludes} {submit_paths}" + remote_archive = PurePosixPath( + f"/tmp/.sforge-submission-{uuid.uuid4().hex}.tar.gz" + ) + path_checks = "\n".join( + "if [[ -e {path} || -L {path} ]]; then " + "submit_paths+=({path}); fi".format(path=shlex.quote(path)) + for path in task_spec.submit_paths ) - backend.exec_run(handle, ["/bin/bash", "-c", tar_cmd]) - raw = backend.copy_from_container(handle, PurePosixPath("/tmp/final.tar.gz")) - outer = _tarfile.open(fileobj=io.BytesIO(raw)) - member = outer.getmembers()[0] - archive = outer.extractfile(member).read() - outer.close() + excludes = " ".join( + shlex.quote(f"--exclude={pattern}") + for pattern in task_spec.submit_exclude + ) + tar_cmd = f""" +cd -- {shlex.quote(task_spec.cwd)} || exit $? +submit_paths=() +{path_checks} +if (( ${{#submit_paths[@]}} )); then + tar czf {shlex.quote(str(remote_archive))} --exclude=.git {excludes} -- "${{submit_paths[@]}}" +else + tar czf {shlex.quote(str(remote_archive))} --files-from /dev/null +fi +""" + try: + result = backend.exec_run_with_exit_code( + handle, ["/bin/bash", "-c", tar_cmd], timeout=300, + ) + if result.timed_out: + raise RuntimeError("Timed out while creating the submission archive") + if result.exit_code != 0: + raise RuntimeError( + "Could not create the submission archive: " + + result.output[-4000:] + ) + if backend.backend_name == "e2b": + raw = backend.copy_from_container( + handle, remote_archive, shutdown_event=shutdown_event, + ) + else: + raw = backend.copy_from_container(handle, remote_archive) + finally: + try: + backend.exec_run( + handle, ["rm", "-f", str(remote_archive)], user="root", + ) + except Exception: + pass + with _tarfile.open(fileobj=io.BytesIO(raw)) as outer: + members = [member for member in outer.getmembers() if member.isfile()] + if len(members) != 1: + raise RuntimeError( + f"Unexpected archive wrapper: expected 1 file, got {len(members)}" + ) + source = outer.extractfile(members[0]) + if source is None: + raise RuntimeError("Could not read the submission archive wrapper") + archive = source.read() return archive @@ -200,6 +261,7 @@ def _auto_eval_loop( stop_event: threading.Event, logger, log_dir: Path, + admin_secret: str, ) -> None: """Host-side auto-eval: periodically extract code and submit to judge. @@ -212,13 +274,17 @@ def _auto_eval_loop( if stop_event.is_set(): break try: - archive = _extract_archive_from_container(backend, handle, task_spec) + archive = _extract_archive_from_container( + backend, handle, task_spec, shutdown_event=stop_event, + ) + if stop_event.is_set(): + break resp = requests.post( f"{host_judge_url}/api/v1/submit", data={ "token": session_token, "kind": "auto", - "admin_secret": ADMIN_SECRET, + "admin_secret": admin_secret, }, files={"archive": ("archive.tar.gz", archive, "application/gzip")}, timeout=120, @@ -294,10 +360,11 @@ def run_agent( handle = None net_isolation = None - api_proxy = None install_parts: list[str] = [] + admin_secret = "" try: + admin_secret = get_admin_secret(config.log_dir) # 0. Clean up stale iptables chains from previous runs that were killed if backend.backend_name == "docker": from sforge.harness.network_isolation import cleanup_stale_chains @@ -306,13 +373,20 @@ def run_agent( # 1. Check images exist (must run `sforge build` first) for image_key in (task_spec.work_image_key, task_spec.judge_image_key): if not backend.image_exists(image_key): + if backend.backend_name == "e2b": + raise RuntimeError( + f"E2B Template for {image_key!r} was not found. Make sure " + "the task uses a published image version with an official " + "Template and SFORGE_E2B_TEMPLATE_NAMESPACE is set, or " + "point SFORGE_E2B_TEMPLATE_MAP at a self-built Template." + ) raise RuntimeError( f"Image '{image_key}' not found. Run `sforge pull --task {task_spec.task_id}` to fetch from registry, or `sforge build --task {task_spec.task_id}` to build locally." ) logger.info("Images ready") # 1b. Register session with judge server - reg_body: dict = {"task_id": task_spec.task_id, "run_id": run_id, "admin_secret": ADMIN_SECRET} + reg_body: dict = {"task_id": task_spec.task_id, "run_id": run_id, "admin_secret": admin_secret} if config.judge_cpu_limit is not None: reg_body["judge_cpu_limit"] = config.judge_cpu_limit if config.judge_mem_limit is not None: @@ -321,7 +395,7 @@ def run_agent( reg_body["max_agent_submissions"] = max_submissions if submission_cooldown is not None: reg_body["submission_cooldown"] = submission_cooldown - if backend.backend_name != "docker": + if backend.backend_name == "k8s": reg_body["backend"] = backend.backend_name reg_body["k8s_image_registry"] = config.k8s_image_registry reg_body["k8s_namespace"] = config.k8s_namespace @@ -329,6 +403,11 @@ def run_agent( reg_body["k8s_node_selector"] = config.k8s_node_selector if config.k8s_kubeconfig: reg_body["k8s_kubeconfig"] = config.k8s_kubeconfig + elif backend.backend_name == "e2b": + reg_body["backend"] = "e2b" + reg_body["e2b_template_map"] = config.e2b_template_map + reg_body["e2b_template_namespace"] = config.e2b_template_namespace + reg_body["e2b_sandbox_ttl"] = config.e2b_sandbox_ttl for _reg_attempt in range(1, 6): try: reg_resp = requests.post( @@ -353,6 +432,8 @@ def run_agent( backend.remove_container_by_name(container_name) env = _build_agent_env(agent, model) + if backend.backend_name == "e2b": + sanitize_e2b_proxy_env(env, internet=internet) env["SFORGE_JUDGE_URL"] = judge_url env["SFORGE_TOKEN"] = session_token env["SFORGE_PATCH_DIR"] = task_spec.cwd @@ -379,11 +460,13 @@ def run_agent( container_extra_hosts = {"host.docker.internal": "host-gateway"} container_cap_drop: list[str] = [] - if not internet: + if not internet and backend.backend_name == "docker": + # Docker-only preflight: isolation runs as host iptables rules, so + # it needs sudo and conflicts with host proxy env leaking into the + # container. Remote backends (k8s, e2b) isolate inside the cluster/ + # provider and are unaffected by either. from sforge.harness.network_isolation import ( check_iptables_permission, - is_ip_address, - resolve_hostname, ) if config.http_proxy or config.https_proxy: @@ -401,10 +484,21 @@ def run_agent( "Ensure passwordless sudo is configured for iptables." ) + if not internet: + from sforge.harness.network_isolation import ( + is_ip_address, + resolve_hostname, + ) + api_url = config.agent_api_base_url or agent.default_api_base_url if api_url: api_host = urlparse(api_url).hostname or "" - if api_host and api_host != "host.docker.internal" and not is_ip_address(api_host): + if ( + backend.backend_name != "e2b" + and api_host + and api_host != "host.docker.internal" + and not is_ip_address(api_host) + ): resolved_ips = resolve_hostname(api_host, logger) if resolved_ips: container_extra_hosts[api_host] = resolved_ips[0] @@ -418,11 +512,17 @@ def run_agent( handle = backend.create_container( task_spec.work_image_key, container_name, - environment=env, + environment=( + _e2b_install_env(env, agent) + if backend.backend_name == "e2b" else env + ), extra_hosts=container_extra_hosts, cap_drop=container_cap_drop or None, cpu_limit=cpu, mem_limit=mem, + # E2B execs default to the SDK's "user" account rather than the + # image USER. Docker and Kubernetes retain the image default. + user="agent" if backend.backend_name == "e2b" else None, ) backend.start_container(handle) logger.info(f"Container started: {container_name} (judge_url={judge_url})") @@ -474,6 +574,7 @@ def run_agent( backend, handle, task_spec, host_judge_url, session_token, effective_eval_interval, auto_eval_stop, logger, log_dir, + admin_secret, ), daemon=True, ) @@ -485,7 +586,6 @@ def run_agent( # 4b. Apply network isolation (after install + tools, before agent) if not internet: from sforge.harness.network_isolation import ( - AllowedEndpoint, build_allowed_endpoints, ) @@ -498,6 +598,7 @@ def run_agent( effective_api_url = config.agent_api_base_url or agent.default_api_base_url endpoints = build_allowed_endpoints( judge_url, effective_api_url, gateway_ip, logger, + resolve_hostnames=backend.backend_name != "e2b", ) net_isolation = backend.create_network_isolation(handle, endpoints, logger) net_isolation.apply() @@ -601,7 +702,9 @@ def run_agent( # 8. Extract final archive (tar of submit_paths) try: - final_archive = _extract_archive_from_container(backend, handle, task_spec) + final_archive = _extract_archive_from_container( + backend, handle, task_spec, + ) (log_dir / "final_archive.tar.gz").write_bytes(final_archive) logger.info(f"Final archive: {len(final_archive)} bytes") except Exception as e: @@ -694,7 +797,7 @@ def run_agent( try: h = requests.get( f"{host_judge_url}/api/v1/history", - params={"token": session_token, "admin_secret": ADMIN_SECRET}, + params={"token": session_token, "admin_secret": admin_secret}, timeout=10, ).json() pending = [e for e in h.get("entries", []) if e.get("status") in ("running", "queued")] @@ -711,7 +814,7 @@ def run_agent( try: history_resp = requests.get( f"{host_judge_url}/api/v1/history", - params={"token": session_token, "admin_secret": ADMIN_SECRET}, + params={"token": session_token, "admin_secret": admin_secret}, timeout=10, ) history_resp.raise_for_status() @@ -772,7 +875,9 @@ def run_agent( interrupted_archive = b"" try: if handle is not None: - interrupted_archive = _extract_archive_from_container(backend, handle, task_spec) + interrupted_archive = _extract_archive_from_container( + backend, handle, task_spec, + ) (log_dir / "final_archive.tar.gz").write_bytes(interrupted_archive) logger.info(f"Final archive (interrupted): {len(interrupted_archive)} bytes") except Exception: @@ -782,7 +887,7 @@ def run_agent( try: history_resp = requests.get( f"{host_judge_url}/api/v1/history", - params={"token": session_token, "admin_secret": ADMIN_SECRET}, + params={"token": session_token, "admin_secret": admin_secret}, timeout=10, ) history_resp.raise_for_status() @@ -827,7 +932,12 @@ def run_agent( ) except Exception as e: logger.error(f"Error: {e}\n{traceback.format_exc()}") + if backend.backend_name == "e2b": + # E2B transport or lifecycle failures must remain visible to the + # caller instead of being reported as a successful zero score. + raise + # Preserve the pre-E2B Docker/Kubernetes behavior. try: history_resp = requests.get( f"{host_judge_url}/api/v1/history?token={session_token}", @@ -860,6 +970,8 @@ def run_agent( resume_count=locals().get("resume_count", 0), ) finally: + if auto_eval_stop is not None: + auto_eval_stop.set() if net_isolation is not None: try: net_isolation.cleanup() diff --git a/sforge/harness/run_evaluation.py b/sforge/harness/run_evaluation.py index 058d895..2625c86 100644 --- a/sforge/harness/run_evaluation.py +++ b/sforge/harness/run_evaluation.py @@ -25,15 +25,17 @@ from pathlib import Path, PurePosixPath from sforge.harness.backend import ContainerBackend -from sforge.harness.config import SForgeConfig, get_container_env -from sforge.harness.constants import ( - DOCKER_USER, +from sforge.harness.config import ( + SForgeConfig, + get_container_env, + sanitize_e2b_proxy_env, ) +from sforge.harness.constants import DOCKER_USER from sforge.harness.docker_build import ( BuildImageError, build_judge_image, - setup_logger, close_logger, + setup_logger, ) from sforge.harness.grading import EvalReport, grade_output from sforge.harness.score_rescale import rescale_score @@ -155,6 +157,8 @@ def judge_submission( # 2. Create + start ephemeral container container_name = f"{task_spec.benchmark_name}.judge.{task_spec.task_id}.{submission_id}" env = get_container_env(config, include_judge_extra=True) + if backend.backend_name == "e2b": + sanitize_e2b_proxy_env(env) env.setdefault("SFORGE_JUDGE_URL", "http://host.docker.internal:8080") # Resolve resource limits: config > task defaults cpu = config.judge_cpu_limit if config.judge_cpu_limit is not None else task_spec.judge.cpu_limit @@ -183,7 +187,10 @@ def judge_submission( # List archive contents list_result = backend.exec_run( handle, - f"tar tzf {tar_path}", + # --quoting-style=literal: GNU tar octal-escapes non-ASCII + # names in listings under some locales, which breaks the + # submit_paths match for CJK filenames + f"tar --quoting-style=literal -tzf {tar_path}", workdir=task_spec.cwd, user=DOCKER_USER, ) @@ -237,10 +244,12 @@ def judge_submission( eval_script_local.write_text(task_spec.eval_script) backend.copy_to_container(handle, eval_script_local, PurePosixPath("/tmp/eval.sh")) - # 5. Run eval script + # 5. Run eval script. workdir matters for eval_cmds that use relative + # paths: docker defaults to the image WORKDIR but envd execs land in + # the user's home, so pin the task cwd explicitly. logger.info("Running eval script...") result = backend.exec_run_with_timeout( - handle, "/bin/bash /tmp/eval.sh", timeout + handle, "/bin/bash /tmp/eval.sh", timeout, workdir=task_spec.cwd ) test_output, timed_out, runtime = result.output, result.timed_out, result.elapsed_seconds @@ -280,5 +289,7 @@ def judge_submission( raw_output=str(e), ) finally: - backend.cleanup_container(handle, logger) - close_logger(logger) + try: + backend.cleanup_container(handle, logger) + finally: + close_logger(logger) diff --git a/sforge/visualizer/parsers/agent_output.py b/sforge/visualizer/parsers/agent_output.py index 29bd7ca..568401f 100644 --- a/sforge/visualizer/parsers/agent_output.py +++ b/sforge/visualizer/parsers/agent_output.py @@ -63,14 +63,19 @@ def summary(self) -> str: cmd = str(inp.get("command", "")) return cmd.splitlines()[0][:160] if cmd else "" if self.name in ("Read", "NotebookEdit"): - return str(inp.get("file_path") or inp.get("notebook_path", "")) + return str( + inp.get("file_path") + or inp.get("filePath") # opencode uses camelCase + or inp.get("notebook_path", "") + ) if self.name == "Edit": - fp = inp.get("file_path", "") - old = len(str(inp.get("old_string", ""))) - new = len(str(inp.get("new_string", ""))) + fp = inp.get("file_path") or inp.get("filePath", "") + old = len(str(inp.get("old_string") or inp.get("oldString", ""))) + new = len(str(inp.get("new_string") or inp.get("newString", ""))) return f"{fp} ({old}→{new} chars)" if self.name == "Write": - return f"{inp.get('file_path', '')} ({len(str(inp.get('content', '')))} chars)" + fp = inp.get("file_path") or inp.get("filePath", "") + return f"{fp} ({len(str(inp.get('content', '')))} chars)" if self.name == "Glob": return str(inp.get("pattern", "")) if self.name == "Grep": @@ -348,6 +353,7 @@ def get_trajectory(path: Path) -> Optional[Trajectory]: Dispatches to the appropriate parser based on the file's first bytes: - Claude Code: stream-json (first line is JSON starting with `{`) + - opencode: JSONL whose events carry a top-level camelCase "sessionID" - OpenAI Codex: plain text starting with "OpenAI Codex" or "Reading additional input from stdin..." """ @@ -372,6 +378,10 @@ def get_trajectory(path: Path) -> Optional[Trajectory]: if "OpenAI Codex" in head or head.lstrip().startswith("Reading additional input"): from sforge.visualizer.parsers.codex_output import parse as _codex_parse traj = _codex_parse(path) + elif head.lstrip().startswith("{") and '"sessionID"' in head: + # opencode events use camelCase sessionID; Claude Code uses session_id. + from sforge.visualizer.parsers.opencode_output import parse as _opencode_parse + traj = _opencode_parse(path) else: traj = parse(path) diff --git a/sforge/visualizer/parsers/opencode_output.py b/sforge/visualizer/parsers/opencode_output.py new file mode 100644 index 0000000..698dce6 --- /dev/null +++ b/sforge/visualizer/parsers/opencode_output.py @@ -0,0 +1,201 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parse opencode ``run --format json`` output into the shared Trajectory model. + +opencode format (one JSON event per line; every event carries a top-level +``sessionID`` and a ``part`` object): + + {"type":"step_start","sessionID":..,"part":{"messageID":..,"snapshot":..}} + {"type":"text", "part":{"id":..,"messageID":..,"text":..,"time":..}} + {"type":"tool_use", "part":{"tool":"bash","callID":.., + "state":{"status","input","output",..}}} + {"type":"step_finish","part":{"reason":"stop"|"tool-calls", + "tokens":{..},"cost":..}} + {"type":"error", "error":{"name":..,"data":{"message":..}}} + +Each assistant step (grouped by ``part.messageID``) becomes one Exchange. +Tool results arrive inline in the ``tool_use`` event's ``state``, so no +cross-event folding is needed. The trailing ``step_finish`` may be missing +when the process exits early — the parser tolerates truncated files. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from sforge.visualizer.parsers.agent_output import ( + Exchange, + ToolCall, + Trajectory, +) + +# opencode tool names are lowercase; map them onto the canonical names the +# shared ToolCall.summary property understands. +_TOOL_NAMES = { + "bash": "Bash", + "read": "Read", + "write": "Write", + "edit": "Edit", + "patch": "Patch", + "glob": "Glob", + "grep": "Grep", + "list": "List", + "task": "Task", + "todowrite": "TodoWrite", + "todoread": "TodoRead", + "webfetch": "WebFetch", + "websearch": "WebSearch", +} + + +def parse(path: Path) -> Trajectory: + traj = Trajectory() + if not path.is_file(): + return traj + + exchanges: list[Exchange] = [] + current: Exchange | None = None + current_message_id: str | None = None + # part.id → position, so re-emitted parts update in place. + text_parts: dict[str, int] = {} + tool_parts: dict[str, ToolCall] = {} + tokens_in = tokens_out = cache_read = cache_write = 0 + cost = 0.0 + + def start_exchange(message_id: str | None) -> Exchange: + nonlocal current, current_message_id + ex = Exchange(idx=len(exchanges), role="assistant") + exchanges.append(ex) + current = ex + current_message_id = message_id + text_parts.clear() + return ex + + def ensure_exchange(message_id: str | None) -> Exchange: + if current is None or ( + message_id and message_id != current_message_id + ): + return start_exchange(message_id) + return current + + with path.open("r", errors="replace") as fh: + for line in fh: + line = line.strip() + if not line or not line.startswith("{"): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + traj.total_events += 1 + + etype = event.get("type", "") + part = event.get("part") or {} + message_id = part.get("messageID") + + if not traj.meta.get("session_id") and event.get("sessionID"): + traj.meta["session_id"] = event["sessionID"] + + if etype == "step_start": + start_exchange(message_id) + + elif etype == "text": + ex = ensure_exchange(message_id) + text = str(part.get("text", "")) + part_id = part.get("id") + blocks = ex.text.split("\n\n") if ex.text else [] + if part_id and part_id in text_parts: + blocks[text_parts[part_id]] = text + else: + if part_id: + text_parts[part_id] = len(blocks) + blocks.append(text) + ex.text = "\n\n".join(blocks) + + elif etype == "tool_use": + ex = ensure_exchange(message_id) + state = part.get("state") or {} + call_id = str(part.get("callID") or part.get("id") or "") + status = state.get("status", "") + raw_name = str(part.get("tool", "")) + tool = tool_parts.get(call_id) + if tool is None: + tool = ToolCall( + id=call_id or f"opencode-tool-{traj.total_events}", + name=_TOOL_NAMES.get(raw_name, raw_name or "tool"), + ) + tool_parts[call_id] = tool + ex.tool_calls.append(tool) + inp = state.get("input") + if isinstance(inp, dict): + tool.input = inp + if status == "error": + tool.result_content = str( + state.get("error") or state.get("output") or "" + ) + tool.result_is_error = True + tool.has_result = True + elif status == "completed": + tool.result_content = str(state.get("output", "")) + tool.has_result = True + + elif etype == "step_finish": + ex = ensure_exchange(message_id) + ex.stop_reason = str(part.get("reason", "")) + tokens = part.get("tokens") or {} + tokens_in += int(tokens.get("input", 0) or 0) + tokens_out += int(tokens.get("output", 0) or 0) + cache = tokens.get("cache") or {} + cache_read += int(cache.get("read", 0) or 0) + cache_write += int(cache.get("write", 0) or 0) + cost += float(part.get("cost", 0) or 0) + + elif etype == "error": + err = event.get("error") or {} + data = err.get("data") or {} + msg = str(data.get("message") or err.get("name") or "error") + ex = Exchange( + idx=len(exchanges), role="system", text=f"Error: {msg}", + ) + exchanges.append(ex) + current = None + current_message_id = None + + traj.meta.update( + { + "agent_cli": "opencode", + "tokens_input": tokens_in, + "tokens_output": tokens_out, + "cache_read": cache_read, + "cache_write": cache_write, + } + ) + if cost: + traj.meta["cost"] = round(cost, 4) + + # Drop empty exchanges left by step_start events with no content. + exchanges = [ + ex for ex in exchanges + if ex.text or ex.thinking or ex.tool_calls or ex.stop_reason + ] + for i, ex in enumerate(exchanges): + ex.idx = i + + traj.exchanges = exchanges + traj.tool_use_count = sum(len(ex.tool_calls) for ex in exchanges) + return traj diff --git a/sforge/visualizer/scanner.py b/sforge/visualizer/scanner.py index 00f5592..4099c9d 100644 --- a/sforge/visualizer/scanner.py +++ b/sforge/visualizer/scanner.py @@ -382,6 +382,8 @@ def _peek_agent(task_dir: Path) -> str: return "claude-code" if "Running agent: codex " in head or "Running agent: codex\n" in head: return "codex" + if "Running agent: opencode " in head or "Running agent: opencode\n" in head: + return "opencode" if "Running agent: aider " in head or "Running agent: aider\n" in head: return "aider" # New harness format: "Agent supervisor attempt N: claude -p ..." @@ -393,6 +395,8 @@ def _peek_agent(task_dir: Path) -> str: return "claude-code" if after_colon.startswith("codex ") or after_colon.startswith("codex\n"): return "codex" + if after_colon.startswith("opencode ") or after_colon.startswith("opencode\n"): + return "opencode" if after_colon.startswith("aider ") or after_colon.startswith("aider\n"): return "aider" return ""