From 492747cb0279f260b6d3dfc11c9f951c5303d333 Mon Sep 17 00:00:00 2001 From: Tao Wang Date: Tue, 25 Aug 2026 16:59:39 +0800 Subject: [PATCH 01/17] feat: add e2b backend support --- .gitignore | 6 + pyproject.toml | 3 + sforge/cli.py | 338 ++++++- sforge/harness/agent/claude_code.py | 7 + sforge/harness/backend/__init__.py | 2 + sforge/harness/backend/base.py | 16 + sforge/harness/backend/e2b_backend.py | 1005 ++++++++++++++++++++ sforge/harness/backend/factory.py | 13 +- sforge/harness/config.py | 40 +- sforge/harness/constants.py | 35 +- sforge/harness/dockerfiles.py | 51 +- sforge/harness/e2b_controller.py | 391 ++++++++ sforge/harness/e2b_templates.py | 1248 +++++++++++++++++++++++++ sforge/harness/evolve_scripts.py | 20 +- sforge/harness/game_server_app.py | 16 +- sforge/harness/judge_client.py | 102 ++ sforge/harness/judge_server.py | 422 ++++++--- sforge/harness/network_isolation.py | 10 + sforge/harness/run_agent.py | 445 +++++++-- sforge/harness/run_evaluation.py | 24 +- sforge/harness/task_spec.py | 14 +- 21 files changed, 3932 insertions(+), 276 deletions(-) create mode 100644 sforge/harness/backend/e2b_backend.py create mode 100644 sforge/harness/e2b_controller.py create mode 100644 sforge/harness/e2b_templates.py create mode 100644 sforge/harness/judge_client.py diff --git a/.gitignore b/.gitignore index 8d042e0..d50df51 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,12 @@ logs/ # Experiments (generated at runtime) experiments/ +# Benchmark task definitions downloaded by sforge fetch-tasks +/tasks/ + +# Local review tests (the upstream release tree does not ship a tests directory) +/tests/ + # IDE .idea/ .vscode/ diff --git a/pyproject.toml b/pyproject.toml index f197ec8..cbfbd82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,9 @@ dependencies = [ "huggingface-hub>=0.20", ] +[project.optional-dependencies] +e2b = ["e2b>=2.42.0"] + [project.scripts] sforge = "sforge.cli:main" diff --git a/sforge/cli.py b/sforge/cli.py index ad46d50..ba0ec1c 100644 --- a/sforge/cli.py +++ b/sforge/cli.py @@ -20,6 +20,7 @@ import concurrent.futures import copy import json +import os import signal import sys import threading @@ -28,19 +29,18 @@ import docker +from sforge.harness.benchmark import load_benchmark from sforge.harness.config import SForgeConfig, create_backend_from_config, load_config from sforge.harness.constants import DEFAULT_EVAL_INTERVAL from sforge.harness.docker_build import ( build_all_images, - build_work_image, - build_judge_image, pull_all_images, push_all_images, ) -from sforge.harness.docker_utils import cleanup_container from sforge.harness.run_evaluation import judge_submission -from sforge.harness.benchmark import load_benchmark -from sforge.harness.task_spec import TaskSpec, make_task_spec, load_all_tasks +from sforge.harness.task_spec import TaskSpec, load_all_tasks, make_task_spec + +E2B_DEFAULT_MAX_WORKERS = 4 def _resolve_task(args, config: SForgeConfig) -> TaskSpec: @@ -91,6 +91,116 @@ def _make_config(args) -> SForgeConfig: return load_config(overrides) +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return parsed + + +def _run_max_workers( + backend_name: str, task_count: int, requested: int | None, +) -> int: + if requested is not None: + return min(task_count, requested) + if backend_name == "e2b": + return min(task_count, E2B_DEFAULT_MAX_WORKERS) + return task_count + + +def _e2b_registry_credentials(portable: bool) -> tuple[str | None, str | None]: + prefix = "SFORGE_PORTABLE_REGISTRY" if portable else "SFORGE_REGISTRY" + return os.environ.get(f"{prefix}_USERNAME"), os.environ.get( + f"{prefix}_PASSWORD" + ) + + +def _build_e2b_task_template( + task_spec: TaskSpec, + role: str, + source_registry: str, + docker_client, + args, +) -> tuple[object, dict]: + """Build one task role, retrying through a portable source if needed.""" + import tempfile + + from e2b import default_build_logger + + from sforge.harness.e2b_templates import ( + AptSourceError, + build_e2b_template, + make_template_manifest, + stage_runtime_artifacts, + ) + + cpu = args.work_cpu if role == "work" else args.judge_cpu + memory = args.work_memory_mb if role == "work" else args.judge_memory_mb + force_portable = False + + for stage_attempt in range(2): + manifest = make_template_manifest( + task_spec, + role, + source_registry, + cpu_count=cpu, + memory_mb=memory, + ) + with tempfile.TemporaryDirectory( + prefix=f"sforge-e2b-{task_spec.task_id}-{role}-" + ) as context_dir: + context_path = Path(context_dir) + source_username, source_password = _e2b_registry_credentials(False) + portable_username, portable_password = _e2b_registry_credentials(True) + manifest, runtime_sources, portable = stage_runtime_artifacts( + manifest, + docker_client, + context_path, + pull_attempts=args.attempts, + portable_registry=args.portable_registry, + source_registry_username=source_username, + source_registry_password=source_password, + portable_registry_username=portable_username, + portable_registry_password=portable_password, + force_portable=force_portable, + ) + print( + f"[{task_spec.task_id}/{role}] {manifest.source_image} " + f"-> {manifest.reference}" + ) + username, password = ( + (portable_username, portable_password) + if portable + else (source_username, source_password) + ) + try: + result = build_e2b_template( + manifest, + attempts=args.attempts, + build_timeout=args.build_timeout, + force=args.force, + on_build_logs=( + None if args.silent else default_build_logger() + ), + registry_username=username, + registry_password=password, + context_dir=context_path, + runtime_sources=runtime_sources, + ) + except AptSourceError: + if stage_attempt or not args.portable_registry or portable: + raise + print( + f"[{task_spec.task_id}/{role}] retrying through a portable " + "source image after E2B could not reach its APT source" + ) + force_portable = True + continue + return manifest, result + + raise RuntimeError("E2B Template build exhausted portable-image retries") + + # --- Commands --- @@ -139,6 +249,57 @@ def _build_one(ts: TaskSpec): print("Done.") +def cmd_e2b_template(args): + """Build deterministic official E2B Templates from task images.""" + config = _make_config(args) + task_specs = _resolve_tasks(args, config) + source_registry = args.source_registry or config.registry + if not source_registry: + print( + "Error: --source-registry or SFORGE_REGISTRY is required; E2B " + "must be able to pull immutable Work/Judge images", + file=sys.stderr, + ) + sys.exit(1) + + from sforge.harness.e2b_templates import write_template_manifest + + docker_client = docker.from_env() + roles = [args.role] if args.role != "both" else ["work", "judge"] + failures = [] + for task_spec in task_specs: + task_failed = False + for role in roles: + if task_failed: + print( + f"[{task_spec.task_id}/{role}] SKIPPED: an earlier role " + "failed for this task", + file=sys.stderr, + ) + continue + try: + manifest, result = _build_e2b_task_template( + task_spec, role, source_registry, docker_client, args, + ) + path = write_template_manifest( + manifest, result, + config.log_dir / "e2b_templates" / task_spec.task_id, + ) + print( + f" {result['status']}: build={result.get('build_id')} " + f"manifest={path}" + ) + except Exception as exc: + failures.append((task_spec.task_id, role, str(exc))) + task_failed = True + print( + f"[{task_spec.task_id}/{role}] FAILED: {exc}", + file=sys.stderr, + ) + if failures: + raise SystemExit(1) + + def cmd_pull(args): """Pull pre-built images from a remote registry (parallel).""" config = _make_config(args) @@ -319,6 +480,7 @@ def _run_single_task( shutdown_event=shutdown_event, max_submissions=getattr(args, "max_submissions", None), submission_cooldown=getattr(args, "submission_cooldown", None), + host_judge_connection=getattr(args, "host_judge_connection", None), ) print(f"\nAgent completed in {result.runtime_seconds:.1f}s") @@ -327,13 +489,15 @@ def _run_single_task( print(f" Game sessions: {result.total_rounds}") if result.best_score is not None: print(f" Best score: {result.best_score:.0f}") + print(f" Game history: {run_log_dir / 'game_history.json'}") else: print(f" Total rounds: {result.total_rounds}") print(f" Best pass rate: {result.best_pass_rate:.2%}") if result.best_score is not None: print(f" Best score: {result.best_score:.0f}") print(f" Best round: {result.best_round}") - print(f" Final archive: {run_log_dir / 'final_archive.tar.gz'}") + if not task_spec.game_mode: + print(f" Final archive: {run_log_dir / 'final_archive.tar.gz'}") combined = { "agent": agent.name, @@ -498,9 +662,17 @@ def _effective_config_dict( if config.agent_api_base_url: d["api_base_url"] = config.agent_api_base_url if config.agent_api_key: - d["api_key"] = config.agent_api_key[:8] + "..." + d["api_key"] = "" if config.agent_extra_env: - d["extra_env"] = config.agent_extra_env + sensitive_markers = ("KEY", "TOKEN", "SECRET", "PASSWORD", "CREDENTIAL") + d["extra_env"] = { + key: ( + "" + if any(marker in key.upper() for marker in sensitive_markers) + else value + ) + for key, value in config.agent_extra_env.items() + } return d @@ -520,7 +692,6 @@ def cmd_run(args): # Apply experiment-level backend/judge_url defaults before creating backend if experiment: - from sforge.harness.experiment import resolve_task_overrides exp_defaults = experiment.defaults if exp_defaults.backend and base_config.backend == "docker" and getattr(args, "backend", None) is None: base_config.backend = exp_defaults.backend @@ -536,9 +707,8 @@ def cmd_run(args): print("Error: --task or --experiment is required") sys.exit(1) - backend = create_backend_from_config(base_config) - run_id = args.run_id or uuid.uuid4().hex[:12] + managed_controller = None multi = len(task_specs) > 1 verbose = not args.silent and not multi @@ -558,22 +728,14 @@ def cmd_run(args): ) task_runs.append((ts, task_config, task_args)) - cfg_dict = _effective_config_dict(ts, task_args, task_config) - unified_tasks[ts.task_id] = cfg_dict - - task_log_dir = run_root / ts.task_id - task_log_dir.mkdir(parents=True, exist_ok=True) - (task_log_dir / "run_config.json").write_text( - json.dumps(cfg_dict, indent=2, ensure_ascii=False) + backend_names = {task_config.backend for _, task_config, _ in task_runs} + if len(backend_names) != 1: + raise ValueError( + "A run must use one container backend; resolved backends: " + + ", ".join(sorted(backend_names)) ) - - unified = { - "run_id": run_id, - "experiment": args.experiment or None, - "stagger": args.stagger or (experiment.stagger if experiment else None), - "tasks": unified_tasks, - } - (run_root / "run_config.json").write_text(json.dumps(unified, indent=2, ensure_ascii=False)) + base_config.backend = backend_names.pop() + backend = create_backend_from_config(base_config) # Resolve stagger: CLI flag wins over experiment YAML stagger = args.stagger @@ -590,7 +752,12 @@ def cmd_run(args): if stagger: print(f" Tasks: {', '.join(t.task_id for t in task_specs)} (staggered over {stagger}s, {stagger_delay:.1f}s apart)") else: - print(f" Tasks: {', '.join(t.task_id for t in task_specs)} (all in parallel)") + print(f" Tasks: {', '.join(t.task_id for t in task_specs)} (parallel)") + max_workers = _run_max_workers( + backend.backend_name, len(task_runs), + getattr(args, "max_workers", None), + ) + print(f" Workers: {max_workers}") print() shutdown_event = threading.Event() @@ -617,8 +784,54 @@ def _invoke(ts: TaskSpec, task_config: SForgeConfig, task_args): summaries: list[dict] = [] try: + managed_tasks = [ + (task_config, task_args) + for _, task_config, task_args in task_runs + if backend.backend_name == "e2b" + and task_args.judge_url == "http://host.docker.internal:8080" + ] + if managed_tasks: + from sforge.harness.e2b_controller import ManagedE2BController + + managed_controller = ManagedE2BController( + backend, base_config, run_id, + ) + connection = managed_controller.start() + access_token = connection.headers.get( + "e2b-traffic-access-token" + ) + for task_config, task_args in managed_tasks: + task_args.judge_url = connection.url + task_config.e2b_judge_access_token = access_token + task_args.host_judge_connection = managed_controller.host_connection + print(f"Managed E2B Judge controller: {connection.url}") + + for ts, task_config, task_args in task_runs: + cfg_dict = _effective_config_dict(ts, task_args, task_config) + unified_tasks[ts.task_id] = cfg_dict + task_log_dir = run_root / ts.task_id + task_log_dir.mkdir(parents=True, exist_ok=True) + (task_log_dir / "run_config.json").write_text( + json.dumps(cfg_dict, indent=2, ensure_ascii=False) + ) + unified = { + "run_id": run_id, + "experiment": args.experiment or None, + "stagger": args.stagger or ( + experiment.stagger if experiment else None + ), + "tasks": unified_tasks, + } + (run_root / "run_config.json").write_text( + json.dumps(unified, indent=2, ensure_ascii=False) + ) + if multi: - with concurrent.futures.ThreadPoolExecutor(max_workers=len(task_runs)) as ex: + max_workers = _run_max_workers( + backend.backend_name, len(task_runs), + getattr(args, "max_workers", None), + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex: futures = [] for i, (ts, tc, ta) in enumerate(task_runs): if i > 0 and stagger_delay > 0: @@ -646,7 +859,19 @@ def _invoke(ts: TaskSpec, task_config: SForgeConfig, task_args): signal.signal(signal.SIGINT, signal.SIG_IGN) print("\nShutting down — stopping containers...") finally: + primary_error = sys.exc_info()[1] signal.signal(signal.SIGINT, old_sigint) + if managed_controller is not None: + try: + managed_controller.close() + except Exception as cleanup_error: + if primary_error is None: + raise + print( + "Managed E2B controller cleanup failed while preserving " + f"the original run error: {cleanup_error}", + file=sys.stderr, + ) if multi: run_root = base_config.log_dir / "runs" / run_id @@ -671,6 +896,8 @@ def _invoke(ts: TaskSpec, task_config: SForgeConfig, task_args): mark = "-" print(f" {tid:<30} {mark}") print(f"\nSummary saved: {summary_path}") + if any("error" in summary for summary in summaries): + raise SystemExit(1) def cmd_eval(args): @@ -743,9 +970,10 @@ def cmd_serve(args): """Start the judge HTTP server.""" config = _make_config(args) - from sforge.harness.judge_server import create_app import uvicorn + from sforge.harness.judge_server import create_app + app = create_app(config) print(f"Starting SForge judge server on port {args.port}") uvicorn.run(app, host=args.host, port=args.port) @@ -787,9 +1015,10 @@ def cmd_proxy(args): def cmd_visualizer(args): """Start the run-results visualizer web UI.""" - from sforge.visualizer.server import create_app as create_viz_app import uvicorn + from sforge.visualizer.server import create_app as create_viz_app + config = _make_config(args) runs_dir = Path(args.runs_dir).resolve() tasks_dir = Path(args.tasks_dir).resolve() if args.tasks_dir else config.tasks_dir @@ -842,6 +1071,7 @@ def _pad(s: str, width: int) -> str: def cmd_fetch_tasks(args): """Download benchmark task definitions from HuggingFace Hub.""" from huggingface_hub import snapshot_download + from sforge.harness.constants import BENCHMARK_REGISTRY, DEFAULT_BENCHMARK benchmark = args.benchmark or DEFAULT_BENCHMARK @@ -907,6 +1137,40 @@ def main(): help="Force rebuild ALL images including base") p_build.set_defaults(func=cmd_build) + # e2b-template + p_e2b_template = subparsers.add_parser( + "e2b-template", + help="Build official E2B Templates from immutable task images", + ) + p_e2b_group = p_e2b_template.add_mutually_exclusive_group(required=True) + p_e2b_group.add_argument("--task", nargs="+") + p_e2b_group.add_argument("--all", action="store_true", default=False) + p_e2b_template.add_argument( + "--source-registry", default=None, + help="Registry prefix containing the Work/Judge images", + ) + p_e2b_template.add_argument( + "--portable-registry", default=None, + help=( + "Registry prefix for sanitized derivatives of images that contain " + "APT sources unreachable from the official E2B build network" + ), + ) + p_e2b_template.add_argument( + "--role", choices=["work", "judge", "both"], default="both", + ) + p_e2b_template.add_argument("--work-cpu", type=_positive_int, default=None) + p_e2b_template.add_argument("--work-memory-mb", type=_positive_int, default=None) + p_e2b_template.add_argument("--judge-cpu", type=_positive_int, default=None) + p_e2b_template.add_argument("--judge-memory-mb", type=_positive_int, default=None) + p_e2b_template.add_argument("--attempts", type=_positive_int, default=3) + p_e2b_template.add_argument( + "--build-timeout", type=_positive_int, default=600, + help="Maximum seconds to wait for each E2B Template build", + ) + p_e2b_template.add_argument("--force", action="store_true", default=False) + p_e2b_template.set_defaults(func=cmd_e2b_template) + # pull p_pull = subparsers.add_parser("pull", help="Pull pre-built images from remote registry") p_pull_group = p_pull.add_mutually_exclusive_group(required=True) @@ -931,11 +1195,12 @@ def main(): # run p_run = subparsers.add_parser("run", help="Run an agent on one or more tasks") - p_run.add_argument("--backend", choices=["docker", "k8s"], default=None, + p_run.add_argument("--backend", choices=["docker", "k8s", "e2b"], default=None, help="Container backend (default from SFORGE_BACKEND or 'docker')") p_run.add_argument("--task", default=None, nargs="+", help="One or more task IDs (e.g. --task ahc056 ahc057). " - "Multiple tasks are run fully in parallel.") + "Multiple tasks are run in parallel, subject to " + "--max-workers.") p_run.add_argument("--experiment", default=None, help="Path to experiment YAML config file (model config + per-task overrides)") p_run.add_argument("--agent", default=None, help="Agent name (claude-code, aider, codex)") @@ -954,6 +1219,13 @@ def main(): help="Minimum seconds between agent submissions (default: no cooldown)") p_run.add_argument("--stagger", type=int, default=None, dest="stagger", help="Spread task launches evenly over N seconds (e.g. --stagger 300)") + p_run.add_argument( + "--max-workers", type=_positive_int, default=None, + help=( + "Maximum parallel tasks (default: all tasks for Docker/Kubernetes; " + f"{E2B_DEFAULT_MAX_WORKERS} for E2B)" + ), + ) p_run.add_argument("--judge-url", default="http://host.docker.internal:8080", help="Judge server URL") p_run.add_argument("--run-id", default=None, help="Run ID for tracking") net_group = p_run.add_mutually_exclusive_group() @@ -980,7 +1252,7 @@ def main(): # eval p_eval = subparsers.add_parser("eval", help="Evaluate an archive") - p_eval.add_argument("--backend", choices=["docker", "k8s"], default=None, + p_eval.add_argument("--backend", choices=["docker", "k8s", "e2b"], default=None, help="Container backend (default from SFORGE_BACKEND or 'docker')") p_eval.add_argument("--task", required=True, help="Task ID") p_eval.add_argument("--archive", required=True, help="Path to .tar.gz archive (or - for stdin)") diff --git a/sforge/harness/agent/claude_code.py b/sforge/harness/agent/claude_code.py index 00c3640..166c16c 100644 --- a/sforge/harness/agent/claude_code.py +++ b/sforge/harness/agent/claude_code.py @@ -110,6 +110,13 @@ def install_stop_hook( local_settings, PurePosixPath("/home/agent/.claude/settings.json"), ) + result = backend.exec_run( + handle, "chown -R agent:agent /home/agent/.claude", user="root", + ) + if result.exit_code != 0: + raise RuntimeError( + f"Failed to set Claude Code config ownership: {result.output}" + ) logger.info("Configured Claude Code settings with hooks") diff --git a/sforge/harness/backend/__init__.py b/sforge/harness/backend/__init__.py index a869252..43dc231 100644 --- a/sforge/harness/backend/__init__.py +++ b/sforge/harness/backend/__init__.py @@ -23,6 +23,7 @@ ContainerHandle, ExecResult, NetworkIsolationStrategy, + ServiceEndpoint, StreamingExecResult, ) from sforge.harness.backend.factory import create_backend @@ -32,6 +33,7 @@ "ContainerHandle", "ExecResult", "NetworkIsolationStrategy", + "ServiceEndpoint", "StreamingExecResult", "create_backend", ] diff --git a/sforge/harness/backend/base.py b/sforge/harness/backend/base.py index 12aa9c1..6b307fb 100644 --- a/sforge/harness/backend/base.py +++ b/sforge/harness/backend/base.py @@ -42,6 +42,14 @@ class StreamingExecResult: elapsed_seconds: float = 0.0 +@dataclass(frozen=True) +class ServiceEndpoint: + """A routable service URL and provider gateway credentials.""" + + url: str + headers: dict[str, str] = field(default_factory=dict) + + class ContainerHandle(abc.ABC): """Opaque handle to a running container or pod.""" @@ -184,6 +192,14 @@ def exec_run_with_exit_code( def get_container_ip(self, handle: ContainerHandle) -> str: ... + def get_service_endpoint( + self, handle: ContainerHandle, port: int, + ) -> ServiceEndpoint: + """Return the URL plus headers needed to cross the provider gateway.""" + return ServiceEndpoint( + url=f"http://{self.get_container_ip(handle)}:{port}", + ) + @abc.abstractmethod def get_container_gateway_ip(self, handle: ContainerHandle) -> str | None: ... diff --git a/sforge/harness/backend/e2b_backend.py b/sforge/harness/backend/e2b_backend.py new file mode 100644 index 0000000..ea902dc --- /dev/null +++ b/sforge/harness/backend/e2b_backend.py @@ -0,0 +1,1005 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""E2B backend: runs work/judge containers on official E2B. + +Targets native E2B semantics (https://e2b.dev): + +- endpoint/credentials via the standard e2b env vars + (``E2B_API_KEY``, ``E2B_API_URL`` / ``E2B_DOMAIN``); +- image-key -> template mapping via ``SFORGE_E2B_TEMPLATE_MAP`` + (see :meth:`E2BBackend.resolve_template`). + +Semantics notes vs the docker backend: + +- Commands run through the sandbox's envd daemon via ``/bin/bash -l -c``, + so string commands get full shell interpretation (docker exec splits + them into argv without a shell). All harness call sites are compatible. +- ``copy_from_container`` preserves the docker contract of returning a + tar archive containing the file. +- stdout/stderr arrive as separate streams; chunks are interleaved in + arrival order, which is close to but not guaranteed identical to the + container's write order. +- cpu/mem limits and cap_drop are template-level concepts in e2b and are + ignored at container creation (logged once). +""" + +from __future__ import annotations + +import io +import json +import logging +import os +import shlex +import tarfile +import threading +import time +import uuid +from collections.abc import Callable +from pathlib import Path, PurePosixPath + +from sforge.harness.backend.base import ( + ContainerBackend, + ContainerHandle, + ExecResult, + NetworkIsolationStrategy, + ServiceEndpoint, + StreamingExecResult, +) + +logger = logging.getLogger(__name__) + +# Sandbox lifetime lease. The renewal thread extends every live sandbox back +# to the full TTL at TTL/3 intervals, so a crashed harness leaks a sandbox for +# at most one TTL. +DEFAULT_SANDBOX_TTL = 3600 + +# Upper bound for the blocking, non-streaming exec_run path. The interface +# carries no per-call timeout there and every caller only runs short +# housekeeping commands (chmod, mkdir, tar list/extract, chown, kill), so a +# generous ceiling turns a hung remote stream into a failure instead of a +# forever-block. The streaming exec paths manage their own deadlines and the +# detached path stays unbounded on purpose so background servers keep running. +DEFAULT_EXEC_TIMEOUT = 600 + +# Files API calls are idempotent at the paths used by the harness. Uploads use +# a bounded request deadline; downloads stream so progress can continue for +# large archives while the SDK still aborts an idle stream. Retry transient +# transport failures, but surface deterministic provider errors immediately. +FILE_TRANSFER_ATTEMPTS = 3 +FILE_TRANSFER_REQUEST_TIMEOUT = 300 +SANDBOX_CREATE_ATTEMPTS = 3 + +_METADATA_NAME_KEY = "sforge_name" + + +class E2BSandboxHandle(ContainerHandle): + """Wraps an e2b Sandbox object.""" + + def __init__( + self, + sandbox, + name: str, + default_user: str | None = None, + ) -> None: + self._sandbox = sandbox + self._name = name + self.default_user = default_user + + @property + def id(self) -> str: + return self._sandbox.sandbox_id + + @property + def name(self) -> str: + return self._name + + @property + def ip_address(self) -> str | None: + return None + + @property + def raw(self): + return self._sandbox + + +class E2BNetworkIsolation(NetworkIsolationStrategy): + """Egress whitelist via the e2b network API. + + Translates the harness's ``AllowedEndpoint`` list into + ``update_network(allow_out=, deny_out=[0.0.0.0/0])`` on the running + sandbox. Allow rules take precedence over deny in e2b, which gives exactly + the whitelist-plus-default-drop semantics supported by the official API. + Port granularity is not supported by e2b selectors: allowing an endpoint + allows all ports on that host. + """ + + def __init__(self, sandbox, allowed_endpoints: list, log: logging.Logger) -> None: + self._sandbox = sandbox + self._endpoints = allowed_endpoints + self._logger = log + + def apply(self) -> None: + hosts: list[str] = [] + for ep in self._endpoints: + host = getattr(ep, "hostname", None) or getattr(ep, "ip", None) + if host and host not in hosts: + hosts.append(host) + try: + self._sandbox.update_network( + {"allow_out": hosts, "deny_out": ["0.0.0.0/0"]} + ) + except Exception as exc: + raise RuntimeError( + "e2b network isolation failed: the provider rejected " + f"update_network ({exc}). Run with --enable-internet to skip " + "isolation, or use an internet-enabled template." + ) from exc + self._logger.info( + "e2b network isolation applied: allow=%s deny=all (host-level, " + "ports are not filtered)", + hosts, + ) + + def cleanup(self) -> None: + # E2B Sandboxes are never reused. Keep the deny rules in place until + # deletion so a transient kill failure cannot leave an unrestricted + # workload alive for the remainder of its TTL. + return None + + +class E2BBackend(ContainerBackend): + """Container backend for the official E2B Sandbox API.""" + + def __init__( + self, + template_map: dict[str, str] | None = None, + sandbox_ttl: int | None = None, + *, + sandbox_cls=None, + template_cls=None, + ) -> None: + if sandbox_cls is None: + try: + from e2b import Sandbox as sandbox_cls + except ImportError as exc: + raise RuntimeError( + "The e2b backend requires the 'e2b' package. " + "Install with: uv sync --extra e2b (or pip install e2b)" + ) from exc + self._sandbox_cls = sandbox_cls + if template_cls is None: + try: + from e2b import Template as template_cls + except ImportError: + template_cls = None + self._template_cls = template_cls + self._template_map = ( + dict(template_map) if template_map else template_map_from_env() + ) + self._sandbox_ttl = ( + int(os.environ.get("SFORGE_E2B_SANDBOX_TTL", DEFAULT_SANDBOX_TTL)) + if sandbox_ttl is None + else sandbox_ttl + ) + if self._sandbox_ttl <= 0: + raise ValueError("E2B Sandbox TTL must be greater than zero") + self._handles: dict[str, E2BSandboxHandle] = {} + self._handles_lock = threading.Lock() + self._renewal_thread: threading.Thread | None = None + self._renewal_wakeup = threading.Event() + self._warned_limits = False + + @property + def backend_name(self) -> str: + return "e2b" + + # --- Template resolution --- + + def resolve_template(self, image_key: str) -> str: + """Map an image key to an e2b template reference. + + Explicit entries in ``SFORGE_E2B_TEMPLATE_MAP`` win (matched with and + without the image tag); otherwise the repo part is normalized to the + template-alias charset and the image tag carries over as the template + tag (``edgebench.work.foo_bar:abc123`` -> ``edgebench-work-foo-bar:abc123``). + Templates built from SForge images must be tagged with the image tag. + """ + if image_key in self._template_map: + return self._template_map[image_key] + repo, sep, tag = image_key.partition(":") + if repo in self._template_map: + return self._template_map[repo] + sep + tag + return repo.lower().replace(".", "-").replace("_", "-") + sep + tag + + # --- Lifecycle --- + + def create_container( + self, + image: str, + name: str, + *, + command: str = "tail -f /dev/null", + environment: dict[str, str] | None = None, + extra_hosts: dict[str, str] | None = None, + cap_drop: list[str] | None = None, + cpu_limit: int | None = None, + mem_limit: str | None = None, + user: str | None = None, + annotations: dict[str, str] | None = None, + ) -> E2BSandboxHandle: + if (cpu_limit or mem_limit or cap_drop) and not self._warned_limits: + self._warned_limits = True + logger.warning( + "e2b backend: cpu/mem limits and cap_drop are fixed at " + "template build time; per-container values are ignored" + ) + + template = self.resolve_template(image) + envs = {k: str(v) for k, v in (environment or {}).items() if v is not None} + metadata = {str(key): str(value) for key, value in (annotations or {}).items()} + metadata[_METADATA_NAME_KEY] = name + create_kwargs = { + "template": template, + "envs": envs, + "metadata": metadata, + "timeout": self._sandbox_ttl, + "secure": True, + "network": {"allow_public_traffic": False}, + "lifecycle": {"on_timeout": "kill"}, + } + sandbox = self._create_sandbox(name, create_kwargs) + handle = E2BSandboxHandle(sandbox, name, default_user=user) + with self._handles_lock: + self._handles[handle.id] = handle + self._ensure_renewal_thread() + try: + # host.docker.internal has no meaning outside the docker backend and + # would shadow real DNS if written into /etc/hosts. + hosts = { + h: ip + for h, ip in (extra_hosts or {}).items() + if h != "host.docker.internal" and ip != "host-gateway" + } + if hosts: + lines = "".join(f"{ip} {h}\\n" for h, ip in hosts.items()) + sandbox.commands.run( + f"printf %s {shlex.quote(lines)} >> /etc/hosts", + user="root", + timeout=30, + ) + + if command and command != "tail -f /dev/null": + sandbox.commands.run( + command, + background=True, + user=user, + timeout=0, + envs=envs or None, + ) + except BaseException: + try: + self.cleanup_container(handle, logger) + except Exception: + logger.exception( + "Failed to clean up E2B Sandbox %s after initialization error", + handle.id, + ) + raise + return handle + + def _create_sandbox(self, name: str, create_kwargs: dict): + """Create a Sandbox, reconciling an ambiguous API timeout by name. + + A create request may reach E2B even when the client times out before + receiving its response. The unique SForge name in metadata lets us + reconnect to that Sandbox instead of leaking it and creating another. + """ + error = None + for attempt in range(1, SANDBOX_CREATE_ATTEMPTS + 1): + try: + return self._sandbox_cls.create(**create_kwargs) + except Exception as exc: + error = exc + if not _is_retryable_transport_error(exc): + raise + try: + matches = list(self._iter_sandboxes_by_name(name)) + except RuntimeError as lookup_exc: + logger.warning( + "Could not reconcile timed-out E2B Sandbox creation " + "for %s (attempt %d/%d): %s", + name, attempt, SANDBOX_CREATE_ATTEMPTS, lookup_exc, + ) + else: + if matches: + sandbox_id = matches[0].sandbox_id + logger.warning( + "E2B Sandbox creation response was lost; " + "reconnecting to %s (%s)", name, sandbox_id, + ) + return self._sandbox_cls.connect( + sandbox_id, timeout=self._sandbox_ttl, + ) + if attempt < SANDBOX_CREATE_ATTEMPTS: + logger.warning( + "E2B Sandbox creation failed for %s " + "(attempt %d/%d): %s", + name, attempt, SANDBOX_CREATE_ATTEMPTS, exc, + ) + time.sleep(attempt) + raise RuntimeError( + f"Failed to create E2B Sandbox {name!r} after " + f"{SANDBOX_CREATE_ATTEMPTS} attempts: {error}" + ) from error + + def start_container(self, handle: ContainerHandle) -> None: + self._raw(handle) + # E2B Sandboxes are running as soon as create() returns. + + def cleanup_container( + self, + handle: ContainerHandle | None, + logger_: logging.Logger | None = None, + ) -> None: + if handle is None: + return + log = logger_ or logger + sandbox = self._raw(handle) + # Stop extending the lease before deletion. If every kill attempt + # fails, E2B's on-timeout=kill lifecycle remains the final safeguard + # instead of this process keeping the leaked Sandbox alive forever. + with self._handles_lock: + self._handles.pop(handle.id, None) + self._renewal_wakeup.set() + log.info(f"Killing sandbox {handle.name} ({handle.id})...") + for attempt in range(1, 4): + try: + sandbox.kill() + log.info(f"Sandbox {handle.name} killed.") + return + except Exception as exc: + if attempt == 3: + log.error( + "Failed to kill sandbox %s after %d attempts: %s", + handle.id, + attempt, + exc, + ) + raise RuntimeError( + f"Failed to kill E2B Sandbox {handle.id} after " + f"{attempt} attempts: {exc}" + ) from exc + time.sleep(attempt) + + def container_exists(self, name: str) -> bool: + return any(True for _ in self._iter_sandboxes_by_name(name)) + + def remove_container_by_name(self, name: str) -> None: + for info in self._iter_sandboxes_by_name(name): + for attempt in range(1, 4): + try: + self._sandbox_cls.kill(info.sandbox_id) + with self._handles_lock: + self._handles.pop(info.sandbox_id, None) + self._renewal_wakeup.set() + break + except Exception as exc: + if attempt == 3: + raise RuntimeError( + f"Failed to remove stale E2B Sandbox " + f"{info.sandbox_id}: {exc}" + ) from exc + time.sleep(attempt) + + def _iter_sandboxes_by_name(self, name: str): + from e2b import SandboxQuery # local: keeps module import-light + + query = SandboxQuery(metadata={_METADATA_NAME_KEY: name}) + try: + paginator = self._sandbox_cls.list(query=query) + while paginator.has_next: + for info in paginator.next_items(): + if (info.metadata or {}).get(_METADATA_NAME_KEY) == name: + yield info + except Exception as exc: + raise RuntimeError( + f"Could not list E2B Sandboxes named {name!r}: {exc}" + ) from exc + + # --- Image --- + + def image_exists(self, image_key: str) -> bool: + if self._template_cls is None: + return False + template = self.resolve_template(image_key) + try: + # Check the complete name:tag alias. Sandbox.create remains the + # authoritative launchability check performed later in the flow. + return bool(self._template_cls.exists(template)) + except Exception as exc: + raise RuntimeError( + f"Could not verify E2B template {template!r}: {exc}" + ) from exc + + # --- File transfer --- + + def copy_to_container( + self, + handle: ContainerHandle, + src: Path, + dst: PurePosixPath, + ) -> None: + if str(dst.parent) == ".": + raise ValueError(f"Destination parent directory cannot be empty: {dst}") + sandbox = self._raw(handle) + # Harness file transfers include privileged destinations such as + # /usr/local/bin and /etc. Match the effective daemon-level behavior + # of Docker's put_archive by writing as root; callers set ownership + # explicitly when a user-owned destination requires it. + run_user = "root" + sandbox.commands.run( + f"mkdir -p {shlex.quote(str(dst.parent))}", + user=run_user, + timeout=60, + ) + + if src.is_dir(): + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + tar.add(src, arcname=dst.name) + tmp = f"/tmp/.sforge-copy-{uuid.uuid4().hex}.tar" + self._retry_file_transfer( + f"upload {src}", + lambda: sandbox.files.write( + tmp, buf.getvalue(), user=run_user, + request_timeout=FILE_TRANSFER_REQUEST_TIMEOUT, + ), + ) + sandbox.commands.run( + f"tar xf {tmp} -C {shlex.quote(str(dst.parent))}; " + f"status=$?; rm -f {tmp}; exit $status", + user=run_user, + timeout=300, + ) + else: + data = src.read_bytes() + self._retry_file_transfer( + f"upload {src}", + lambda: sandbox.files.write( + str(dst), data, user=run_user, + request_timeout=FILE_TRANSFER_REQUEST_TIMEOUT, + ), + ) + + def copy_from_container( + self, + handle: ContainerHandle, + src: PurePosixPath, + *, + shutdown_event: threading.Event | None = None, + ) -> bytes: + sandbox = self._raw(handle) + archive = f"/tmp/.sforge-download-{uuid.uuid4().hex}.tar" + command = ( + f"tar cf {shlex.quote(archive)} " + f"-C {shlex.quote(str(src.parent))} {shlex.quote(src.name)}" + ) + try: + result = sandbox.commands.run(command, user="root", timeout=300) + if result.exit_code != 0: + raise RuntimeError( + f"Failed to archive {src}: {result.stderr or result.stdout}" + ) + def download() -> bytes: + if shutdown_event is not None and shutdown_event.is_set(): + raise InterruptedError(f"Download of {src} was cancelled") + reader = sandbox.files.read( + archive, format="stream", user="root", + ) + try: + chunks = [] + for chunk in reader: + if ( + shutdown_event is not None + and shutdown_event.is_set() + ): + raise InterruptedError( + f"Download of {src} was cancelled" + ) + chunks.append(chunk) + return b"".join(chunks) + finally: + close = getattr(reader, "close", None) + if close is not None: + close() + + return self._retry_file_transfer(f"download {src}", download) + finally: + try: + sandbox.files.remove(archive, user="root") + except Exception: + logger.debug( + "Could not remove E2B download archive %s", + archive, + exc_info=True, + ) + + def write_to_container( + self, + handle: ContainerHandle, + data: str, + dst: PurePosixPath, + ) -> None: + sandbox = self._raw(handle) + self._retry_file_transfer( + f"write {dst}", + lambda: sandbox.files.write( + str(dst), data, user="root", + request_timeout=FILE_TRANSFER_REQUEST_TIMEOUT, + ), + ) + + # --- Exec --- + + def exec_run( + self, + handle: ContainerHandle, + cmd: str | list[str], + *, + user: str | None = None, + workdir: str | None = None, + environment: dict[str, str] | None = None, + detach: bool = False, + ) -> ExecResult: + sandbox = self._raw(handle) + run_user = user or self._default_user(handle) + cmd_str = _as_shell(cmd) + envs = environment or None + + if detach: + sandbox.commands.run( + cmd_str, + background=True, + user=run_user, + cwd=workdir, + envs=envs, + timeout=0, + ) + return ExecResult(output="", exit_code=0) + + chunks: list[str] = [] + try: + result = sandbox.commands.run( + cmd_str, + user=run_user, + cwd=workdir, + envs=envs, + timeout=DEFAULT_EXEC_TIMEOUT, + on_stdout=chunks.append, + on_stderr=chunks.append, + ) + exit_code = result.exit_code + except Exception as exc: + exit_code = _exit_code_of(exc) + if exit_code is None: + raise + return ExecResult(output="".join(chunks), exit_code=exit_code) + + def exec_run_with_timeout( + self, + handle: ContainerHandle, + cmd: str | list[str], + timeout: int | None = 60, + *, + log_file: Path | None = None, + user: str | None = None, + workdir: str | None = None, + environment: dict[str, str] | None = None, + stream_to_stdout: bool = False, + shutdown_event: threading.Event | None = None, + log_append: bool = False, + on_chunk: Callable[[bytes], None] | None = None, + ) -> StreamingExecResult: + output, exit_code, timed_out, elapsed = self._exec_stream( + handle, + cmd, + timeout, + log_file=log_file, + user=user, + workdir=workdir, + environment=environment, + stream_to_stdout=stream_to_stdout, + shutdown_event=shutdown_event, + log_append=log_append, + on_chunk=on_chunk, + ) + return StreamingExecResult( + output=output, + exit_code=exit_code, + timed_out=timed_out, + elapsed_seconds=elapsed, + ) + + def exec_run_with_exit_code( + self, + handle: ContainerHandle, + cmd: str | list[str], + timeout: int | None = 60, + *, + user: str | None = None, + workdir: str | None = None, + environment: dict[str, str] | None = None, + ) -> StreamingExecResult: + output, exit_code, timed_out, elapsed = self._exec_stream( + handle, + cmd, + timeout, + user=user, + workdir=workdir, + environment=environment, + ) + return StreamingExecResult( + output=output, + exit_code=exit_code, + timed_out=timed_out, + elapsed_seconds=elapsed, + ) + + def _exec_stream( + self, + handle: ContainerHandle, + cmd: str | list[str], + timeout: int | None, + *, + log_file: Path | None = None, + user: str | None = None, + workdir: str | None = None, + environment: dict[str, str] | None = None, + stream_to_stdout: bool = False, + shutdown_event: threading.Event | None = None, + log_append: bool = False, + on_chunk: Callable[[bytes], None] | None = None, + ) -> tuple[str, int, bool, float]: + import sys + + sandbox = self._raw(handle) + run_user = user or self._default_user(handle) + chunks: list[str] = [] + exit_code: int | None = None + exception: Exception | None = None + timed_out = False + log_fh = None + + try: + if log_file: + log_file.parent.mkdir(parents=True, exist_ok=True) + log_fh = open(log_file, "ab" if log_append else "wb") + + def emit(text: str) -> None: + chunks.append(text) + raw = text.encode(errors="replace") + if log_fh: + log_fh.write(raw) + log_fh.flush() + if stream_to_stdout: + sys.stdout.buffer.write(raw) + sys.stdout.buffer.flush() + if on_chunk is not None: + try: + on_chunk(raw) + except Exception: + logger.debug( + "E2B output callback failed", exc_info=True, + ) + + # The wait stream can be interrupted by request timeouts or + # ordinary network failures while agent runs last hours. The + # command therefore runs in the background — it survives a dropped + # stream — and its exit code is persisted in the sandbox so it is + # recoverable even if the process finishes while we are detached. + rc_file = f"/tmp/.sforge-rc-{uuid.uuid4().hex}" + wrapped = ( + f"{{ {_as_shell(cmd)}\n}}; __ec=$?; echo $__ec > {rc_file}; exit $__ec" + ) + proc = sandbox.commands.run( + wrapped, + background=True, + user=run_user, + cwd=workdir, + envs=environment or None, + timeout=0, + ) + + def read_rc_file() -> int | None: + try: + result = sandbox.commands.run( + f"cat {rc_file}", + user="root", + timeout=30, + ) + return int(result.stdout.strip()) + except Exception: + return None + + def wait_command() -> None: + nonlocal exit_code, exception + attach = proc + reconnects = 0 + while True: + try: + result = attach.wait(on_stdout=emit, on_stderr=emit) + exit_code = result.exit_code + return + except Exception as exc: + code = _exit_code_of(exc) + if code is not None: + exit_code = code + return + # Stream dropped, not a command failure. Reattach to + # the still-running process; output produced while + # detached is lost, the exit code is not. + last_exc = exc + for attempt in range(5): + if shutdown_event and shutdown_event.is_set(): + exception = last_exc + return + try: + attach = sandbox.commands.connect(proc.pid, timeout=0) + break + except Exception: + code = read_rc_file() + if code is not None: + # Command finished while we were detached. + exit_code = code + return + time.sleep(min(2**attempt, 15)) + else: + exception = last_exc + return + reconnects += 1 + logger.info( + "e2b exec stream dropped, reattached to pid %s (reconnect #%d)", + proc.pid, + reconnects, + ) + + start_time = time.time() + thread = threading.Thread(target=wait_command, daemon=True) + thread.start() + + deadline = start_time + timeout if timeout else None + while thread.is_alive(): + remaining = (deadline - time.time()) if deadline else 1.0 + if remaining <= 0: + break + thread.join(min(remaining, 1.0)) + if shutdown_event and shutdown_event.is_set(): + break + + if thread.is_alive(): + timed_out = True + try: + killed = sandbox.commands.kill(proc.pid) + if killed is False: + raise RuntimeError( + f"E2B command {proc.pid} was not found while timing out" + ) + except Exception as exc: + raise RuntimeError( + f"Failed to stop timed-out E2B command {proc.pid}: {exc}" + ) from exc + thread.join(10) + if thread.is_alive(): + raise RuntimeError( + f"Timed-out E2B command {proc.pid} did not terminate" + ) + # The late thread result (e.g. 137 from the kill) must not + # shadow the docker-contract exit code for timeouts. + exit_code = -1 + exception = None + finally: + if log_fh: + log_fh.close() + if "rc_file" in locals(): + try: + sandbox.files.remove(rc_file, user="root") + except Exception: + logger.debug( + "Could not remove E2B command status file %s", + rc_file, + exc_info=True, + ) + + if exception: + raise exception + + elapsed = time.time() - start_time + return ( + "".join(chunks), + exit_code if exit_code is not None else -1, + timed_out, + elapsed, + ) + + # --- Inspection --- + + def get_container_ip(self, handle: ContainerHandle) -> str: + raise NotImplementedError( + "e2b sandboxes have no routable IP; use get_service_endpoint() to " + "reach in-sandbox services through the e2b gateway" + ) + + def get_service_endpoint( + self, + handle: ContainerHandle, + port: int, + ) -> ServiceEndpoint: + sandbox = self._raw(handle) + token = getattr(sandbox, "traffic_access_token", None) + if not token: + raise RuntimeError( + "E2B did not return a traffic access token for the secured service" + ) + return ServiceEndpoint( + url=f"https://{sandbox.get_host(port)}", + headers={"e2b-traffic-access-token": token}, + ) + + def get_container_gateway_ip(self, handle: ContainerHandle) -> str | None: + return None + + # --- Network isolation --- + + def create_network_isolation( + self, + handle: ContainerHandle, + allowed_endpoints: list, + logger_: logging.Logger, + ) -> NetworkIsolationStrategy: + return E2BNetworkIsolation(self._raw(handle), allowed_endpoints, logger_) + + # --- Lease renewal --- + + def _ensure_renewal_thread(self) -> None: + """Start one renewal worker while holding ``_handles_lock``.""" + if self._renewal_thread is not None and self._renewal_thread.is_alive(): + return + self._renewal_wakeup.clear() + self._renewal_thread = threading.Thread( + target=self._renewal_loop, + daemon=True, + name="e2b-lease-renewal", + ) + self._renewal_thread.start() + + def _renewal_loop(self) -> None: + interval = max(self._sandbox_ttl // 3, 1) + while True: + woke_early = self._renewal_wakeup.wait(interval) + with self._handles_lock: + handles = list(self._handles.values()) + if not handles: + self._renewal_thread = None + return + if woke_early: + self._renewal_wakeup.clear() + if woke_early: + continue + for handle in handles: + try: + handle.raw.set_timeout(self._sandbox_ttl) + except Exception as exc: + logger.warning( + f"Lease renewal failed for sandbox {handle.id}: {exc}" + ) + + # --- Helpers --- + + @staticmethod + def _default_user(handle: ContainerHandle) -> str | None: + return getattr(handle, "default_user", None) + + @staticmethod + def _raw(handle: ContainerHandle): + if isinstance(handle, E2BSandboxHandle): + return handle.raw + raise TypeError(f"Expected E2BSandboxHandle, got {type(handle).__name__}") + + @staticmethod + def _retry_file_transfer(description: str, operation): + for attempt in range(1, FILE_TRANSFER_ATTEMPTS + 1): + try: + return operation() + except Exception as exc: + if ( + attempt == FILE_TRANSFER_ATTEMPTS + or not _is_retryable_transport_error(exc) + ): + raise + logger.warning( + "E2B file transfer failed while attempting to %s " + "(attempt %d/%d): %s", + description, attempt, FILE_TRANSFER_ATTEMPTS, exc, + ) + time.sleep(attempt) + raise AssertionError("unreachable") + + +def _as_shell(cmd: str | list[str]) -> str: + return shlex.join(cmd) if isinstance(cmd, list) else cmd + + +def _exit_code_of(exc: Exception) -> int | None: + """Exit code carried by a CommandExitException, duck-typed so fakes and + provider variants work.""" + code = getattr(exc, "exit_code", None) + return code if isinstance(code, int) else None + + +def _is_retryable_transport_error(exc: Exception) -> bool: + """Recognize transport failures without retrying provider rejections.""" + current: BaseException | None = exc + while current is not None: + if type(current).__name__ in { + "ConnectError", + "ConnectTimeout", + "ReadError", + "ReadTimeout", + "RemoteProtocolError", + "TimeoutException", + "WriteError", + "WriteTimeout", + }: + return True + current = current.__cause__ or current.__context__ + message = str(exc).lower() + return any( + marker in message + for marker in ( + "connection reset", + "operation timed out", + "request timed out", + "temporarily unavailable", + ) + ) + + +def template_map_from_env() -> dict[str, str]: + """Parse SFORGE_E2B_TEMPLATE_MAP: inline ``key=value,key=value`` pairs or + a path to a JSON object file.""" + raw = os.environ.get("SFORGE_E2B_TEMPLATE_MAP", "").strip() + if not raw: + return {} + if raw.endswith(".json"): + try: + return { + str(k): str(v) for k, v in json.loads(Path(raw).read_text()).items() + } + except (OSError, json.JSONDecodeError) as exc: + raise ValueError( + f"Cannot load SFORGE_E2B_TEMPLATE_MAP file {raw!r}: {exc}" + ) from exc + result: dict[str, str] = {} + for item in raw.split(","): + item = item.strip() + if not item: + continue + if "=" not in item: + raise ValueError( + f"Malformed SFORGE_E2B_TEMPLATE_MAP entry {item!r} (expected key=value)" + ) + k, v = item.split("=", 1) + result[k.strip()] = v.strip() + return result diff --git a/sforge/harness/backend/factory.py b/sforge/harness/backend/factory.py index 707d929..9db876d 100644 --- a/sforge/harness/backend/factory.py +++ b/sforge/harness/backend/factory.py @@ -27,6 +27,8 @@ def create_backend( k8s_node_selector: dict[str, str] | None = None, k8s_image_registry: str = "", k8s_kubeconfig: str | None = None, + e2b_template_map: dict[str, str] | None = None, + e2b_sandbox_ttl: int | None = None, ) -> ContainerBackend: if name == "docker": from sforge.harness.backend.docker_backend import DockerBackend @@ -41,5 +43,12 @@ def create_backend( image_registry=k8s_image_registry, kubeconfig=k8s_kubeconfig, ) - else: - raise ValueError(f"Unknown backend: {name!r}. Must be 'docker' or 'k8s'.") + elif name == "e2b": + from sforge.harness.backend.e2b_backend import E2BBackend + + return E2BBackend( + template_map=e2b_template_map, + sandbox_ttl=e2b_sandbox_ttl, + ) + supported = "docker, k8s, e2b" + raise ValueError(f"Unknown backend: {name!r}. Must be one of: {supported}.") diff --git a/sforge/harness/config.py b/sforge/harness/config.py index c86efd0..1eee502 100644 --- a/sforge/harness/config.py +++ b/sforge/harness/config.py @@ -69,7 +69,7 @@ class SForgeConfig: # Remote container registry registry: str | None = None - # Container backend: "docker" or "k8s" + # Container backend: "docker", "k8s" or "e2b" backend: str = "docker" # K8s backend settings @@ -77,6 +77,12 @@ class SForgeConfig: k8s_node_selector: dict[str, str] = field(default_factory=dict) k8s_image_registry: str = "" k8s_kubeconfig: str | None = None + + # e2b backend settings (endpoint/credentials come from the standard + # E2B_API_KEY / E2B_API_URL / E2B_DOMAIN env vars read by the e2b SDK) + e2b_template_map: dict[str, str] = field(default_factory=dict) + e2b_sandbox_ttl: int | None = None + e2b_judge_access_token: str | None = None # Paths log_dir: Path = field(default_factory=lambda: LOG_DIR) tasks_dir: Path = field(default_factory=lambda: TASKS_DIR) @@ -108,6 +114,7 @@ def load_config(cli_overrides: dict | None = None) -> SForgeConfig: "k8s_namespace": ["SFORGE_K8S_NAMESPACE"], "k8s_image_registry": ["SFORGE_K8S_IMAGE_REGISTRY"], "k8s_kubeconfig": ["SFORGE_K8S_KUBECONFIG"], + "e2b_judge_access_token": ["SFORGE_JUDGE_ACCESS_TOKEN"], "work_cpu_limit": ["SFORGE_WORK_CPU_LIMIT"], "work_mem_limit": ["SFORGE_WORK_MEM_LIMIT"], "log_dir": ["SFORGE_LOG_DIR"], @@ -168,10 +175,19 @@ def load_config(cli_overrides: dict | None = None) -> SForgeConfig: k, v = item.split("=", 1) config.k8s_node_selector[k.strip()] = v.strip() + # SFORGE_E2B_TEMPLATE_MAP: "image_key=template,..." or a JSON file path + if os.environ.get("SFORGE_E2B_TEMPLATE_MAP"): + from sforge.harness.backend.e2b_backend import template_map_from_env + config.e2b_template_map = template_map_from_env() + e2b_ttl = os.environ.get("SFORGE_E2B_SANDBOX_TTL") + if e2b_ttl: + config.e2b_sandbox_ttl = int(e2b_ttl) # Override from CLI flags if cli_overrides: for k, v in cli_overrides.items(): if v is not None and hasattr(config, k): + if k in ("log_dir", "tasks_dir"): + v = Path(v) setattr(config, k, v) return config @@ -267,6 +283,26 @@ def get_container_env(config: SForgeConfig, include_judge_extra: bool = False) - return env +def sanitize_e2b_proxy_env( + env: dict[str, str], *, internet: bool = True, +) -> None: + """Remove host-only proxies before sending an environment to E2B. + + Generic ``HTTP_PROXY``/``HTTPS_PROXY`` values are commonly inherited from + the machine running SForge and may point at private hosts that a remote + Sandbox cannot reach. Only the explicit SForge variants opt a proxy into + E2B; isolated Work Sandboxes never receive one. + """ + proxy_groups = ( + ("SFORGE_HTTP_PROXY", "http_proxy", "HTTP_PROXY"), + ("SFORGE_HTTPS_PROXY", "https_proxy", "HTTPS_PROXY"), + ) + for explicit_name, *runtime_names in proxy_groups: + if not internet or not os.environ.get(explicit_name): + for name in runtime_names: + env.pop(name, None) + + def get_container_resource_kwargs( config_cpu_limit: int | None, config_mem_limit: str | None, @@ -298,4 +334,6 @@ def create_backend_from_config(config: SForgeConfig, docker_client=None): k8s_node_selector=config.k8s_node_selector, k8s_image_registry=config.k8s_image_registry, k8s_kubeconfig=config.k8s_kubeconfig, + e2b_template_map=config.e2b_template_map or None, + e2b_sandbox_ttl=config.e2b_sandbox_ttl, ) diff --git a/sforge/harness/constants.py b/sforge/harness/constants.py index e4c0f1c..a151735 100644 --- a/sforge/harness/constants.py +++ b/sforge/harness/constants.py @@ -14,8 +14,12 @@ from __future__ import annotations +import fcntl import os +import secrets +import stat from pathlib import Path + from typing_extensions import NotRequired, TypedDict # --- Paths --- @@ -39,10 +43,33 @@ # --- Defaults --- DEFAULT_EVAL_INTERVAL = 300 # seconds -# --- Admin secret for judge server registration --- -# Only known to host-side processes (run_agent.py, judge_server.py). -# Never injected into agent containers. -ADMIN_SECRET = "sEb3nCh!aDm1n#2026-x9Kp7qW4mZq" + +def get_admin_secret(log_dir: Path | None = None) -> str: + """Load or create the host/controller secret without import side effects.""" + configured = os.environ.get("SFORGE_ADMIN_SECRET") + if configured: + return configured + path = (log_dir or LOG_DIR) / ".judge-admin-secret" + path.parent.mkdir(parents=True, exist_ok=True) + flags = os.O_RDWR | os.O_CREAT + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + fd = os.open(path, flags, 0o600) + with os.fdopen(fd, "r+") as secret_file: + fcntl.flock(secret_file.fileno(), fcntl.LOCK_EX) + mode = os.fstat(secret_file.fileno()).st_mode + if not stat.S_ISREG(mode): + raise RuntimeError(f"Judge admin secret is not a regular file: {path}") + os.fchmod(secret_file.fileno(), 0o600) + secret = secret_file.read().strip() + if not secret: + secret = secrets.token_urlsafe(32) + secret_file.seek(0) + secret_file.write(secret) + secret_file.truncate() + secret_file.flush() + os.fsync(secret_file.fileno()) + return secret # --- Docker --- DOCKER_USER = "root" diff --git a/sforge/harness/dockerfiles.py b/sforge/harness/dockerfiles.py index 4d8fe41..c19add4 100644 --- a/sforge/harness/dockerfiles.py +++ b/sforge/harness/dockerfiles.py @@ -16,9 +16,6 @@ from __future__ import annotations -from __future__ import annotations - - # --- Base Dockerfile: official image + common tools --- _DOCKERFILE_BASE = """\ FROM --platform={platform} {official_image} @@ -26,9 +23,10 @@ ARG DEBIAN_FRONTEND=noninteractive ENV TZ=Etc/UTC {env_directives} -{apt_mirror_directive}RUN apt-get update && apt-get install -y --no-install-recommends \\ +RUN {apt_mirror_prepare}apt-get update && apt-get install -y --no-install-recommends \\ {packages} sudo \\ && rm -rf /var/lib/apt/lists/* \\ +{apt_mirror_restore}\ && useradd -m -s /bin/bash agent \\ && echo 'agent ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \\ && git config --global safe.directory '*' @@ -68,24 +66,34 @@ def get_dockerfile_base( user_directive = spec.get("user_directive", "") post_install_directive = spec.get("post_install_directive", "") - apt_mirror_directive = "" + apt_mirror_prepare = "" + apt_mirror_restore = "" if apt_mirror_url: url = apt_mirror_url.rstrip("/") - apt_mirror_directive = ( - f"RUN sed -i 's|http://deb.debian.org|{url}|g; " - f"s|https://deb.debian.org|{url}|g; " - f"s|http://archive.ubuntu.com|{url}|g; " - f"s|http://security.ubuntu.com|{url}|g' " - f"/etc/apt/sources.list 2>/dev/null; " - f"sed -i 's|http://deb.debian.org|{url}|g; " - f"s|https://deb.debian.org|{url}|g; " - f"s|http://archive.ubuntu.com|{url}|g; " - f"s|http://security.ubuntu.com|{url}|g' " - f"/etc/apt/sources.list.d/*.list 2>/dev/null; " - f"sed -i 's|http://deb.debian.org|{url}|g; " - f"s|https://deb.debian.org|{url}|g' " - f"/etc/apt/sources.list.d/*.sources 2>/dev/null; " - f"true\n" + apt_mirror_prepare = ( + "rm -rf /tmp/sforge-apt-original && " + "mkdir -p /tmp/sforge-apt-original && " + "if [ -e /etc/apt/sources.list ]; then " + "cp -a /etc/apt/sources.list /tmp/sforge-apt-original/; fi && " + "if [ -d /etc/apt/sources.list.d ]; then " + "cp -a /etc/apt/sources.list.d /tmp/sforge-apt-original/; fi && " + "find /etc/apt -maxdepth 2 -type f " + "\\( -name '*.list' -o -name '*.sources' \\) " + "-exec sed -Ei " + f"-e 's#https?://deb.debian.org#{url}#g' " + f"-e 's#https?://archive.ubuntu.com#{url}#g' " + f"-e 's#https?://security.ubuntu.com#{url}#g' " + "{} + && " + ) + apt_mirror_restore = ( + " && rm -f /etc/apt/sources.list \\\n" + " && rm -rf /etc/apt/sources.list.d \\\n" + " && if [ -e /tmp/sforge-apt-original/sources.list ]; then " + "mv /tmp/sforge-apt-original/sources.list /etc/apt/; fi \\\n" + " && if [ -d /tmp/sforge-apt-original/sources.list.d ]; then " + "mv /tmp/sforge-apt-original/sources.list.d /etc/apt/; " + "else mkdir -p /etc/apt/sources.list.d; fi \\\n" + " && rm -rf /tmp/sforge-apt-original \\\n" ) pip_packages = spec.get("pip_packages", []) @@ -99,7 +107,8 @@ def get_dockerfile_base( official_image=spec["official_image"], user_directive=user_directive, env_directives=env_directives, - apt_mirror_directive=apt_mirror_directive, + apt_mirror_prepare=apt_mirror_prepare, + apt_mirror_restore=apt_mirror_restore, packages=packages, pip_directive=pip_directive, post_install_directive=post_install_directive, diff --git a/sforge/harness/e2b_controller.py b/sforge/harness/e2b_controller.py new file mode 100644 index 0000000..cf5a4cb --- /dev/null +++ b/sforge/harness/e2b_controller.py @@ -0,0 +1,391 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Managed Judge controller for brokerless official E2B runs.""" + +from __future__ import annotations + +import io +import json +import logging +import os +import shlex +import tarfile +import time +import uuid +from importlib.metadata import PackageNotFoundError, requires +from pathlib import Path, PurePosixPath + +import requests + +from sforge.harness.backend import ContainerBackend, ContainerHandle +from sforge.harness.config import SForgeConfig +from sforge.harness.constants import get_admin_secret +from sforge.harness.judge_client import JudgeConnection + +logger = logging.getLogger(__name__) + + +def _controller_requirements() -> tuple[str, ...]: + """Return this installed SForge version's controller dependencies.""" + try: + declared = requires("sforge") or [] + except PackageNotFoundError: + declared = [] + runtime = [item for item in declared if "extra ==" not in item] + e2b = [ + item.split(";", 1)[0].strip() + for item in declared + if "extra == 'e2b'" in item or 'extra == "e2b"' in item + ] + return tuple(runtime + e2b) + + +class ManagedE2BController: + """Create, expose and tear down a Judge Server in an E2B Sandbox.""" + + def __init__( + self, + backend: ContainerBackend, + config: SForgeConfig, + run_id: str, + *, + source_root: Path | None = None, + ) -> None: + self.backend = backend + self.config = config + self.run_id = run_id + self.source_root = source_root or Path(__file__).resolve().parents[2] + self.handle: ContainerHandle | None = None + self.connection: JudgeConnection | None = None + self.host_connection: JudgeConnection | None = None + + def _environment(self) -> dict[str, str]: + allowed = ("E2B_API_KEY", "E2B_API_URL", "E2B_DOMAIN") + env = {key: os.environ[key] for key in allowed if os.environ.get(key)} + env.update( + { + "PYTHONPATH": "/opt/sforge-controller", + "SFORGE_TASKS_DIR": "/opt/sforge-controller/tasks", + "SFORGE_LOG_DIR": "/tmp/sforge-controller/logs", + "SFORGE_BACKEND": "e2b", + "SFORGE_ADMIN_SECRET": get_admin_secret(self.config.log_dir), + "SFORGE_E2B_SANDBOX_TTL": str(self.config.e2b_sandbox_ttl or 3600), + } + ) + if self.config.judge_extra_env: + env["SFORGE_JUDGE_EXTRA_ENV"] = ",".join( + f"{key}={value}" for key, value in self.config.judge_extra_env.items() + ) + if self.config.e2b_template_map: + env["SFORGE_E2B_TEMPLATE_MAP"] = ",".join( + f"{key}={value}" for key, value in self.config.e2b_template_map.items() + ) + return env + + def start(self, timeout: int = 120) -> JudgeConnection: + if self.handle is not None: + raise RuntimeError("Managed E2B controller is already running") + env = self._environment() + self.handle = self.backend.create_container( + "base", + f"sforge.controller.{self.run_id}", + user="root", + ) + try: + self.backend.copy_to_container( + self.handle, + self.source_root / "sforge", + PurePosixPath("/opt/sforge-controller/sforge"), + ) + self.backend.copy_to_container( + self.handle, + self.config.tasks_dir, + PurePosixPath("/opt/sforge-controller/tasks"), + ) + requirements = _controller_requirements() + if not requirements: + raise RuntimeError( + "Could not determine managed controller dependencies from " + "the installed SForge package metadata" + ) + requirement_args = " ".join(shlex.quote(item) for item in requirements) + bootstrap = self.backend.exec_run_with_exit_code( + self.handle, + "python3 -m venv /tmp/sforge-controller-venv && " + "/tmp/sforge-controller-venv/bin/python -m pip install " + "--disable-pip-version-check --no-cache-dir --quiet " + + requirement_args, + timeout=300, + user="root", + ) + if bootstrap.exit_code != 0: + raise RuntimeError( + "Failed to prepare managed E2B Judge controller: " + + bootstrap.output[-8000:] + ) + command = ( + "cd /opt/sforge-controller && " + "/tmp/sforge-controller-venv/bin/python -m sforge.cli serve " + "--host 0.0.0.0 --port 8080 " + "> /tmp/sforge-judge.log 2>&1" + ) + self.backend.exec_run( + self.handle, + command, + detach=True, + user="root", + environment=env, + ) + endpoint = self.backend.get_service_endpoint(self.handle, 8080) + self.connection = JudgeConnection.from_e2b_traffic_token( + endpoint.url, + endpoint.headers.get("e2b-traffic-access-token"), + ) + self.host_connection = JudgeConnection( + url="http://127.0.0.1:8080", + request_handler=self._request_on_controller, + ) + deadline = time.time() + timeout + while time.time() < deadline: + try: + response = self.connection.request( + "GET", + "/openapi.json", + timeout=5, + ) + if response.status_code == 200: + return self.connection + except requests.RequestException: + pass + time.sleep(1) + log = self.backend.exec_run( + self.handle, + "tail -200 /tmp/sforge-judge.log", + user="root", + ) + raise RuntimeError( + "Managed E2B Judge controller did not become ready: " + + log.output[-8000:] + ) + except BaseException: + try: + self.close() + except Exception: + logger.exception( + "Failed to clean up managed E2B controller %s after startup error", + self.run_id, + ) + raise + + def close(self) -> None: + if self.handle is not None: + try: + self._export_run_logs() + except Exception as exc: + logger.warning( + "Could not export managed E2B controller logs for %s: %s", + self.run_id, + exc, + ) + self.backend.cleanup_container(self.handle) + self.handle = None + self.connection = None + self.host_connection = None + + def _request_on_controller( + self, method: str, path: str, **kwargs, + ) -> requests.Response: + """Send a host-side Judge request over the E2B command channel. + + Work Sandboxes still use the secured public gateway. Host orchestration + can avoid that extra hop by issuing the same HTTP request from inside + the Controller Sandbox against loopback. + """ + if self.handle is None: + raise RuntimeError("Managed E2B Judge controller is not running") + + request_id = uuid.uuid4().hex + prefix = f"/tmp/.sforge-host-request-{request_id}" + spec_path = f"{prefix}.json" + response_path = f"{prefix}.response" + files = kwargs.pop("files", None) or {} + timeout = kwargs.pop("timeout", 60) + spec = { + "method": method, + "url": f"http://127.0.0.1:8080/{path.lstrip('/')}", + "headers": kwargs.pop("headers", None) or {}, + "json": kwargs.pop("json", None), + "data": kwargs.pop("data", None), + "timeout": timeout, + "files": {}, + } + if kwargs: + raise TypeError( + "Unsupported managed Judge request arguments: " + + ", ".join(sorted(kwargs)) + ) + + raw = self.handle.raw + uploaded_paths: list[str] = [] + try: + for index, (field_name, file_spec) in enumerate(files.items()): + filename, content, content_type = file_spec + remote_path = f"{prefix}.upload-{index}" + raw.files.write( + remote_path, content, user="root", request_timeout=300, + ) + uploaded_paths.append(remote_path) + spec["files"][field_name] = { + "filename": filename, + "path": remote_path, + "content_type": content_type, + } + raw.files.write( + spec_path, json.dumps(spec), user="root", request_timeout=60, + ) + command = ( + "/tmp/sforge-controller-venv/bin/python -c " + + shlex.quote( + "import json,requests,sys; " + "s=json.load(open(sys.argv[1])); opened=[]; files={}; " + "[(opened.append(open(v['path'],'rb')), " + "files.__setitem__(k,(v['filename'],opened[-1],v['content_type']))) " + "for k,v in s['files'].items()]; " + "r=requests.request(s['method'],s['url'],headers=s['headers']," + "json=s['json'],data=s['data'],files=files or None," + "timeout=s['timeout']); " + "open(sys.argv[2],'wb').write(r.content); print(r.status_code)" + ) + + f" {shlex.quote(spec_path)} {shlex.quote(response_path)}" + ) + exec_timeout = ( + int(max(timeout)) if isinstance(timeout, tuple) else int(timeout) + ) + 30 + result = self.backend.exec_run_with_exit_code( + self.handle, command, timeout=exec_timeout, user="root", + ) + if result.timed_out: + raise requests.Timeout( + f"Managed Judge loopback request timed out: {method} {path}" + ) + if result.exit_code != 0: + raise requests.ConnectionError( + "Managed Judge loopback request failed: " + + result.output[-2000:] + ) + try: + status_code = int(result.output.strip().splitlines()[-1]) + except (ValueError, IndexError) as exc: + raise requests.ConnectionError( + "Managed Judge loopback request returned no HTTP status: " + + result.output[-2000:] + ) from exc + content = raw.files.read( + response_path, format="bytes", user="root", + request_timeout=300, + ) + response = requests.Response() + response.status_code = status_code + response._content = content + response.url = spec["url"] + response.encoding = "utf-8" + return response + finally: + targets = [spec_path, response_path, *uploaded_paths] + try: + self.backend.exec_run( + self.handle, + "rm -f -- " + " ".join(map(shlex.quote, targets)), + user="root", + ) + except Exception: + logger.debug( + "Could not clean up managed Judge request files", + exc_info=True, + ) + + def _export_run_logs(self) -> None: + """Merge Judge artifacts into the host run directory before cleanup.""" + if self.handle is None: + return + remote_run = f"/tmp/sforge-controller/logs/runs/{self.run_id}" + remote_archive = "/tmp/sforge-controller-run-logs.tar.gz" + result = self.backend.exec_run_with_exit_code( + self.handle, + f"test -d {shlex.quote(remote_run)} || exit 3; " + f"tar czf {remote_archive} " + "--exclude='submission.tar.gz' --exclude='final_archive.tar.gz' " + f"-C {shlex.quote(remote_run)} .", + timeout=120, + user="root", + ) + if result.timed_out: + raise TimeoutError("Timed out while exporting controller logs") + if result.exit_code == 3: + return + if result.exit_code != 0: + raise RuntimeError(result.output[-4000:]) + wrapped = self.backend.copy_from_container( + self.handle, + PurePosixPath(remote_archive), + ) + with tarfile.open(fileobj=io.BytesIO(wrapped)) as outer: + files = [member for member in outer.getmembers() if member.isfile()] + if len(files) != 1: + raise RuntimeError( + f"Unexpected controller log archive wrapper: {len(files)} files" + ) + source = outer.extractfile(files[0]) + if source is None: + raise RuntimeError("Could not read controller log archive") + archive = source.read() + _merge_log_archive( + archive, + self.config.log_dir / "runs" / self.run_id, + ) + + def __enter__(self) -> JudgeConnection: + return self.start() + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + +def _merge_log_archive(archive: bytes, target_dir: Path) -> None: + """Safely merge a controller-created tar.gz into a host run tree.""" + target_root = target_dir.resolve() + total_size = 0 + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as bundle: + for member in bundle.getmembers(): + target = (target_root / member.name).resolve() + if target != target_root and target_root not in target.parents: + raise ValueError(f"Unsafe controller log path: {member.name!r}") + if member.issym() or member.islnk(): + raise ValueError( + f"Controller log links are not supported: {member.name!r}" + ) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + if not member.isfile(): + continue + total_size += member.size + if member.size > 64 * 1024 * 1024 or total_size > 256 * 1024 * 1024: + raise ValueError("Controller log archive exceeds the size limit") + source = bundle.extractfile(member) + if source is None: + continue + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(source.read()) diff --git a/sforge/harness/e2b_templates.py b/sforge/harness/e2b_templates.py new file mode 100644 index 0000000..5a548ed --- /dev/null +++ b/sforge/harness/e2b_templates.py @@ -0,0 +1,1248 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Deterministic E2B Template supply chain for SForge task images.""" + +from __future__ import annotations + +import hashlib +import io +import json +import re +import shlex +import stat +import tarfile +import time +from dataclasses import asdict, dataclass, replace +from pathlib import Path + +import docker.errors +import requests + +from sforge.harness.task_spec import TaskSpec + +# schema_version stays 4: the manifest gains a snapshot_roots field, but that +# field is excluded from the fingerprint, and schema_version itself is part of +# every existing fingerprint. Keeping it at 4 lets clean tasks and tasks whose +# image /tmp is empty reuse their current E2B builds; only tasks that actually +# ship files under /tmp get a new fingerprint and rebuild. +MANIFEST_SCHEMA_VERSION = 4 +PORTABLE_IMAGE_RECIPE_VERSION = 2 +WORK_FILESYSTEM_RECIPE_VERSION = 2 +DEFAULT_WORK_MEMORY_MB = 4096 +DEFAULT_JUDGE_MEMORY_MB = 8192 + +# The official task images predate E2B and ship evaluator helpers under /tmp: a +# wrapper the command names plus other files that wrapper reads at runtime. +# Docker and Kubernetes boot from the image, so those files are simply present; +# E2B resets /tmp on every Sandbox, so any helper the command does not name on +# its own line silently disappears. SForge therefore snapshots the image's +# initial /tmp and restores it on Sandbox start, restoring parity with the other +# backends. This is a backend-compatibility shim: it never rewrites the official +# images, the evaluators, or any scoring behavior. +_TMP_SNAPSHOT_ROOT = "/tmp" +_TMP_SNAPSHOT_PERSISTENT = "/opt/sforge/runtime/tmp" +TMP_SNAPSHOT_RECIPE_VERSION = 1 +# A /tmp path named anywhere in the role's command decides *whether* to attempt +# a snapshot; the snapshot itself always captures the whole image /tmp, so +# helpers the command reaches only indirectly come along too. Capturing all of +# /tmp (rather than the named paths) is what makes this robust, and keying the +# decision on a /tmp reference preserves the no-pull fast path for the many +# tasks — including multi-gigabyte ones — that never touch /tmp. +_TMP_PATH = re.compile(r"(? bool: + if isinstance(exc, (docker.errors.ImageNotFound, docker.errors.NotFound)): + return True + # Some Registry implementations surface a missing manifest through the + # Docker Engine distribution endpoint as HTTP 500 instead of 404. Only + # accept its explicit not-found wording; other server errors remain fatal. + message = str(exc).lower() + return "artifact " in message and " not found" in message + + +def _manifest_fingerprint(data: dict, role: str) -> str: + fingerprint_data = dict(data) + if role == "work": + fingerprint_data["filesystem_recipe_version"] = WORK_FILESYSTEM_RECIPE_VERSION + # snapshot_roots records the build-time intent to snapshot /tmp; it is not + # part of the built template's content, so it never enters the fingerprint. + # Only a non-empty captured payload (runtime_artifacts) changes what the + # template ships, so the snapshot recipe version is folded in only then. + # This keeps clean tasks byte-identical to the pre-snapshot fingerprint and + # avoids rebuilding every template. + fingerprint_data.pop("snapshot_roots", None) + if fingerprint_data.get("runtime_artifacts"): + fingerprint_data["tmp_snapshot_recipe_version"] = TMP_SNAPSHOT_RECIPE_VERSION + return hashlib.sha256( + json.dumps(fingerprint_data, sort_keys=True).encode() + ).hexdigest() + + +def template_name(image_key: str) -> str: + repo, _, _ = image_key.partition(":") + return repo.lower().replace(".", "-").replace("_", "-") + + +def template_tag(image_key: str) -> str: + _, sep, tag = image_key.rpartition(":") + if not sep or not tag: + raise ValueError(f"Image key has no immutable tag: {image_key!r}") + return tag + + +def parse_memory_mb(value: str | None, default: int = 4096) -> int: + if value is None: + return default + raw = str(value).strip().lower() + match = re.fullmatch(r"(\d+)([kmgt]i?b?|)", raw) + if not match: + raise ValueError(f"Unsupported memory value: {value!r}") + amount = int(match.group(1)) + if amount <= 0: + raise ValueError("Memory must be greater than zero") + unit = match.group(2) + if unit in ("", "m", "mb", "mib"): + return amount + if unit in ("k", "kb", "kib"): + return max(1, amount // 1024) + if unit in ("g", "gb", "gib"): + return amount * 1024 + if unit in ("t", "tb", "tib"): + return amount * 1024 * 1024 + raise ValueError(f"Unsupported memory unit: {unit!r}") + + +@dataclass(frozen=True) +class RuntimeArtifact: + source: str + persistent: str + digest: str = "" + + +@dataclass(frozen=True) +class TemplateManifest: + schema_version: int + task_id: str + role: str + source_image: str + source_image_digest: str + image_key: str + name: str + tag: str + user: str + workdir: str + cpu_count: int + memory_mb: int + environment: tuple[tuple[str, str], ...] + runtime_artifacts: tuple[RuntimeArtifact, ...] + snapshot_roots: tuple[str, ...] + fingerprint: str + + @property + def reference(self) -> str: + return f"{self.name}:{self.tag}" + + def to_dict(self) -> dict: + return asdict(self) + + +def _command_references_tmp(task_spec: TaskSpec, role: str) -> bool: + """Whether the role's command touches /tmp, so its image /tmp is snapshotted. + + This only decides *whether* to snapshot. The snapshot itself always captures + the entire image /tmp, so evaluator helpers the command reaches only + indirectly (a wrapper that reads a sibling file) are preserved too. Keying + the decision on a /tmp reference keeps the no-pull fast path for the many + tasks — including multi-gigabyte ones — whose commands never touch /tmp. + """ + if role == "work": + command = task_spec.work.agent_query or "" + else: + command = " ".join( + filter( + None, + ( + task_spec.judge.eval_cmd, + task_spec.judge.game_server_cmd, + ), + ) + ) + return bool(_TMP_PATH.search(command)) + + +def make_template_manifest( + task_spec: TaskSpec, + role: str, + source_registry: str, + *, + cpu_count: int | None = None, + memory_mb: int | None = None, +) -> TemplateManifest: + if role not in ("work", "judge"): + raise ValueError("Template role must be 'work' or 'judge'") + image_key = ( + task_spec.work_image_key if role == "work" else task_spec.judge_image_key + ) + spec = task_spec.work if role == "work" else task_spec.judge + snapshot_roots = ( + (_TMP_SNAPSHOT_ROOT,) + if _command_references_tmp(task_spec, role) + else () + ) + resolved_cpu = cpu_count if cpu_count is not None else spec.cpu_limit or 4 + resolved_memory = ( + memory_mb + if memory_mb is not None + else parse_memory_mb( + spec.mem_limit, + default=( + DEFAULT_WORK_MEMORY_MB + if role == "work" + else DEFAULT_JUDGE_MEMORY_MB + ), + ) + ) + if resolved_cpu <= 0: + raise ValueError("Template CPU count must be greater than zero") + if resolved_memory <= 0: + raise ValueError("Template memory must be greater than zero") + core = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "task_id": task_spec.task_id, + "role": role, + "source_image": f"{source_registry.rstrip('/')}/{image_key}", + "source_image_digest": "", + "image_key": image_key, + "name": template_name(image_key), + "tag": template_tag(image_key), + "user": "agent" if role == "work" else "root", + "workdir": task_spec.cwd, + "cpu_count": resolved_cpu, + "memory_mb": resolved_memory, + "environment": [], + "runtime_artifacts": [], + "snapshot_roots": list(snapshot_roots), + } + fingerprint = _manifest_fingerprint(core, role) + return TemplateManifest( + **{ + **core, + "runtime_artifacts": (), + "snapshot_roots": snapshot_roots, + }, + fingerprint=fingerprint, + ) + + +def _restore_command(manifest: TemplateManifest) -> str | None: + if not manifest.runtime_artifacts: + return None + commands = [] + for item in manifest.runtime_artifacts: + parent = str(Path(item.source).parent) + # Merge each captured entry back into place without touching the + # directory root: /tmp keeps its E2B-provisioned runtime files and its + # sticky bit. `cp -aT` treats the destination as the entry itself, so a + # directory that already exists is overwritten in place rather than + # nested under itself, making restore idempotent across restarts. The + # && chain means any failed copy aborts start-up, so a broken restore + # surfaces as a Sandbox that never becomes ready instead of an evaluator + # silently missing a helper. + commands.append( + f"mkdir -p {shlex.quote(parent)} && " + f"cp -aT {shlex.quote(item.persistent)} {shlex.quote(item.source)}" + ) + return " && ".join(commands) + " && sleep infinity" + + +def _replace_runtime_artifacts( + manifest: TemplateManifest, + artifacts: tuple[RuntimeArtifact, ...], +) -> TemplateManifest: + provisional = replace(manifest, runtime_artifacts=artifacts, fingerprint="") + data = asdict(provisional) + data.pop("fingerprint") + fingerprint = _manifest_fingerprint(data, manifest.role) + return replace(provisional, fingerprint=fingerprint) + + +def _replace_image_metadata( + manifest: TemplateManifest, + source_image: str, + environment: tuple[tuple[str, str], ...], +) -> TemplateManifest: + provisional = replace( + manifest, + source_image=source_image, + source_image_digest=source_image.split("@", 1)[1], + environment=environment, + fingerprint="", + ) + data = asdict(provisional) + data.pop("fingerprint") + return replace( + provisional, fingerprint=_manifest_fingerprint(data, manifest.role), + ) + + +def _image_environment(image) -> tuple[tuple[str, str], ...]: + values = {} + for entry in (image.attrs.get("Config") or {}).get("Env") or []: + key, separator, value = entry.partition("=") + normalized = key.lower() + sensitive = any( + marker in normalized + for marker in ( + "api_key", "access_key", "auth", "credential", + "password", "passwd", "private_key", "secret", "token", + ) + ) + host_specific = normalized in { + "http_proxy", "https_proxy", "no_proxy", + } + if ( + separator + and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) + and not sensitive + and not host_specific + ): + values[key] = value + return tuple(sorted(values.items())) + + +_MANIFEST_ACCEPT = ", ".join(( + "application/vnd.oci.image.index.v1+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.docker.distribution.manifest.v2+json", +)) + + +def _registry_location(image_ref: str) -> tuple[str, str, str]: + name, separator, identifier = image_ref.partition("@") + if not separator: + name, separator, identifier = image_ref.rpartition(":") + if not separator or "/" in identifier: + name, identifier = image_ref, "latest" + parts = name.split("/") + if len(parts) > 1 and ("." in parts[0] or ":" in parts[0] or parts[0] == "localhost"): + registry, repository = parts[0], "/".join(parts[1:]) + else: + registry = "registry-1.docker.io" + repository = name if "/" in name else f"library/{name}" + return registry, repository, identifier + + +def _registry_image_environment( + image_ref: str, + *, + attempts: int = 3, + username: str | None = None, + password: str | None = None, +) -> tuple[tuple[str, str], ...]: + """Read OCI Config.Env without downloading image layers.""" + if bool(username) != bool(password): + raise ValueError("Both registry username and password are required") + registry, repository, identifier = _registry_location(image_ref) + base_url = f"https://{registry}/v2/{repository}" + headers = {"Accept": _MANIFEST_ACCEPT} + auth = (username, password) if username and password else None + + def request(path: str): + response = requests.get( + base_url + path, headers=headers, auth=auth, timeout=30, + ) + if response.status_code != 401: + response.raise_for_status() + return response + scheme, _, parameters = response.headers.get("WWW-Authenticate", "").partition(" ") + if scheme.lower() != "bearer": + response.raise_for_status() + challenge = requests.utils.parse_dict_header(parameters) + token_response = requests.get( + challenge["realm"], + params={ + key: challenge[key] + for key in ("service", "scope") + if key in challenge + }, + auth=auth, timeout=30, + ) + token_response.raise_for_status() + token_data = token_response.json() + token = token_data.get("token") or token_data.get("access_token") + if not token: + raise RuntimeError(f"Registry token response for {image_ref!r} had no token") + headers["Authorization"] = f"Bearer {token}" + response = requests.get(base_url + path, headers=headers, timeout=30) + response.raise_for_status() + return response + + error = None + for attempt in range(1, max(1, attempts) + 1): + try: + manifest = request(f"/manifests/{identifier}").json() + if "manifests" in manifest: + descriptor = next( + ( + item for item in manifest["manifests"] + if (item.get("platform") or {}).get("os") == "linux" + and (item.get("platform") or {}).get("architecture") == "amd64" + ), + None, + ) + if descriptor is None: + raise RuntimeError( + f"Image {image_ref!r} has no linux/amd64 manifest" + ) + manifest = request( + f"/manifests/{descriptor['digest']}" + ).json() + config = manifest.get("config") or {} + digest = config.get("digest") + if not isinstance(digest, str) or not digest.startswith("sha256:"): + raise RuntimeError( + f"Image {image_ref!r} has no valid OCI config digest" + ) + raw = request(f"/blobs/{digest}").content + if hashlib.sha256(raw).hexdigest() != digest.removeprefix("sha256:"): + raise RuntimeError(f"OCI config digest mismatch for {image_ref!r}") + config_data = json.loads(raw) + image = type("RegistryImage", (), { + "attrs": {"Config": config_data.get("config") or {}}, + })() + return _image_environment(image) + except Exception as exc: + error = exc + if attempt < max(1, attempts): + time.sleep(min(2 ** (attempt - 1), 8)) + raise RuntimeError( + f"Failed to read OCI config for {image_ref!r} after {attempts} " + f"attempts: {error}" + ) from error + + +def _environment_profile_command( + environment: tuple[tuple[str, str], ...], +) -> str | None: + if not environment: + return None + content = "".join( + f"export {key}={shlex.quote(value)}\n" for key, value in environment + ) + return ( + "mkdir -p /etc/profile.d && printf %s " + f"{shlex.quote(content)} > /etc/profile.d/sforge-image-env.sh && " + "chmod 0644 /etc/profile.d/sforge-image-env.sh" + ) + + +def _runtime_artifact_digest(path: Path) -> str: + """Hash content plus filesystem semantics preserved by cp -a.""" + digest = hashlib.sha256() + members = [path] + if path.is_dir(): + members.extend(sorted(path.rglob("*"))) + for member in members: + relative = "." if member == path else str(member.relative_to(path)) + kind = "dir" if member.is_dir() else "file" + mode = stat.S_IMODE(member.stat().st_mode) + digest.update(f"{relative}\0{kind}\0{mode:o}\0".encode()) + if member.is_file(): + digest.update(member.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def _capture_tmp_snapshot( + container, + context_dir: Path, +) -> tuple[tuple[RuntimeArtifact, ...], dict[str, str]]: + """Snapshot the image's initial /tmp into staged per-entry artifacts. + + The whole directory is captured (not just command-named paths) so evaluator + helpers reached indirectly survive. Each top-level entry becomes its own + artifact so the Sandbox restore merges them back into /tmp without recreating + or clobbering the directory root. Anything unusual — a path escape, a symlink + or special file, or a payload past the ceilings — fails the build loudly + rather than being silently dropped. + """ + try: + stream, _ = container.get_archive(_TMP_SNAPSHOT_ROOT) + archive = b"".join(stream) + except docker.errors.NotFound: + return (), {} + + staging = context_dir / "tmp-snapshot" + staging.mkdir(parents=True, exist_ok=True) + staging_root = staging.resolve() + with tarfile.open(fileobj=io.BytesIO(archive)) as bundle: + members = bundle.getmembers() + if not members: + return (), {} + prefix = members[0].name.split("/", 1)[0] + entry_count = 0 + total_bytes = 0 + keep = [] + for member in members: + relative = member.name[len(prefix):].lstrip("/") + if not relative: + # The /tmp directory itself: E2B provides it, so never recreate + # or re-permission the root; only its contents are restored. + continue + top = relative.split("/", 1)[0] + if top in _SNAPSHOT_EXCLUDE_NAMES: + continue + target = (staging / member.name).resolve() + if staging_root not in target.parents and target != staging_root: + raise ValueError( + f"Unsafe /tmp snapshot path: {member.name!r}" + ) + if member.issym() or member.islnk(): + raise ValueError( + "/tmp snapshot symlinks are not supported: " + f"{member.name!r}" + ) + if not (member.isfile() or member.isdir()): + raise ValueError( + "/tmp snapshot special files are not supported: " + f"{member.name!r}" + ) + entry_count += 1 + if member.isfile(): + total_bytes += member.size + if entry_count > MAX_SNAPSHOT_ENTRIES: + raise ValueError( + f"/tmp snapshot exceeds {MAX_SNAPSHOT_ENTRIES} entries" + ) + if total_bytes > MAX_SNAPSHOT_BYTES: + raise ValueError( + f"/tmp snapshot exceeds {MAX_SNAPSHOT_BYTES} bytes" + ) + keep.append(member) + if not keep: + return (), {} + # Preserve the image's exact modes so the restored /tmp matches what + # Docker and Kubernetes see. The loop above already rejected symlinks, + # hardlinks, special files, and path escapes, so "fully_trusted" adds no + # new exposure here; it also extracts without the Python 3.14 + # extraction-filter deprecation. Extraction runs as the unprivileged + # build user, so setuid/setgid bits cannot escalate on the host. + bundle.extractall(staging, members=keep, filter="fully_trusted") + + source_root = staging / prefix + artifacts = [] + sources = {} + for entry in sorted(source_root.iterdir()): + name = entry.name + source = f"{_TMP_SNAPSHOT_ROOT}/{name}" + artifacts.append( + RuntimeArtifact( + source=source, + persistent=f"{_TMP_SNAPSHOT_PERSISTENT}/{name}", + digest=_runtime_artifact_digest(entry), + ) + ) + sources[source] = str(entry.relative_to(context_dir)) + return tuple(artifacts), sources + + +def stage_runtime_artifacts( + manifest: TemplateManifest, + docker_client, + context_dir: Path, + *, + pull_attempts: int = 3, + portable_registry: str | None = None, + source_registry_username: str | None = None, + source_registry_password: str | None = None, + portable_registry_username: str | None = None, + portable_registry_password: str | None = None, + force_portable: bool = False, +) -> tuple[TemplateManifest, dict[str, str], bool]: + """Resolve the immutable source image and snapshot its initial /tmp. + + Tasks whose command never touches /tmp keep the no-pull fast path: only the + registry digest and OCI environment are resolved, with no image download. + """ + if not manifest.snapshot_roots and not force_portable: + try: + image = docker_client.images.get(manifest.source_image) + except Exception as exc: + if not _is_missing_image_error(exc): + raise + immutable_image = _resolve_registry_digest( + docker_client, + manifest.source_image, + attempts=pull_attempts, + username=source_registry_username, + password=source_registry_password, + ) + environment = _registry_image_environment( + immutable_image, attempts=pull_attempts, + username=source_registry_username, + password=source_registry_password, + ) + resolved = _replace_image_metadata( + manifest, immutable_image, environment, + ) + return _replace_runtime_artifacts(resolved, ()), {}, False + + image = _get_or_pull_image( + docker_client, + manifest.source_image, + attempts=pull_attempts, + username=source_registry_username, + password=source_registry_password, + ) + immutable_source = _matching_repo_digest(image, manifest.source_image) + used_portable_image = force_portable + if used_portable_image: + if not portable_registry: + raise AptSourceError( + f"Source image {manifest.source_image!r} requires APT source " + "normalization. Pass --portable-registry to publish a portable " + "derivative before building the E2B Template." + ) + portable_ref = _portable_image_ref( + portable_registry, + manifest.image_key, + immutable_source.split("@", 1)[1], + ) + immutable_image = _optional_registry_digest( + docker_client, + portable_ref, + attempts=pull_attempts, + username=portable_registry_username, + password=portable_registry_password, + ) + if immutable_image is None: + image, immutable_image = _build_and_push_portable_image( + docker_client, + image, + manifest.source_image, + portable_ref, + context_dir, + username=portable_registry_username, + password=portable_registry_password, + ) + else: + # The Registry manifest is authoritative. A tag may still exist in + # the local Docker cache after its remote manifest was deleted, so + # never derive an E2B source digest from the local tag alone. + image = _get_or_pull_image( + docker_client, + immutable_image, + attempts=pull_attempts, + username=portable_registry_username, + password=portable_registry_password, + ) + else: + immutable_image = immutable_source + manifest = _replace_image_metadata( + manifest, immutable_image, _image_environment(image), + ) + if not manifest.snapshot_roots: + return _replace_runtime_artifacts(manifest, ()), {}, used_portable_image + + container = docker_client.containers.create(manifest.source_image) + try: + artifacts, sources = _capture_tmp_snapshot(container, context_dir) + finally: + container.remove(force=True) + return ( + _replace_runtime_artifacts(manifest, artifacts), + sources, + used_portable_image, + ) + + +def _registry_auth( + username: str | None, + password: str | None, +) -> dict[str, dict[str, str]]: + if not username and not password: + return {} + if not username or not password: + raise ValueError("Both registry username and password are required") + return {"auth_config": {"username": username, "password": password}} + + +def _get_or_pull_image( + docker_client, + image_ref: str, + *, + attempts: int = 3, + username: str | None = None, + password: str | None = None, +): + """Resolve an image locally, retrying transient registry failures.""" + try: + return docker_client.images.get(image_ref) + except Exception as exc: + if not _is_missing_image_error(exc): + raise + + attempts = max(1, attempts) + auth = _registry_auth(username, password) + error = None + for attempt in range(1, attempts + 1): + try: + return docker_client.images.pull(image_ref, **auth) + except Exception as exc: + error = exc + # A failed pull can still leave a complete image in the daemon. + try: + return docker_client.images.get(image_ref) + except Exception as lookup_exc: + if not _is_missing_image_error(lookup_exc): + raise + if attempt < attempts: + time.sleep(min(2 ** (attempt - 1), 8)) + raise RuntimeError( + f"Failed to pull source image {image_ref!r} after {attempts} attempts: {error}" + ) from error + + +def _resolve_registry_digest( + docker_client, + image_ref: str, + *, + attempts: int = 3, + username: str | None = None, + password: str | None = None, +) -> str: + """Resolve an immutable registry digest without downloading layers.""" + attempts = max(1, attempts) + auth = _registry_auth(username, password) + error = None + for attempt in range(1, attempts + 1): + try: + data = docker_client.images.get_registry_data(image_ref, **auth) + digest = data.id + if not digest.startswith("sha256:"): + raise ValueError( + f"Registry returned an invalid digest for {image_ref!r}: {digest!r}" + ) + repository, separator, _ = image_ref.rpartition(":") + if not separator: + repository = image_ref + return f"{repository}@{digest}" + except Exception as exc: + error = exc + if attempt < attempts: + time.sleep(min(2 ** (attempt - 1), 8)) + raise RuntimeError( + f"Failed to resolve source image {image_ref!r} after {attempts} " + f"attempts: {error}" + ) from error + + +def _optional_registry_digest( + docker_client, + image_ref: str, + *, + attempts: int = 3, + username: str | None = None, + password: str | None = None, +) -> str | None: + """Resolve a remote image digest, returning None only for NotFound. + + A local Docker tag is not proof that the corresponding remote manifest + still exists. Authentication and transport failures remain hard errors so + they cannot silently trigger a rebuild or overwrite an inaccessible tag. + """ + attempts = max(1, attempts) + auth = _registry_auth(username, password) + for attempt in range(1, attempts + 1): + try: + data = docker_client.images.get_registry_data(image_ref, **auth) + digest = data.id + if not digest.startswith("sha256:"): + raise ValueError( + f"Registry returned an invalid digest for {image_ref!r}: " + f"{digest!r}" + ) + repository, separator, _ = image_ref.rpartition(":") + if not separator: + repository = image_ref + return f"{repository}@{digest}" + except Exception as exc: + if _is_missing_image_error(exc): + return None + if attempt == attempts: + raise RuntimeError( + f"Failed to inspect portable image {image_ref!r} after " + f"{attempts} attempts: {exc}" + ) from exc + time.sleep(min(2 ** (attempt - 1), 8)) + return None + + +def _portable_image_ref( + portable_registry: str, + image_key: str, + source_digest: str, +) -> str: + repo, sep, tag = image_key.rpartition(":") + if not sep: + raise ValueError(f"Image key has no immutable tag: {image_key!r}") + digest_suffix = source_digest.removeprefix("sha256:")[:12] + if not digest_suffix: + raise ValueError("Source image digest is required for a portable image") + return ( + f"{portable_registry.rstrip('/')}/{repo}:" + f"{tag}-{digest_suffix}-e2b-v{PORTABLE_IMAGE_RECIPE_VERSION}" + ) + + +def _matching_repo_digest(image, image_ref: str) -> str: + repo_digests = image.attrs.get("RepoDigests") or [] + source_repo, separator, _ = image_ref.rpartition(":") + if not separator: + source_repo = image_ref + matching = [ + item + for item in repo_digests + if "@" in item and item.split("@", 1)[0] == source_repo + ] + if not matching: + raise ValueError( + f"Source image {image_ref!r} has no matching registry digest " + f"(available: {repo_digests}); push it to that registry before " + "building an E2B Template" + ) + return matching[0] + + +def _portable_apt_sources_command() -> str: + """Return a distro-aware command that restores public APT sources.""" + debian_components = "main" + ubuntu_components = "main universe" + debian_sources = " ".join( + ( + f'"deb http://deb.debian.org/debian $codename {debian_components}"', + f'"deb http://deb.debian.org/debian $codename-updates {debian_components}"', + f'"deb http://security.debian.org/debian-security ' + f'$codename-security {debian_components}"', + ) + ) + ubuntu_sources = " ".join( + ( + f'"deb http://archive.ubuntu.com/ubuntu $codename {ubuntu_components}"', + f'"deb http://archive.ubuntu.com/ubuntu $codename-updates ' + f'{ubuntu_components}"', + f'"deb http://security.ubuntu.com/ubuntu $codename-security ' + f'{ubuntu_components}"', + ) + ) + return ( + "set -eu; . /etc/os-release; " + "codename=${VERSION_CODENAME:-${UBUNTU_CODENAME:-}}; " + 'test -n "$codename"; ' + "rm -f /etc/apt/sources.list; " + "rm -rf /etc/apt/sources.list.d; " + "mkdir -p /etc/apt/sources.list.d; " + 'case "$ID" in ' + f"debian) printf '%s\\n' {debian_sources} " + "> /etc/apt/sources.list ;; " + f"ubuntu) printf '%s\\n' {ubuntu_sources} " + "> /etc/apt/sources.list ;; " + '*) echo "Unsupported APT distribution: $ID" >&2; exit 1 ;; ' + "esac" + ) + + +def _build_and_push_portable_image( + docker_client, + source_image, + source_ref: str, + portable_ref: str, + context_dir: Path, + *, + username: str | None = None, + password: str | None = None, +) -> tuple[object, str]: + """Publish a source-equivalent image with public APT sources. + + E2B provisions its own packages before user Template instructions run, so + source-list repair must already be present in the imported image. + """ + immutable_source = _matching_repo_digest(source_image, source_ref) + config = source_image.attrs.get("Config") or {} + original_user = config.get("User") or "" + original_workdir = config.get("WorkingDir") or "/" + dockerfile = [ + f"FROM {immutable_source}", + "USER root", + "RUN " + _portable_apt_sources_command(), + ] + if original_user: + dockerfile.append(f"USER {original_user}") + dockerfile.append(f"WORKDIR {original_workdir}") + dockerfile_path = context_dir / "Portable.Dockerfile" + dockerfile_path.write_text("\n".join(dockerfile) + "\n") + + image, logs = docker_client.images.build( + path=str(context_dir), + dockerfile=dockerfile_path.name, + tag=portable_ref, + rm=True, + forcerm=True, + ) + for item in logs: + if item.get("error") or item.get("errorDetail"): + raise RuntimeError( + f"Failed to build portable source image {portable_ref!r}: " + f"{item.get('error') or item['errorDetail'].get('message')}" + ) + + repository, tag = portable_ref.rsplit(":", 1) + pushed_digest = None + push_kwargs = { + "tag": tag, + "stream": True, + "decode": True, + **_registry_auth(username, password), + } + for item in docker_client.images.push(repository, **push_kwargs): + if item.get("error") or item.get("errorDetail"): + raise RuntimeError( + f"Failed to push portable source image {portable_ref!r}: " + f"{item.get('error') or item['errorDetail'].get('message')}" + ) + pushed_digest = (item.get("aux") or {}).get("Digest") or pushed_digest + image.reload() + immutable_portable = ( + f"{repository}@{pushed_digest}" + if pushed_digest + else _matching_repo_digest(image, portable_ref) + ) + return image, immutable_portable + + +def build_template_definition( + manifest: TemplateManifest, + template_cls, + *, + registry_username: str | None = None, + registry_password: str | None = None, + context_dir: Path | None = None, + runtime_sources: dict[str, str] | None = None, +): + template = ( + template_cls(file_context_path=context_dir) if context_dir else template_cls() + ) + auth = {} + if registry_username or registry_password: + if not registry_username or not registry_password: + raise ValueError("Both registry username and password are required") + auth = { + "username": registry_username, + "password": registry_password, + } + builder = template.from_image(manifest.source_image, **auth) + environment = dict(manifest.environment) + if environment: + builder = builder.set_envs(environment) + builder = builder.run_cmd( + _environment_profile_command(manifest.environment), user="root", + ) + runtime_sources = runtime_sources or {} + for item in manifest.runtime_artifacts: + source = runtime_sources.get(item.source) + if source is None: + raise ValueError( + f"Runtime artifact {item.source!r} was not staged from the image" + ) + builder = builder.copy( + source, + item.persistent, + user="root", + force_upload=True, + ) + if manifest.runtime_artifacts: + # The copies above land root-owned, but the Sandbox start command runs + # as manifest.user (agent for work). A non-root user cannot `cp -a` a + # root-owned directory back into /tmp (preserving times/ownership is + # denied), so hand the staged tree to the run user. Modes are preserved, + # so the restored files keep the image's permissions; only ownership + # normalizes to the run identity, which is exactly who reads them. + builder = builder.run_cmd( + f"chown -R {shlex.quote(manifest.user)}:{shlex.quote(manifest.user)} " + f"{shlex.quote(_TMP_SNAPSHOT_PERSISTENT)}", + user="root", + ) + if manifest.role == "work": + # Match the Docker work-image contract. Imported images can preserve + # root-owned subdirectories even when their default user is `agent`. + builder = builder.run_cmd( + f"chown -R agent:agent {shlex.quote(manifest.workdir)} && " + f"chmod -R u+rwX {shlex.quote(manifest.workdir)}", + user="root", + ) + builder = builder.set_user(manifest.user).set_workdir(manifest.workdir) + restore = _restore_command(manifest) + if restore: + ready = " && ".join( + f"test -e {shlex.quote(item.source)}" for item in manifest.runtime_artifacts + ) + builder = builder.set_start_cmd(restore, ready) + return builder + + +def _matching_build_id( + template_cls, name: str, tag: str, fingerprint_tag: str +) -> str | None: + """Return the build shared by the image and manifest aliases.""" + try: + if not template_cls.exists(f"{name}:{tag}"): + return None + if not template_cls.exists(f"{name}:{fingerprint_tag}"): + return None + tags = template_cls.get_tags(name) + except Exception as exc: + raise RuntimeError( + f"Could not inspect E2B Template aliases for {name!r}: {exc}" + ) from exc + builds = {item.tag: item.build_id for item in tags} + image_build = builds.get(tag) + manifest_build = builds.get(fingerprint_tag) + if not image_build or image_build != manifest_build: + return None + return image_build + + +def _is_retryable_build_error(exc: Exception) -> bool: + text = " ".join( + str(item).lower() + for item in (exc, getattr(exc, "__cause__", None)) + if item is not None + ) + deterministic = ( + "provisioning output", + "could not resolve", + "no installation candidate", + ) + if any(marker in text for marker in deterministic): + return False + transient = ( + "timeout", + "timed out", + "connection", + "eof", + "tls", + "temporarily unavailable", + "internal error", + "502", + "503", + "504", + ) + return any(marker in text for marker in transient) + + +def _is_apt_source_error(message: str) -> bool: + """Recognize E2B provisioning failures caused by unreachable APT repos.""" + lowered = message.lower() + return "provisioning output" in lowered and any( + marker in lowered + for marker in ( + "apt", + "could not resolve", + "temporary failure resolving", + "no installation candidate", + ) + ) + + +def _build_failure_message(status) -> str: + reason = getattr(status, "reason", None) + parts = [] + message = getattr(reason, "message", None) + if message: + parts.append(message) + for entry in getattr(reason, "log_entries", None) or []: + text = getattr(entry, "message", None) + if text: + parts.append(text) + parts.extend(str(item) for item in (getattr(status, "logs", None) or [])) + return "\n".join(parts)[-12000:] or "Build failed" + + +def build_e2b_template( + manifest: TemplateManifest, + *, + template_cls=None, + attempts: int = 3, + build_timeout: int = 600, + force: bool = False, + on_build_logs=None, + registry_username: str | None = None, + registry_password: str | None = None, + context_dir: Path | None = None, + runtime_sources: dict[str, str] | None = None, +): + """Build idempotently, reconciling ambiguous API failures by tag.""" + if attempts <= 0: + raise ValueError("Template build attempts must be greater than zero") + if build_timeout <= 0: + raise ValueError("Template build timeout must be greater than zero") + if template_cls is None: + from e2b import Template as template_cls + fingerprint_tag = f"sforge-{manifest.fingerprint[:12]}" + previous_ready_build = _matching_build_id( + template_cls, + manifest.name, + manifest.tag, + fingerprint_tag, + ) + if previous_ready_build and not force: + return {"status": "existing", "build_id": previous_ready_build} + definition = build_template_definition( + manifest, + template_cls, + registry_username=registry_username, + registry_password=registry_password, + context_dir=context_dir, + runtime_sources=runtime_sources, + ) + error = None + used_attempts = 0 + for attempt in range(1, attempts + 1): + used_attempts = attempt + try: + info = template_cls.build_in_background( + definition, + manifest.name, + tags=[manifest.tag, fingerprint_tag], + cpu_count=manifest.cpu_count, + memory_mb=manifest.memory_mb, + skip_cache=force, + on_build_logs=on_build_logs, + ) + except Exception as exc: + if _is_apt_source_error(str(exc)): + raise AptSourceError(str(exc)) from exc + error = exc + accepted = _matching_build_id( + template_cls, + manifest.name, + manifest.tag, + fingerprint_tag, + ) + if accepted and accepted != previous_ready_build: + return {"status": "reconciled", "build_id": accepted} + if attempt < attempts and _is_retryable_build_error(exc): + time.sleep(min(2 ** (attempt - 1), 8)) + continue + break + + deadline = time.monotonic() + build_timeout + logs_offset = 0 + last_status_error = None + while time.monotonic() < deadline: + try: + status = template_cls.get_build_status( + info, + logs_offset=logs_offset, + ) + last_status_error = None + except Exception as exc: + last_status_error = exc + if not _is_retryable_build_error(exc): + raise RuntimeError( + f"Could not inspect E2B template build " + f"{info.template_id}/{info.build_id}: {exc}" + ) from exc + time.sleep(1) + continue + logs_offset += len(status.log_entries) + if on_build_logs is not None: + for entry in status.log_entries: + on_build_logs(entry) + state = getattr(status.status, "value", status.status) + if state == "ready": + return { + "status": "built", + "template_id": info.template_id, + "build_id": info.build_id, + } + if state == "error": + message = _build_failure_message(status) + if _is_apt_source_error(message): + raise AptSourceError(message) + error = RuntimeError(message) + accepted = _matching_build_id( + template_cls, manifest.name, manifest.tag, fingerprint_tag, + ) + if accepted and accepted != previous_ready_build: + return {"status": "reconciled", "build_id": accepted} + if attempt < attempts and _is_retryable_build_error(error): + time.sleep(min(2 ** (attempt - 1), 8)) + break + raise error + time.sleep(1) + else: + accepted = _matching_build_id( + template_cls, + manifest.name, + manifest.tag, + fingerprint_tag, + ) + if accepted == info.build_id: + return {"status": "reconciled", "build_id": accepted} + detail = ( + f": last status error: {last_status_error}" + if last_status_error else "" + ) + raise TimeoutError( + f"E2B template build timed out after {build_timeout}s " + f"(template_id={info.template_id}, build_id={info.build_id}); " + f"the E2B SDK has no per-build cancellation API{detail}" + ) + raise RuntimeError( + f"Failed to build E2B template {manifest.reference!r} after " + f"{used_attempts} attempts: {error}" + ) from error + + +def write_template_manifest( + manifest: TemplateManifest, + result: dict, + output_dir: Path, +) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + path = output_dir / f"{manifest.role}.json" + path.write_text( + json.dumps( + {**manifest.to_dict(), "result": result}, + indent=2, + ensure_ascii=False, + ) + ) + return path diff --git a/sforge/harness/evolve_scripts.py b/sforge/harness/evolve_scripts.py index 14d8f30..bf1fd67 100644 --- a/sforge/harness/evolve_scripts.py +++ b/sforge/harness/evolve_scripts.py @@ -81,6 +81,7 @@ def _generate_submit_script() -> str: JUDGE_URL="${SFORGE_JUDGE_URL}" TOKEN="${SFORGE_TOKEN}" +ACCESS_TOKEN="${SFORGE_JUDGE_ACCESS_TOKEN:-}" PATCH_DIR="${SFORGE_PATCH_DIR:-$(pwd)}" STATE_FILE="/tmp/sforge_state.json" @@ -93,8 +94,14 @@ def _generate_submit_script() -> str: exit 1 fi +CURL_AUTH=() +if [ -n "$ACCESS_TOKEN" ]; then + CURL_AUTH=(-H "e2b-traffic-access-token: $ACCESS_TOKEN") +fi +CURL_SESSION=(-H "Authorization: Bearer $TOKEN") + if [ "$LIST_MODE" -eq 1 ]; then - HIST=$(curl -s -m 30 "$JUDGE_URL/api/v1/history?token=$TOKEN") + HIST=$(curl -s -m 30 "${CURL_AUTH[@]}" "${CURL_SESSION[@]}" "$JUDGE_URL/api/v1/history") if [ -z "$HIST" ]; then echo "ERROR: No response from judge server" >&2 exit 1 @@ -158,10 +165,12 @@ def _generate_submit_script() -> str: # ── Submit + poll ── HTTP_CODE=$(curl -s -o /tmp/_submit_resp.json -w '%{http_code}' -m 120 -X POST "$JUDGE_URL/api/v1/submit" \ - -F "token=$TOKEN" \ + "${CURL_AUTH[@]}" \ + "${CURL_SESSION[@]}" \ -F "archive=@$ARCHIVE_FILE") rm -f "$ARCHIVE_FILE" SUBMIT_RESP=$(cat /tmp/_submit_resp.json) +rm -f /tmp/_submit_resp.json if [ "$HTTP_CODE" = "429" ]; then DETAIL=$(echo "$SUBMIT_RESP" | jq -r '.detail // empty') @@ -195,7 +204,7 @@ def _generate_submit_script() -> str: for _ in $(seq 1 720); do sleep 10 - RESULT=$(curl -s -m 30 "$JUDGE_URL/api/v1/result/$SUBMISSION_ID" 2>/dev/null || true) + RESULT=$(curl -s -m 30 "${CURL_AUTH[@]}" "${CURL_SESSION[@]}" "$JUDGE_URL/api/v1/result/$SUBMISSION_ID" 2>/dev/null || true) STATUS=$(echo "$RESULT" | jq -r '.status // empty') if [ "$STATUS" = "completed" ] || [ "$STATUS" = "error" ]; then break @@ -419,7 +428,10 @@ def generate_game_prompt(original_query: str, internet: bool = True) -> str: "---\n\n" f"{network_note}" "## Game Server HTTP API\n\n" - "The server URL is available in the `GAME_SERVER_URL` environment variable.\n\n" + "The server URL is available in the `GAME_SERVER_URL` environment variable. " + "Every request must include `Authorization: Bearer $SFORGE_TOKEN`; when " + "`SFORGE_JUDGE_ACCESS_TOKEN` is set, also include " + "`e2b-traffic-access-token: $SFORGE_JUDGE_ACCESS_TOKEN`.\n\n" "### Start a new game\n" "```\n" "POST {GAME_SERVER_URL}/new\n" diff --git a/sforge/harness/game_server_app.py b/sforge/harness/game_server_app.py index 8bc54b7..ba5dc63 100644 --- a/sforge/harness/game_server_app.py +++ b/sforge/harness/game_server_app.py @@ -24,11 +24,14 @@ from __future__ import annotations import argparse +import hmac +import os import sys import threading import uvicorn -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse from pydantic import BaseModel try: @@ -78,6 +81,17 @@ def create_app(rom_path: str) -> FastAPI: app = FastAPI(title="SForge Game Server (Container)") state = GameState(rom_path) + # On backends where this port is exposed through a public gateway (e2b), + # only the judge server knows the per-session token it set in our env. + token = os.environ.get("SFORGE_GAME_TOKEN", "") + if token: + @app.middleware("http") + async def require_token(request: Request, call_next): + got = request.headers.get("x-sforge-game-token", "") + if not hmac.compare_digest(got, token): + return JSONResponse({"detail": "unauthorized"}, status_code=401) + return await call_next(request) + @app.get("/health") def health() -> dict: return {"ok": True} diff --git a/sforge/harness/judge_client.py b/sforge/harness/judge_client.py new file mode 100644 index 0000000..aacd424 --- /dev/null +++ b/sforge/harness/judge_client.py @@ -0,0 +1,102 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One transport contract for every SForge Judge HTTP request.""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable +from dataclasses import dataclass, field + +import requests + +E2B_TRAFFIC_ACCESS_HEADER = "e2b-traffic-access-token" +SFORGE_ADMIN_SECRET_HEADER = "X-SForge-Admin-Secret" + +logger = logging.getLogger(__name__) + +E2B_CONNECT_ATTEMPTS = 3 +E2B_CONNECT_TIMEOUT = 10 + + +@dataclass(frozen=True) +class JudgeConnection: + url: str + headers: dict[str, str] = field(default_factory=dict) + connect_attempts: int = 1 + request_handler: Callable[..., requests.Response] | None = field( + default=None, repr=False, compare=False, + ) + + @classmethod + def from_e2b_traffic_token( + cls, + url: str, + token: str | None, + ) -> JudgeConnection: + """Create a Judge connection through an optional secured E2B gateway.""" + headers = {E2B_TRAFFIC_ACCESS_HEADER: token} if token else {} + return cls( + url=url.rstrip("/"), + headers=headers, + connect_attempts=E2B_CONNECT_ATTEMPTS if token else 1, + ) + + def request(self, method: str, path: str, **kwargs): + headers = {**self.headers, **(kwargs.pop("headers", None) or {})} + if self.request_handler is not None: + return self.request_handler( + method, path, headers=headers, **kwargs, + ) + url = f"{self.url}/{path.lstrip('/')}" + timeout = kwargs.get("timeout") + if self.connect_attempts > 1 and isinstance(timeout, (int, float)): + # A scalar applies to both connect and response reads. Keep the + # caller's response budget, but fail fast enough to retry a flaky + # E2B public-gateway connection. A ConnectTimeout happens before + # the request is sent, so retrying POST is safe here. + kwargs["timeout"] = (min(E2B_CONNECT_TIMEOUT, timeout), timeout) + + for attempt in range(1, self.connect_attempts + 1): + try: + return requests.request( + method, url, headers=headers, **kwargs, + ) + except requests.ConnectTimeout: + if attempt == self.connect_attempts: + raise + logger.warning( + "E2B Judge gateway connect timed out for %s %s " + "(attempt %d/%d); retrying", + method, path, attempt, self.connect_attempts, + ) + time.sleep(attempt) + raise AssertionError("unreachable") + + def agent_env(self) -> dict[str, str]: + env = {"SFORGE_JUDGE_URL": self.url} + token = self.headers.get(E2B_TRAFFIC_ACCESS_HEADER) + if token: + env["SFORGE_JUDGE_ACCESS_TOKEN"] = token + return env + + +def merge_no_proxy(current: str, hostname: str) -> str: + """Append one exact hostname without substring-based false matches.""" + entries = [item.strip() for item in current.split(",") if item.strip()] + if hostname and hostname not in entries: + entries.append(hostname) + return ",".join(entries) diff --git a/sforge/harness/judge_server.py b/sforge/harness/judge_server.py index 0135830..6594a2d 100644 --- a/sforge/harness/judge_server.py +++ b/sforge/harness/judge_server.py @@ -28,16 +28,23 @@ from typing import Any import requests -from fastapi import FastAPI, File, Form, HTTPException, Query, UploadFile +from fastapi import FastAPI, File, Form, Header, HTTPException, UploadFile from pydantic import BaseModel from sforge.harness.backend import ContainerBackend -from sforge.harness.config import SForgeConfig, create_backend_from_config, get_container_env, load_config -from sforge.harness.constants import ADMIN_SECRET +from sforge.harness.benchmark import load_benchmark +from sforge.harness.config import ( + SForgeConfig, + create_backend_from_config, + get_container_env, + load_config, + sanitize_e2b_proxy_env, +) +from sforge.harness.constants import get_admin_secret from sforge.harness.docker_build import BuildImageError, build_judge_image +from sforge.harness.judge_client import SFORGE_ADMIN_SECRET_HEADER from sforge.harness.run_evaluation import judge_submission from sforge.harness.selection import select_best -from sforge.harness.benchmark import load_benchmark from sforge.harness.task_spec import TaskSpec, load_all_tasks logger = logging.getLogger("sforge.judge_server") @@ -118,7 +125,6 @@ class GameCloseResponse(BaseModel): class RegisterRequest(BaseModel): task_id: str run_id: str - admin_secret: str = "" judge_cpu_limit: int | None = None judge_mem_limit: str | None = None backend: str | None = None @@ -128,6 +134,8 @@ class RegisterRequest(BaseModel): k8s_kubeconfig: str | None = None max_agent_submissions: int | None = None submission_cooldown: int | None = None + e2b_template_map: dict[str, str] | None = None + e2b_sandbox_ttl: int | None = None class RegisterResponse(BaseModel): @@ -150,6 +158,8 @@ class SubmitResponse(BaseModel): GAME_CONTAINER_READY_TIMEOUT = 30 GAME_CONTAINER_READY_POLL = 0.5 GAME_CONTAINER_PORT = 8000 +GAME_GATEWAY_ATTEMPTS = 3 +GAME_GATEWAY_BACKOFF = 1.0 @dataclass @@ -160,6 +170,7 @@ class GameSessionState: game_num: int container: object # ContainerHandle container_url: str + headers: dict = field(default_factory=dict) max_score: int = 0 peak_score: int = 0 current_score: int = 0 @@ -176,13 +187,14 @@ class GameSessionState: class JudgeState: def __init__(self, config: SForgeConfig): self.config = config + self.admin_secret = get_admin_secret(config.log_dir) self.tasks: dict[str, TaskSpec] = {} self.submissions: dict[str, dict] = {} # submission_id -> {status, report, ...} self.game_sessions: dict[str, GameSessionState] = {} self.run_history: dict[str, list[dict]] = {} # run_id -> [entries] self.tokens: dict[str, dict] = {} # token -> {task_id, run_id, next_agent, next_auto, judge_cpu_limit, judge_mem_limit} - self.run_resource_limits: dict[str, dict] = {} # run_id -> {judge_cpu_limit, judge_mem_limit} - self.run_backends: dict[str, ContainerBackend] = {} # run_id -> backend + self.run_resource_limits: dict[str, dict] = {} # run/task -> limits + self.run_backends: dict[str, ContainerBackend] = {} # run/task -> backend self._game_lock = threading.Lock() self._history_lock = threading.Lock() self._tokens_lock = threading.Lock() @@ -214,12 +226,15 @@ def _destroy_game_session(self, session_id: str) -> None: sess = self.game_sessions.pop(session_id, None) if sess is None: return - self._archive_game_session(sess) try: - backend = self._get_backend(sess.run_id) + backend = self._get_backend(sess.run_id, sess.task_id) backend.cleanup_container(sess.container, logger) except Exception: logger.exception("Error cleaning up game container for session %s", session_id) + with self._game_lock: + self.game_sessions.setdefault(session_id, sess) + return + self._archive_game_session(sess) def _game_log_dir(self, sess: GameSessionState) -> Path: return ( @@ -270,7 +285,9 @@ def register_session(self, task_id: str, run_id: str, k8s_node_selector: dict[str, str] | None = None, k8s_kubeconfig: str | None = None, max_agent_submissions: int | None = None, - submission_cooldown: int | None = None) -> str: + submission_cooldown: int | None = None, + e2b_template_map: dict[str, str] | None = None, + e2b_sandbox_ttl: int | None = None) -> str: if task_id not in self.tasks: raise ValueError(f"Unknown task: {task_id}") token = secrets.token_hex(16) @@ -286,25 +303,37 @@ def register_session(self, task_id: str, run_id: str, "submission_cooldown": submission_cooldown, "last_agent_submit_at": 0.0, } - self.run_resource_limits[run_id] = { + run_key = self._run_key(run_id, task_id) + self.run_resource_limits[run_key] = { "judge_cpu_limit": judge_cpu_limit, "judge_mem_limit": judge_mem_limit, } if backend and backend != self.backend.backend_name: from sforge.harness.backend.factory import create_backend - self.run_backends[run_id] = create_backend( + self.run_backends[run_key] = create_backend( backend, k8s_namespace=k8s_namespace or self.config.k8s_namespace, k8s_node_selector=k8s_node_selector or self.config.k8s_node_selector, k8s_image_registry=k8s_image_registry or self.config.k8s_image_registry, k8s_kubeconfig=k8s_kubeconfig or self.config.k8s_kubeconfig, + e2b_template_map=e2b_template_map or self.config.e2b_template_map, + e2b_sandbox_ttl=e2b_sandbox_ttl or self.config.e2b_sandbox_ttl, ) return token - def _get_backend(self, run_id: str | None = None) -> ContainerBackend: - if run_id and run_id in self.run_backends: - return self.run_backends[run_id] + @staticmethod + def _run_key(run_id: str, task_id: str) -> str: + return f"{run_id}/{task_id}" + + def _get_backend( + self, run_id: str | None = None, task_id: str | None = None, + ) -> ContainerBackend: + if run_id and task_id: + backend = self.run_backends.get(self._run_key(run_id, task_id)) + if backend is not None: + return backend return self.backend + def resolve_token(self, token: str) -> dict: with self._tokens_lock: info = self.tokens.get(token) @@ -312,6 +341,15 @@ def resolve_token(self, token: str) -> dict: raise KeyError("Invalid token") return dict(info) + def require_session( + self, token: str, run_id: str, task_id: str, + ) -> dict: + """Resolve a token and bind it to the route's run/task scope.""" + info = self.resolve_token(token) + if info["run_id"] != run_id or info["task_id"] != task_id: + raise KeyError("Session token does not match run/task") + return info + def consume_round(self, token: str, kind: str = "agent") -> tuple[str, str, str, int | None, str | None, int | None]: """Allocate the next round ID for a token. Returns (task_id, run_id, round_id, cpu, mem, remaining). @@ -467,7 +505,7 @@ def _grade_worker(self, submission_id: str, task_id: str, archive: bytes, task_spec=task_spec, archive=archive, config=config, - backend=self._get_backend(run_id), + backend=self._get_backend(run_id, task_id), submission_id=submission_id, log_dir=sub_log_dir, ) @@ -514,7 +552,7 @@ def create_game_session(self, run_id: str, task_id: str) -> tuple[GameSessionSta logger.info("Evicting oldest game session %s for %s/%s", sess.session_id, run_id, task_id) self._destroy_game_session(sess.session_id) - backend = self._get_backend(run_id) + backend = self._get_backend(run_id, task_id) if backend.backend_name == "docker": from sforge.harness.backend.docker_backend import DockerBackend @@ -525,8 +563,16 @@ def create_game_session(self, run_id: str, task_id: str) -> tuple[GameSessionSta container_name = f"sforge.game.{task_id}.{session_id}" port = GAME_CONTAINER_PORT env = get_container_env(self.config, include_judge_extra=True) - - rl = self.run_resource_limits.get(run_id, {}) + game_headers = {} + if backend.backend_name == "e2b": + sanitize_e2b_proxy_env(env) + # E2B exposes the service through a public gateway. Protect the + # in-sandbox app as well as the outer Judge route. + game_token = uuid.uuid4().hex + env["SFORGE_GAME_TOKEN"] = game_token + game_headers["X-SForge-Game-Token"] = game_token + + rl = self.run_resource_limits.get(self._run_key(run_id, task_id), {}) cpu = rl.get("judge_cpu_limit") mem = rl.get("judge_mem_limit") # fall back to task defaults @@ -534,76 +580,108 @@ def create_game_session(self, run_id: str, task_id: str) -> tuple[GameSessionSta cpu = task_spec.judge.cpu_limit if mem is None: mem = task_spec.judge.mem_limit - handle = backend.create_container( - task_spec.judge_image_key, - container_name, - environment=env, - cpu_limit=cpu, - mem_limit=mem, - ) - backend.start_container(handle) - logger.info("Game container started: %s (%s)", container_name, handle.id[:12]) - - backend.copy_to_container(handle, GAME_SERVER_APP_PATH, PurePosixPath("/tmp/game_server_app.py")) - - backend.exec_run( - handle, - f"/bin/bash -c '{task_spec.judge.game_server_cmd} &'", - detach=True, - ) + handle = None + committed = False + try: + handle = backend.create_container( + task_spec.judge_image_key, + container_name, + environment=env, + cpu_limit=cpu, + mem_limit=mem, + ) + backend.start_container(handle) + logger.info( + "Game container started: %s (%s)", container_name, handle.id[:12], + ) - container_ip = backend.get_container_ip(handle) - if not container_ip: - backend.cleanup_container(handle, logger) - raise RuntimeError("Cannot determine container IP address") + backend.copy_to_container( + handle, GAME_SERVER_APP_PATH, + PurePosixPath("/tmp/game_server_app.py"), + ) + backend.exec_run( + handle, f"/bin/bash -c '{task_spec.judge.game_server_cmd} &'", + detach=True, + ) - container_url = f"http://{container_ip}:{port}" + endpoint = backend.get_service_endpoint(handle, port) + container_url = endpoint.url + game_headers = {**endpoint.headers, **game_headers} - deadline = time.time() + GAME_CONTAINER_READY_TIMEOUT - while time.time() < deadline: - try: - resp = requests.get(f"{container_url}/health", timeout=2) - if resp.status_code == 200: + deadline = time.time() + GAME_CONTAINER_READY_TIMEOUT + while time.time() < deadline: + try: + resp = requests.get( + f"{container_url}/health", headers=game_headers, timeout=5, + ) + if resp.status_code == 200: + break + except requests.RequestException: + pass + time.sleep(GAME_CONTAINER_READY_POLL) + else: + raise RuntimeError("Game container did not become ready in time") + + gateway_attempts = ( + GAME_GATEWAY_ATTEMPTS if backend.backend_name == "e2b" else 1 + ) + for attempt in range(1, gateway_attempts + 1): + try: + resp = requests.post( + f"{container_url}/new", json={}, headers=game_headers, + timeout=10, + ) + resp.raise_for_status() + data = resp.json() break - except requests.ConnectionError: - pass - time.sleep(GAME_CONTAINER_READY_POLL) - else: - backend.cleanup_container(handle, logger) - raise RuntimeError("Game container did not become ready in time") - - resp = requests.post( - f"{container_url}/new", - json={}, - timeout=10, - ) - resp.raise_for_status() - data = resp.json() - - with self._history_lock: - history_key = f"{run_id}/{task_id}" - archived = sum(1 for e in self.run_history.get(history_key, []) if e.get("type") == "game") - with self._game_lock: - active = sum(1 for s in self.game_sessions.values() if s.run_id == run_id) - game_num = archived + active + 1 - - sess = GameSessionState( - session_id=session_id, - run_id=run_id, - task_id=task_id, - game_num=game_num, - container=handle, - container_url=container_url, - max_score=data.get("max_score", 0), - peak_score=data.get("peak_score", 0), - current_score=data.get("score", 0), - moves=data.get("moves", 0), - done=data.get("done", False), - ) - with self._game_lock: - self.game_sessions[session_id] = sess - - return sess, data + except (requests.RequestException, ValueError) as exc: + if attempt == gateway_attempts: + raise RuntimeError( + "Game gateway failed while creating a session after " + f"{gateway_attempts} attempts: {exc}" + ) from exc + time.sleep(GAME_GATEWAY_BACKOFF * attempt) + + with self._history_lock: + history_key = f"{run_id}/{task_id}" + archived = sum( + 1 for e in self.run_history.get(history_key, []) + if e.get("type") == "game" + ) + with self._game_lock: + active = sum( + 1 for s in self.game_sessions.values() if s.run_id == run_id + ) + game_num = archived + active + 1 + + sess = GameSessionState( + session_id=session_id, + run_id=run_id, + task_id=task_id, + game_num=game_num, + container=handle, + container_url=container_url, + headers=game_headers, + max_score=data.get("max_score", 0), + peak_score=data.get("peak_score", 0), + current_score=data.get("score", 0), + moves=data.get("moves", 0), + done=data.get("done", False), + ) + with self._game_lock: + self.game_sessions[session_id] = sess + committed = True + return sess, data + except Exception as original_error: + if handle is not None and not committed: + try: + backend.cleanup_container(handle, logger) + except Exception as cleanup_error: + raise RuntimeError( + "Game container initialization failed and cleanup also " + f"failed: {cleanup_error}" + ) from original_error + raise def game_step(self, session_id: str, action: str) -> dict: with self._game_lock: @@ -614,6 +692,7 @@ def game_step(self, session_id: str, action: str) -> dict: resp = requests.post( f"{sess.container_url}/step", json={"action": action}, + headers=sess.headers, timeout=10, ) resp.raise_for_status() @@ -651,12 +730,24 @@ def _auto_close_session(self, session_id: str) -> None: sess = self.game_sessions.pop(session_id, None) if sess is None: return - self._archive_game_session(sess) + backend = self._get_backend(sess.run_id, sess.task_id) + if backend.backend_name != "e2b": + self._archive_game_session(sess) + try: + backend.cleanup_container(sess.container, logger) + except Exception: + logger.exception( + "Error auto-closing game session %s", session_id, + ) + return try: - backend = self._get_backend(sess.run_id) backend.cleanup_container(sess.container, logger) except Exception: logger.exception("Error auto-closing game session %s", session_id) + with self._game_lock: + self.game_sessions.setdefault(session_id, sess) + return + self._archive_game_session(sess) logger.info("Auto-closed done game session %s (max_score=%d)", session_id, sess.max_score) def game_status(self, session_id: str) -> dict: @@ -665,7 +756,7 @@ def game_status(self, session_id: str) -> dict: if sess is None: raise KeyError(f"Game session not found: {session_id}") - resp = requests.get(f"{sess.container_url}/status", timeout=5) + resp = requests.get(f"{sess.container_url}/status", headers=sess.headers, timeout=5) resp.raise_for_status() data = resp.json() @@ -685,7 +776,7 @@ def close_game_session(self, session_id: str) -> dict: raise KeyError(f"Game session not found: {session_id}") try: - resp = requests.post(f"{sess.container_url}/close", timeout=10) + resp = requests.post(f"{sess.container_url}/close", headers=sess.headers, timeout=10) resp.raise_for_status() data = resp.json() except Exception: @@ -695,13 +786,24 @@ def close_game_session(self, session_id: str) -> dict: "max_score": sess.max_score, "moves": sess.moves, } - finally: + backend = self._get_backend(sess.run_id, sess.task_id) + if backend.backend_name != "e2b": self._archive_game_session(sess) try: - backend = self._get_backend(sess.run_id) backend.cleanup_container(sess.container, logger) except Exception: - logger.exception("Error cleaning up game container for session %s", session_id) + logger.exception( + "Error cleaning up game container for session %s", + session_id, + ) + return data + try: + backend.cleanup_container(sess.container, logger) + except Exception: + with self._game_lock: + self.game_sessions.setdefault(session_id, sess) + raise + self._archive_game_session(sess) return data @@ -711,18 +813,37 @@ def close_all_game_sessions(self, run_id: str, task_id: str) -> int: (sid, sess) for sid, sess in self.game_sessions.items() if sess.run_id == run_id and sess.task_id == task_id ] - # Phase 1: archive all sessions synchronously (fast — JSON write + dict - # append). This is what later /history calls will read, so it must - # complete before we return. + backend = self._get_backend(run_id, task_id) + if backend.backend_name == "e2b": + with self._game_lock: + for sid, _ in sessions: + self.game_sessions.pop(sid, None) + failures = [] + for sid, sess in sessions: + try: + backend.cleanup_container(sess.container, logger) + except Exception as exc: + logger.exception( + "Error cleaning up game container for session %s", sid, + ) + failures.append((sid, exc)) + with self._game_lock: + self.game_sessions.setdefault(sid, sess) + continue + self._archive_game_session(sess) + if failures: + detail = ", ".join(f"{sid}: {exc}" for sid, exc in failures) + raise RuntimeError(f"Failed to clean up game containers: {detail}") + return len(sessions) + + # Preserve the existing non-blocking Docker/K8s cleanup behavior. containers_to_cleanup = [] for sid, sess in sessions: with self._game_lock: self.game_sessions.pop(sid, None) self._archive_game_session(sess) containers_to_cleanup.append((sid, sess.container)) - # Phase 2: docker cleanup is slow (stop+rm per container). Run it in - # a background thread so we can return to the caller immediately. - backend = self._get_backend(run_id) + def _bg_cleanup(): for sid, container in containers_to_cleanup: try: @@ -745,12 +866,52 @@ def create_app(config: SForgeConfig | None = None) -> FastAPI: state = JudgeState(config) state.load_tasks() + def require_bearer_token(authorization: str | None) -> str: + scheme, _, token = (authorization or "").partition(" ") + if scheme.lower() != "bearer" or not token: + raise HTTPException(status_code=401, detail="Bearer token required") + return token + + def require_game_session( + authorization: str | None, run_id: str, task_id: str, + ) -> str: + token = require_bearer_token(authorization) + try: + state.require_session(token, run_id, task_id) + except KeyError: + raise HTTPException(status_code=401, detail="Invalid session token") + return token + + def require_game_scope( + session_id: str, run_id: str, task_id: str, + ) -> GameSessionState: + with state._game_lock: + sess = state.game_sessions.get(session_id) + if sess is None: + raise HTTPException(404, f"Game session not found: {session_id}") + if sess.run_id != run_id or sess.task_id != task_id: + raise HTTPException(404, f"Game session not found: {session_id}") + return sess + @app.get("/api/v1/result/{submission_id}") - def get_result(submission_id: str) -> ResultResponse: + def get_result( + submission_id: str, + authorization: str | None = Header(default=None), + ) -> ResultResponse: """Get result of an async submission.""" result = state.get_result(submission_id) if result is None: raise HTTPException(status_code=404, detail="Submission not found") + token = require_bearer_token(authorization) + try: + info = state.resolve_token(token) + except KeyError: + raise HTTPException(status_code=401, detail="Invalid token") + if ( + result.get("run_id") != info["run_id"] + or result.get("task_id") != info["task_id"] + ): + raise HTTPException(status_code=404, detail="Submission not found") return ResultResponse( submission_id=submission_id, status=result["status"], @@ -759,9 +920,14 @@ def get_result(submission_id: str) -> ResultResponse: ) @app.post("/api/v1/register") - def register(req: RegisterRequest) -> RegisterResponse: + def register( + req: RegisterRequest, + admin_secret: str | None = Header( + default=None, alias=SFORGE_ADMIN_SECRET_HEADER, + ), + ) -> RegisterResponse: """Register a session and get a token for submissions.""" - if req.admin_secret != ADMIN_SECRET: + if admin_secret != state.admin_secret: raise HTTPException(status_code=403, detail="Invalid admin secret") try: token = state.register_session( @@ -775,6 +941,8 @@ def register(req: RegisterRequest) -> RegisterResponse: k8s_kubeconfig=req.k8s_kubeconfig, max_agent_submissions=req.max_agent_submissions, submission_cooldown=req.submission_cooldown, + e2b_template_map=req.e2b_template_map, + e2b_sandbox_ttl=req.e2b_sandbox_ttl, ) return RegisterResponse(token=token) except ValueError as e: @@ -782,19 +950,22 @@ def register(req: RegisterRequest) -> RegisterResponse: @app.post("/api/v1/submit") def submit( - token: str = Form(...), archive: UploadFile = File(...), kind: str = Form("agent"), - admin_secret: str = Form(""), + authorization: str | None = Header(default=None), + admin_secret: str | None = Header( + default=None, alias=SFORGE_ADMIN_SECRET_HEADER, + ), ) -> SubmitResponse: """Submit an archive using a session token. The server resolves task_id, run_id, and assigns a round_id. kind=auto requires a valid admin_secret (host-side only). """ + token = require_bearer_token(authorization) if kind not in ("agent", "auto"): raise HTTPException(status_code=400, detail="kind must be 'agent' or 'auto'") - if kind == "auto" and admin_secret != ADMIN_SECRET: + if kind == "auto" and admin_secret != state.admin_secret: raise HTTPException(status_code=403, detail="admin_secret required for auto submissions") try: task_id, run_id, round_id, judge_cpu_limit, judge_mem_limit, remaining = state.consume_round(token, kind) @@ -819,18 +990,24 @@ def submit( raise HTTPException(status_code=404, detail=str(e)) @app.get("/api/v1/history") - def history(token: str = Query(...), admin_secret: str = Query("")) -> dict: + def history( + authorization: str | None = Header(default=None), + admin_secret: str | None = Header( + default=None, alias=SFORGE_ADMIN_SECRET_HEADER, + ), + ) -> dict: """Get run history using a session token. Without admin_secret: returns only agent submissions (agent-visible view). With valid admin_secret: returns all entries including auto-eval (host-side view). """ + token = require_bearer_token(authorization) try: info = state.resolve_token(token) except KeyError: raise HTTPException(status_code=401, detail="Invalid token") full = state.get_run_history(info["run_id"], info["task_id"]) - if admin_secret == ADMIN_SECRET: + if admin_secret == state.admin_secret: return full # Filter to agent-only view: hide auto-eval entries agent_entries = [ @@ -855,8 +1032,12 @@ def history(token: str = Query(...), admin_secret: str = Query("")) -> dict: # --- Game session routes --- @app.post("/api/v1/game/{run_id}/{task_id}/new") - def game_new(run_id: str, task_id: str, req: GameNewRequest) -> GameNewResponse: + def game_new( + run_id: str, task_id: str, req: GameNewRequest, + authorization: str | None = Header(default=None), + ) -> GameNewResponse: """Start a new game session in a dedicated container.""" + require_game_session(authorization, run_id, task_id) try: sess, data = state.create_game_session(run_id, task_id) except ValueError as e: @@ -874,7 +1055,12 @@ def game_new(run_id: str, task_id: str, req: GameNewRequest) -> GameNewResponse: ) @app.post("/api/v1/game/{run_id}/{task_id}/{session_id}/step") - def game_step(run_id: str, task_id: str, session_id: str, req: GameStepRequest) -> GameStepResponse: + def game_step( + run_id: str, task_id: str, session_id: str, req: GameStepRequest, + authorization: str | None = Header(default=None), + ) -> GameStepResponse: + require_game_session(authorization, run_id, task_id) + require_game_scope(session_id, run_id, task_id) try: data = state.game_step(session_id, req.action) except KeyError: @@ -892,7 +1078,12 @@ def game_step(run_id: str, task_id: str, session_id: str, req: GameStepRequest) ) @app.get("/api/v1/game/{run_id}/{task_id}/{session_id}/status") - def game_status(run_id: str, task_id: str, session_id: str) -> GameStatusResponse: + def game_status( + run_id: str, task_id: str, session_id: str, + authorization: str | None = Header(default=None), + ) -> GameStatusResponse: + require_game_session(authorization, run_id, task_id) + require_game_scope(session_id, run_id, task_id) try: data = state.game_status(session_id) except KeyError: @@ -907,7 +1098,12 @@ def game_status(run_id: str, task_id: str, session_id: str) -> GameStatusRespons ) @app.post("/api/v1/game/{run_id}/{task_id}/{session_id}/close") - def game_close(run_id: str, task_id: str, session_id: str) -> GameCloseResponse: + def game_close( + run_id: str, task_id: str, session_id: str, + authorization: str | None = Header(default=None), + ) -> GameCloseResponse: + require_game_session(authorization, run_id, task_id) + require_game_scope(session_id, run_id, task_id) try: data = state.close_game_session(session_id) except KeyError: @@ -921,7 +1117,11 @@ def game_close(run_id: str, task_id: str, session_id: str) -> GameCloseResponse: ) @app.post("/api/v1/game/{run_id}/{task_id}/close-all") - def game_close_all(run_id: str, task_id: str) -> dict: + def game_close_all( + run_id: str, task_id: str, + authorization: str | None = Header(default=None), + ) -> dict: + require_game_session(authorization, run_id, task_id) count = state.close_all_game_sessions(run_id, task_id) return {"closed": count} diff --git a/sforge/harness/network_isolation.py b/sforge/harness/network_isolation.py index 5a65a3f..b66f7cd 100644 --- a/sforge/harness/network_isolation.py +++ b/sforge/harness/network_isolation.py @@ -278,6 +278,8 @@ def build_allowed_endpoints( api_url: str | None, gateway_ip: str, logger: logging.Logger, + *, + resolve_hostnames: bool = True, ) -> list[AllowedEndpoint]: """Build the whitelist of TCP endpoints from judge + API URLs.""" endpoints: list[AllowedEndpoint] = [] @@ -289,6 +291,10 @@ def build_allowed_endpoints( endpoints.append(AllowedEndpoint(ip=gateway_ip, port=judge_port)) elif is_ip_address(judge_host): endpoints.append(AllowedEndpoint(ip=judge_host, port=judge_port)) + elif not resolve_hostnames: + endpoints.append( + AllowedEndpoint(ip="", port=judge_port, hostname=judge_host) + ) else: for ip in resolve_hostname(judge_host, logger): endpoints.append(AllowedEndpoint(ip=ip, port=judge_port, hostname=judge_host)) @@ -301,6 +307,10 @@ def build_allowed_endpoints( endpoints.append(AllowedEndpoint(ip=gateway_ip, port=api_port)) elif is_ip_address(api_host): endpoints.append(AllowedEndpoint(ip=api_host, port=api_port)) + elif not resolve_hostnames: + endpoints.append( + AllowedEndpoint(ip="", port=api_port, hostname=api_host) + ) else: for ip in resolve_hostname(api_host, logger): endpoints.append( diff --git a/sforge/harness/run_agent.py b/sforge/harness/run_agent.py index af4ff72..4c36465 100644 --- a/sforge/harness/run_agent.py +++ b/sforge/harness/run_agent.py @@ -30,12 +30,14 @@ import io import json -import signal import shlex +import signal +import sys import tarfile as _tarfile import threading import time import traceback +import uuid from dataclasses import dataclass from pathlib import Path, PurePosixPath from urllib.parse import urlparse @@ -44,8 +46,8 @@ from sforge.harness.agent import Agent from sforge.harness.backend import ContainerBackend, ContainerHandle -from sforge.harness.config import SForgeConfig -from sforge.harness.constants import ADMIN_SECRET +from sforge.harness.config import SForgeConfig, sanitize_e2b_proxy_env +from sforge.harness.constants import get_admin_secret from sforge.harness.docker_build import ( close_logger, setup_logger, @@ -55,6 +57,11 @@ generate_game_prompt, generate_submit_script, ) +from sforge.harness.judge_client import ( + SFORGE_ADMIN_SECRET_HEADER, + JudgeConnection, + merge_no_proxy, +) from sforge.harness.task_spec import TaskSpec @@ -135,6 +142,21 @@ def _build_agent_env( return env + +def _e2b_install_env(env: dict[str, str], agent: Agent) -> dict[str, str]: + """Keep only non-secret values required while installing the agent.""" + allowed = { + "http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", + "no_proxy", "NO_PROXY", "SFORGE_NODEJS_MIRROR_URL", + "npm_config_registry", + } + if agent.api_base_env: + allowed.add(agent.api_base_env) + if agent.model_env: + allowed.add(agent.model_env) + return {key: value for key, value in env.items() if key in allowed} + + # --------------------------------------------------------------------------- # Container setup helpers # --------------------------------------------------------------------------- @@ -172,21 +194,66 @@ def _extract_archive_from_container( backend: ContainerBackend, handle: ContainerHandle, task_spec: TaskSpec, + *, + shutdown_event: threading.Event | None = None, ) -> bytes: """Extract the current submission archive from a running work container.""" - patch_dir = task_spec.cwd - submit_paths = " ".join(task_spec.submit_paths) - excludes = " ".join(f"--exclude={e}" for e in task_spec.submit_exclude) - tar_cmd = ( - f"cd {patch_dir} && tar czf /tmp/final.tar.gz " - f"--exclude=.git {excludes} {submit_paths}" + remote_archive = PurePosixPath( + f"/tmp/.sforge-submission-{uuid.uuid4().hex}.tar.gz" + ) + path_checks = "\n".join( + "if [[ -e {path} || -L {path} ]]; then " + "submit_paths+=({path}); fi".format(path=shlex.quote(path)) + for path in task_spec.submit_paths + ) + excludes = " ".join( + shlex.quote(f"--exclude={pattern}") + for pattern in task_spec.submit_exclude + ) + tar_cmd = f""" +cd -- {shlex.quote(task_spec.cwd)} || exit $? +submit_paths=() +{path_checks} +if (( ${{#submit_paths[@]}} )); then + tar czf {shlex.quote(str(remote_archive))} --exclude=.git {excludes} -- "${{submit_paths[@]}}" +else + tar czf {shlex.quote(str(remote_archive))} --files-from /dev/null +fi +""" + result = backend.exec_run_with_exit_code( + handle, ["/bin/bash", "-c", tar_cmd], timeout=300, + user="agent", ) - backend.exec_run(handle, ["/bin/bash", "-c", tar_cmd]) - raw = backend.copy_from_container(handle, PurePosixPath("/tmp/final.tar.gz")) - outer = _tarfile.open(fileobj=io.BytesIO(raw)) - member = outer.getmembers()[0] - archive = outer.extractfile(member).read() - outer.close() + if result.timed_out: + raise RuntimeError("Timed out while creating the submission archive") + if result.exit_code != 0: + raise RuntimeError( + "Could not create the submission archive: " + result.output[-4000:] + ) + try: + if backend.backend_name == "e2b": + raw = backend.copy_from_container( + handle, remote_archive, shutdown_event=shutdown_event, + ) + else: + raw = backend.copy_from_container(handle, remote_archive) + finally: + try: + backend.exec_run( + handle, ["rm", "-f", str(remote_archive)], user="root", + ) + except Exception: + pass + with _tarfile.open(fileobj=io.BytesIO(raw)) as outer: + members = [member for member in outer.getmembers() if member.isfile()] + if len(members) != 1: + raise RuntimeError( + f"Unexpected archive wrapper: expected 1 file, got {len(members)}" + ) + source = outer.extractfile(members[0]) + if source is None: + raise RuntimeError("Could not read the submission archive wrapper") + archive = source.read() return archive @@ -194,12 +261,15 @@ def _auto_eval_loop( backend: ContainerBackend, handle: ContainerHandle, task_spec: TaskSpec, - host_judge_url: str, + host_judge: JudgeConnection, session_token: str, eval_interval: int, stop_event: threading.Event, logger, log_dir: Path, + admin_secret: str, + latest_archive: list[bytes], + transfer_cancel: threading.Event, ) -> None: """Host-side auto-eval: periodically extract code and submit to judge. @@ -212,16 +282,23 @@ def _auto_eval_loop( if stop_event.is_set(): break try: - archive = _extract_archive_from_container(backend, handle, task_spec) - resp = requests.post( - f"{host_judge_url}/api/v1/submit", + archive = _extract_archive_from_container( + backend, handle, task_spec, shutdown_event=transfer_cancel, + ) + latest_archive[:] = [archive] + if stop_event.is_set() or transfer_cancel.is_set(): + break + resp = host_judge.request( + "POST", "/api/v1/submit", data={ - "token": session_token, "kind": "auto", - "admin_secret": ADMIN_SECRET, }, files={"archive": ("archive.tar.gz", archive, "application/gzip")}, timeout=120, + headers={ + "Authorization": f"Bearer {session_token}", + SFORGE_ADMIN_SECRET_HEADER: admin_secret, + }, ) resp.raise_for_status() data = resp.json() @@ -238,6 +315,86 @@ def _auto_eval_loop( logger.debug("Auto-eval tick failed: %s", e) +def _stop_auto_eval_thread( + stop_event: threading.Event | None, + thread: threading.Thread | None, + transfer_cancel: threading.Event | None, + logger, + *, + timeout: int = 30, + shutdown_event: threading.Event | None = None, +) -> bool: + if stop_event is None or thread is None: + return True + stop_event.set() + deadline = time.monotonic() + timeout + while thread.is_alive() and time.monotonic() < deadline: + if shutdown_event is not None and shutdown_event.is_set(): + break + thread.join(min(1, max(0, deadline - time.monotonic()))) + if not thread.is_alive(): + return True + logger.warning( + "Auto-eval did not finish within %ds; cancelling its transfer", + timeout, + ) + if transfer_cancel is not None: + transfer_cancel.set() + thread.join(20) + if thread.is_alive(): + logger.warning( + "Auto-eval thread is still stopping; continuing container cleanup" + ) + return False + return True + + +def _drain_pending_evaluations( + host_judge: JudgeConnection, + session_token: str, + admin_secret: str, + logger, + *, + shutdown_event: threading.Event | None = None, + timeout: int = 900, + poll_interval: int = 15, +) -> None: + """Wait for submitted evaluations, while remaining interruptible.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if shutdown_event is not None and shutdown_event.is_set(): + logger.info("Judge drain interrupted; continuing cleanup") + return + try: + history = host_judge.request( + "GET", "/api/v1/history", + timeout=10, + headers={ + "Authorization": f"Bearer {session_token}", + SFORGE_ADMIN_SECRET_HEADER: admin_secret, + }, + ).json() + pending = [ + entry for entry in history.get("entries", []) + if entry.get("status") in ("running", "queued") + ] + if not pending: + return + logger.info(f"Draining {len(pending)} pending judge evals...") + except Exception: + return + wait_seconds = min(poll_interval, max(0, deadline - time.monotonic())) + if shutdown_event is None: + time.sleep(wait_seconds) + elif shutdown_event.wait(wait_seconds): + logger.info("Judge drain interrupted; continuing cleanup") + return + logger.warning( + f"Drain timed out after {timeout / 60:g} min; " + "some reports may be missing." + ) + + # --------------------------------------------------------------------------- # Main entry point # --------------------------------------------------------------------------- @@ -261,6 +418,7 @@ def run_agent( shutdown_event: threading.Event | None = None, max_submissions: int | None = None, submission_cooldown: int | None = None, + host_judge_connection: JudgeConnection | None = None, ) -> RunResult: """Run an agent on a task with iterative evaluation. @@ -282,22 +440,37 @@ def run_agent( verbose=verbose, ) - # judge_url is for container use; derive a host-local URL for registration/polling + # judge_url is the Sandbox-facing URL. Docker's host alias is the only + # address that needs a host-side rewrite; E2B gateway URLs are routable + # from both the host and the Sandbox. parsed_judge = urlparse(judge_url) if parsed_judge.hostname == "host.docker.internal": host_judge_url = judge_url.replace("host.docker.internal", "127.0.0.1") elif backend.backend_name == "k8s": - # judge_url points to VPC IP (for pods); host talks to judge server locally host_judge_url = f"http://127.0.0.1:{parsed_judge.port or 8080}" else: host_judge_url = judge_url + remote_judge = JudgeConnection.from_e2b_traffic_token( + judge_url, config.e2b_judge_access_token, + ) + host_judge = host_judge_connection or ( + JudgeConnection.from_e2b_traffic_token( + host_judge_url, + config.e2b_judge_access_token, + ) + ) handle = None net_isolation = None - api_proxy = None + auto_eval_stop = None + auto_eval_thread = None + auto_eval_transfer_cancel = None + latest_auto_archive: list[bytes] = [] install_parts: list[str] = [] + admin_secret = "" try: + admin_secret = get_admin_secret(config.log_dir) # 0. Clean up stale iptables chains from previous runs that were killed if backend.backend_name == "docker": from sforge.harness.network_isolation import cleanup_stale_chains @@ -306,13 +479,22 @@ def run_agent( # 1. Check images exist (must run `sforge build` first) for image_key in (task_spec.work_image_key, task_spec.judge_image_key): if not backend.image_exists(image_key): + if backend.backend_name == "e2b": + raise RuntimeError( + f"E2B Template for {image_key!r} was not found. Build it with: " + f"sforge e2b-template --task {task_spec.task_id} " + "--source-registry " + ) raise RuntimeError( f"Image '{image_key}' not found. Run `sforge pull --task {task_spec.task_id}` to fetch from registry, or `sforge build --task {task_spec.task_id}` to build locally." ) logger.info("Images ready") # 1b. Register session with judge server - reg_body: dict = {"task_id": task_spec.task_id, "run_id": run_id, "admin_secret": ADMIN_SECRET} + reg_body: dict = { + "task_id": task_spec.task_id, + "run_id": run_id, + } if config.judge_cpu_limit is not None: reg_body["judge_cpu_limit"] = config.judge_cpu_limit if config.judge_mem_limit is not None: @@ -321,7 +503,7 @@ def run_agent( reg_body["max_agent_submissions"] = max_submissions if submission_cooldown is not None: reg_body["submission_cooldown"] = submission_cooldown - if backend.backend_name != "docker": + if backend.backend_name == "k8s": reg_body["backend"] = backend.backend_name reg_body["k8s_image_registry"] = config.k8s_image_registry reg_body["k8s_namespace"] = config.k8s_namespace @@ -329,11 +511,18 @@ def run_agent( reg_body["k8s_node_selector"] = config.k8s_node_selector if config.k8s_kubeconfig: reg_body["k8s_kubeconfig"] = config.k8s_kubeconfig + elif backend.backend_name == "e2b": + reg_body.update({ + "backend": "e2b", + "e2b_template_map": config.e2b_template_map, + "e2b_sandbox_ttl": config.e2b_sandbox_ttl, + }) for _reg_attempt in range(1, 6): try: - reg_resp = requests.post( - f"{host_judge_url}/api/v1/register", + reg_resp = host_judge.request( + "POST", "/api/v1/register", json=reg_body, + headers={SFORGE_ADMIN_SECRET_HEADER: admin_secret}, timeout=10, ) reg_resp.raise_for_status() @@ -353,7 +542,9 @@ def run_agent( backend.remove_container_by_name(container_name) env = _build_agent_env(agent, model) - env["SFORGE_JUDGE_URL"] = judge_url + if backend.backend_name == "e2b": + sanitize_e2b_proxy_env(env, internet=internet) + env.update(remote_judge.agent_env()) env["SFORGE_TOKEN"] = session_token env["SFORGE_PATCH_DIR"] = task_spec.cwd env["SFORGE_SUBMIT_PATHS"] = " ".join(task_spec.submit_paths) @@ -371,19 +562,19 @@ def run_agent( # Ensure judge server URL bypasses proxy judge_host = urlparse(judge_url).hostname or "" for key in ("NO_PROXY", "no_proxy"): - existing = env.get(key, "") - if judge_host and judge_host not in existing: - env[key] = f"{existing},{judge_host}" if existing else judge_host + env[key] = merge_no_proxy(env.get(key, ""), judge_host) # Network isolation: preflight + extra_hosts pre-resolve container_extra_hosts = {"host.docker.internal": "host-gateway"} container_cap_drop: list[str] = [] - if not internet: + if not internet and backend.backend_name == "docker": + # Docker-only preflight: isolation runs as host iptables rules, so + # it needs sudo and conflicts with host proxy env leaking into the + # container. Remote backends (k8s, e2b) isolate inside the cluster/ + # provider and are unaffected by either. from sforge.harness.network_isolation import ( check_iptables_permission, - is_ip_address, - resolve_hostname, ) if config.http_proxy or config.https_proxy: @@ -401,10 +592,21 @@ def run_agent( "Ensure passwordless sudo is configured for iptables." ) + if not internet: + from sforge.harness.network_isolation import ( + is_ip_address, + resolve_hostname, + ) + api_url = config.agent_api_base_url or agent.default_api_base_url if api_url: api_host = urlparse(api_url).hostname or "" - if api_host and api_host != "host.docker.internal" and not is_ip_address(api_host): + if ( + backend.backend_name != "e2b" + and api_host + and api_host != "host.docker.internal" + and not is_ip_address(api_host) + ): resolved_ips = resolve_hostname(api_host, logger) if resolved_ips: container_extra_hosts[api_host] = resolved_ips[0] @@ -418,7 +620,10 @@ def run_agent( handle = backend.create_container( task_spec.work_image_key, container_name, - environment=env, + environment=( + _e2b_install_env(env, agent) + if backend.backend_name == "e2b" else env + ), extra_hosts=container_extra_hosts, cap_drop=container_cap_drop or None, cpu_limit=cpu, @@ -453,7 +658,6 @@ def run_agent( # the stop hook — otherwise the agent exits naturally as soon as # the model decides it's "done", losing the full timeout budget. effective_eval_interval = 0 if disable_auto_eval else eval_interval - auto_eval_stop = None if not task_spec.game_mode: _install_tools( backend, @@ -468,12 +672,15 @@ def run_agent( # Start host-side auto-eval thread (if enabled) if effective_eval_interval > 0: auto_eval_stop = threading.Event() + auto_eval_transfer_cancel = threading.Event() auto_eval_thread = threading.Thread( target=_auto_eval_loop, args=( - backend, handle, task_spec, host_judge_url, + backend, handle, task_spec, host_judge, session_token, effective_eval_interval, auto_eval_stop, logger, log_dir, + admin_secret, + latest_auto_archive, auto_eval_transfer_cancel, ), daemon=True, ) @@ -485,7 +692,6 @@ def run_agent( # 4b. Apply network isolation (after install + tools, before agent) if not internet: from sforge.harness.network_isolation import ( - AllowedEndpoint, build_allowed_endpoints, ) @@ -498,6 +704,7 @@ def run_agent( effective_api_url = config.agent_api_base_url or agent.default_api_base_url endpoints = build_allowed_endpoints( judge_url, effective_api_url, gateway_ip, logger, + resolve_hostnames=backend.backend_name != "e2b", ) net_isolation = backend.create_network_isolation(handle, endpoints, logger) net_isolation.apply() @@ -596,36 +803,66 @@ def run_agent( # 7. Stop auto-eval thread before extracting final archive if auto_eval_stop is not None: - auto_eval_stop.set() + _stop_auto_eval_thread( + auto_eval_stop, auto_eval_thread, auto_eval_transfer_cancel, + logger, timeout=180, shutdown_event=shutdown_event, + ) logger.info("Auto-eval thread stopped") - # 8. Extract final archive (tar of submit_paths) - try: - final_archive = _extract_archive_from_container(backend, handle, task_spec) - (log_dir / "final_archive.tar.gz").write_bytes(final_archive) - logger.info(f"Final archive: {len(final_archive)} bytes") - except Exception as e: - logger.warning(f"Failed to extract final archive (container may have stopped): {e}") - final_archive = b"" + # 8. Extract final archive for code tasks. Game tasks have no submit + # paths; their authoritative output is the Judge Server game history. + final_archive = b"" + if not task_spec.game_mode: + try: + interrupted = ( + shutdown_event is not None and shutdown_event.is_set() + ) + if interrupted: + final_archive = ( + latest_auto_archive[-1] if latest_auto_archive else b"" + ) + else: + final_archive = _extract_archive_from_container( + backend, handle, task_spec, + ) + (log_dir / "final_archive.tar.gz").write_bytes(final_archive) + logger.info(f"Final archive: {len(final_archive)} bytes") + except Exception as exc: + if backend.backend_name == "e2b": + raise + # Preserve the pre-E2B Docker/Kubernetes behavior. + logger.warning( + "Failed to extract final archive " + f"(container may have stopped): {exc}" + ) + final_archive = b"" # 8. Collect results if task_spec.game_mode: try: - requests.post( - f"{host_judge_url}/api/v1/game/{run_id}/{task_spec.task_id}/close-all", + close_response = host_judge.request( + "POST", + f"/api/v1/game/{run_id}/{task_spec.task_id}/close-all", timeout=30, + headers={"Authorization": f"Bearer {session_token}"}, ) + close_response.raise_for_status() except Exception: + # Keep the pre-E2B best-effort semantics. The managed + # controller still exports any game artifacts during cleanup. logger.warning("Failed to close active game sessions") try: - history_resp = requests.get( - f"{host_judge_url}/api/v1/history?token={session_token}", + history_resp = host_judge.request( + "GET", "/api/v1/history", timeout=10, + headers={"Authorization": f"Bearer {session_token}"}, ) history_resp.raise_for_status() history = history_resp.json() except Exception: + # Docker and Kubernetes historically completed the run with + # an empty result when the Judge history was unavailable. logger.warning("Failed to fetch run history from judge server") history = {"run_id": run_id, "best_score": None, "entries": []} @@ -689,34 +926,27 @@ def run_agent( pass # Drain pending judge evaluations before querying final results - drain_deadline = time.time() + 900 - while time.time() < drain_deadline: - try: - h = requests.get( - f"{host_judge_url}/api/v1/history", - params={"token": session_token, "admin_secret": ADMIN_SECRET}, - timeout=10, - ).json() - pending = [e for e in h.get("entries", []) if e.get("status") in ("running", "queued")] - if not pending: - break - logger.info(f"Draining {len(pending)} pending judge evals...") - except Exception: - break - time.sleep(15) - else: - logger.warning("Drain timed out after 15 min; some reports may be missing.") + _drain_pending_evaluations( + host_judge, session_token, admin_secret, logger, + shutdown_event=shutdown_event, + ) # Query Judge Server for authoritative results (with admin_secret to get full history) try: - history_resp = requests.get( - f"{host_judge_url}/api/v1/history", - params={"token": session_token, "admin_secret": ADMIN_SECRET}, + history_resp = host_judge.request( + "GET", "/api/v1/history", timeout=10, + headers={ + "Authorization": f"Bearer {session_token}", + SFORGE_ADMIN_SECRET_HEADER: admin_secret, + }, ) history_resp.raise_for_status() history = history_resp.json() except Exception: + # Preserve the original Docker/Kubernetes behavior: a + # transient history failure must not turn an otherwise + # completed agent run into a task-level infrastructure error. logger.warning("Failed to fetch run history from judge server") history = { "run_id": run_id, "best_score": None, @@ -766,13 +996,23 @@ def run_agent( # Stop auto-eval thread if auto_eval_stop is not None: - auto_eval_stop.set() + _stop_auto_eval_thread( + auto_eval_stop, auto_eval_thread, auto_eval_transfer_cancel, + logger, shutdown_event=shutdown_event, + ) # Try to extract archive from the container before it's destroyed interrupted_archive = b"" try: - if handle is not None: - interrupted_archive = _extract_archive_from_container(backend, handle, task_spec) + if handle is not None and not task_spec.game_mode: + interrupted_archive = ( + latest_auto_archive[-1] + if latest_auto_archive + else _extract_archive_from_container( + backend, handle, task_spec, + shutdown_event=shutdown_event, + ) + ) (log_dir / "final_archive.tar.gz").write_bytes(interrupted_archive) logger.info(f"Final archive (interrupted): {len(interrupted_archive)} bytes") except Exception: @@ -780,10 +1020,13 @@ def run_agent( # Query judge server for results accumulated before interruption (full history) try: - history_resp = requests.get( - f"{host_judge_url}/api/v1/history", - params={"token": session_token, "admin_secret": ADMIN_SECRET}, + history_resp = host_judge.request( + "GET", "/api/v1/history", timeout=10, + headers={ + "Authorization": f"Bearer {session_token}", + SFORGE_ADMIN_SECRET_HEADER: admin_secret, + }, ) history_resp.raise_for_status() history = history_resp.json() @@ -827,17 +1070,25 @@ def run_agent( ) except Exception as e: logger.error(f"Error: {e}\n{traceback.format_exc()}") + if backend.backend_name == "e2b": + # E2B transport or lifecycle failures must remain visible to the + # caller instead of being reported as a successful zero score. + raise + # Preserve the pre-E2B Docker/Kubernetes behavior. try: - history_resp = requests.get( - f"{host_judge_url}/api/v1/history?token={session_token}", + history_resp = host_judge.request( + "GET", "/api/v1/history", timeout=10, + headers={"Authorization": f"Bearer {session_token}"}, ) history_resp.raise_for_status() history = history_resp.json() best_pass_rate = history.get("best_pass_rate", 0.0) best_score_raw = history.get("best_score") - best_score = float(best_score_raw) if best_score_raw is not None else None + best_score = ( + float(best_score_raw) if best_score_raw is not None else None + ) best_round = history.get("best_round", "") agent_subs = history.get("agent_submissions", 0) auto_subs = history.get("auto_submissions", 0) @@ -860,17 +1111,35 @@ def run_agent( resume_count=locals().get("resume_count", 0), ) finally: + primary_error = sys.exc_info()[1] + if auto_eval_stop is not None: + auto_eval_stop.set() + if auto_eval_transfer_cancel is not None: + auto_eval_transfer_cancel.set() if net_isolation is not None: try: net_isolation.cleanup() except Exception as exc: if logger: logger.warning(f"Failed to cleanup network isolation: {exc}") - if threading.current_thread() is threading.main_thread(): - prev_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) + main_thread = threading.current_thread() is threading.main_thread() + previous_handler = None + if main_thread: + previous_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) print("\nStopping container, please wait... (Ctrl+C disabled during cleanup)") + cleanup_error = None + try: backend.cleanup_container(handle, logger) - signal.signal(signal.SIGINT, prev_handler) - else: - backend.cleanup_container(handle, logger) - close_logger(logger) + except Exception as exc: + cleanup_error = exc + if primary_error is not None: + logger.exception( + "Work container cleanup failed while preserving the " + "original run error" + ) + finally: + if previous_handler is not None: + signal.signal(signal.SIGINT, previous_handler) + close_logger(logger) + if cleanup_error is not None and primary_error is None: + raise cleanup_error diff --git a/sforge/harness/run_evaluation.py b/sforge/harness/run_evaluation.py index 058d895..be7ad9d 100644 --- a/sforge/harness/run_evaluation.py +++ b/sforge/harness/run_evaluation.py @@ -25,15 +25,17 @@ from pathlib import Path, PurePosixPath from sforge.harness.backend import ContainerBackend -from sforge.harness.config import SForgeConfig, get_container_env -from sforge.harness.constants import ( - DOCKER_USER, +from sforge.harness.config import ( + SForgeConfig, + get_container_env, + sanitize_e2b_proxy_env, ) +from sforge.harness.constants import DOCKER_USER from sforge.harness.docker_build import ( BuildImageError, build_judge_image, - setup_logger, close_logger, + setup_logger, ) from sforge.harness.grading import EvalReport, grade_output from sforge.harness.score_rescale import rescale_score @@ -155,6 +157,8 @@ def judge_submission( # 2. Create + start ephemeral container container_name = f"{task_spec.benchmark_name}.judge.{task_spec.task_id}.{submission_id}" env = get_container_env(config, include_judge_extra=True) + if backend.backend_name == "e2b": + sanitize_e2b_proxy_env(env) env.setdefault("SFORGE_JUDGE_URL", "http://host.docker.internal:8080") # Resolve resource limits: config > task defaults cpu = config.judge_cpu_limit if config.judge_cpu_limit is not None else task_spec.judge.cpu_limit @@ -237,10 +241,12 @@ def judge_submission( eval_script_local.write_text(task_spec.eval_script) backend.copy_to_container(handle, eval_script_local, PurePosixPath("/tmp/eval.sh")) - # 5. Run eval script + # 5. Run eval script. workdir matters for eval_cmds that use relative + # paths: docker defaults to the image WORKDIR but envd execs land in + # the user's home, so pin the task cwd explicitly. logger.info("Running eval script...") result = backend.exec_run_with_timeout( - handle, "/bin/bash /tmp/eval.sh", timeout + handle, "/bin/bash /tmp/eval.sh", timeout, workdir=task_spec.cwd ) test_output, timed_out, runtime = result.output, result.timed_out, result.elapsed_seconds @@ -280,5 +286,7 @@ def judge_submission( raw_output=str(e), ) finally: - backend.cleanup_container(handle, logger) - close_logger(logger) + try: + backend.cleanup_container(handle, logger) + finally: + close_logger(logger) diff --git a/sforge/harness/task_spec.py b/sforge/harness/task_spec.py index a69234d..ff3ceaf 100644 --- a/sforge/harness/task_spec.py +++ b/sforge/harness/task_spec.py @@ -21,12 +21,14 @@ from sforge.harness.benchmark import BenchmarkMeta from sforge.harness.constants import ( - SForgeTask, - START_TEST_OUTPUT, END_TEST_OUTPUT, + START_TEST_OUTPUT, + SForgeTask, ) from sforge.harness.score_rescale import RescaleSpec, parse_rescale_spec +IMAGE_RECIPE_VERSION = 2 + @dataclass class WorkSpec: @@ -54,7 +56,11 @@ class JudgeSpec: def _compute_base_image_hash(base_image_key: str, base_image_spec: dict) -> str: - data = json.dumps({"key": base_image_key, "spec": base_image_spec}, sort_keys=True) + data = json.dumps({ + "recipe_version": IMAGE_RECIPE_VERSION, + "key": base_image_key, + "spec": base_image_spec, + }, sort_keys=True) return hashlib.sha256(data.encode()).hexdigest() @@ -100,6 +106,7 @@ def work_image_hash(self) -> str: f"Task '{self.task_id}': cannot compute work image hash without setup_cmds" ) data = json.dumps({ + "recipe_version": IMAGE_RECIPE_VERSION, "base_hash": self.base_image_hash, "platform": self.platform, "cwd": self.cwd, @@ -114,6 +121,7 @@ def judge_image_hash(self) -> str: f"Task '{self.task_id}': cannot compute judge image hash without setup_cmds" ) data = json.dumps({ + "recipe_version": IMAGE_RECIPE_VERSION, "base_hash": self.base_image_hash, "platform": self.platform, "cwd": self.cwd, From 3bb12f0a031ef300d9f83fba42c959377e6548b2 Mon Sep 17 00:00:00 2001 From: Tao Wang Date: Tue, 25 Aug 2026 17:00:24 +0800 Subject: [PATCH 02/17] docs: document e2b backend usage --- README.md | 1 + docs/en/configuration/container-backends.md | 210 ++++++++++++++++-- .../en/configuration/environment-variables.md | 15 +- docs/en/configuration/experiments.md | 2 +- docs/en/features/network-isolation.md | 11 + docs/en/reference/cli.md | 13 +- docs/en/reference/judge-api.md | 53 +++-- docs/zh/configuration/container-backends.md | 186 ++++++++++++++-- .../zh/configuration/environment-variables.md | 14 +- docs/zh/configuration/experiments.md | 2 +- docs/zh/features/network-isolation.md | 9 + docs/zh/reference/cli.md | 12 +- docs/zh/reference/judge-api.md | 50 +++-- 13 files changed, 490 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 6bcc671..aef1459 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,7 @@ SFORGE_AGENT_API_KEY="sk-xxx" \ **Step-by-step examples:** - [Single task on local Docker](examples/single-task-docker/) — run one task end-to-end with Docker - [All tasks on Kubernetes](examples/all-tasks-k8s/) — run the full suite on a K8s cluster with the **official leaderboard setting** +- [E2B backend](https://bytedance-seed.github.io/EdgeBench/en/configuration/container-backends#e2b-backend) — run Work and Judge environments on E2B without operating a cluster > [!IMPORTANT] > - **Official setting** — leaderboard numbers use the [official experiment YAMLs](examples/all-tasks-k8s/) unchanged, including the time budget, stop hook, auto-eval, submission cooldowns, and hardware resource limits. diff --git a/docs/en/configuration/container-backends.md b/docs/en/configuration/container-backends.md index 8476fe5..7e4eb7d 100644 --- a/docs/en/configuration/container-backends.md +++ b/docs/en/configuration/container-backends.md @@ -4,7 +4,9 @@ title: Container Backends # Container Backends -SForge uses a container backend to create work containers and judge containers. The default backend is local Docker. Switch to the `k8s` backend when you want SForge workloads to run in a Kubernetes cluster. +SForge uses a container backend to create work and judge environments. The +default backend is local Docker. Use `k8s` for an existing Kubernetes cluster, +or `e2b` for E2B-hosted environments. ## Choosing a Backend @@ -12,8 +14,11 @@ SForge uses a container backend to create work containers and judge containers. |---------|----------|-------------------| | `docker` | Local development, task debugging, single-machine evaluation, small experiments | A working local Docker daemon | | `k8s` | Shared clusters, high-concurrency batch evaluation, running Work/Judge pods in a cluster | `kubectl` access, images pullable by the cluster, and a Judge URL reachable from pods | +| `e2b` | Managed remote Work/Judge Sandboxes without operating a cluster or Judge broker | E2B API access and E2B Templates; see [E2B Backend](#e2b-backend) | If you are trying a task locally, use the default Docker backend first. The Kubernetes backend is intended for team environments that already have a cluster and container registry. +The E2B backend has a one-time Template preparation step before a new or +changed task image can run. ::: warning Docker backend does not scale to large batches Each task runs a work container plus ephemeral judge containers, each with its own CPU/memory limits. Running many tasks concurrently on one host (roughly **20+**) causes severe resource contention even on a high-end server. For large batch runs, use the `k8s` backend. @@ -29,15 +34,13 @@ sforge run \ --task ad_placement_optimization \ --agent claude-code \ --backend k8s \ - --judge-url http://10.0.0.12:8080 --backend k8s + --judge-url http://10.0.0.12:8080 # Override through environment variables export SFORGE_BACKEND=k8s sforge run \ --task ad_placement_optimization \ - --agent claude-code \ - --backend k8s \ - --judge-url http://10.0.0.12:8080 + --agent claude-code ``` Experiment YAML can also set the default backend: @@ -62,9 +65,7 @@ sforge serve export SFORGE_AGENT_API_KEY="sk-..." sforge run \ --task ad_placement_optimization \ - --agent claude-code \ - --backend k8s \ - --judge-url http://10.0.0.12:8080 + --agent claude-code ``` The Docker backend: @@ -222,13 +223,190 @@ Notes: - Your kubeconfig must be allowed to create and delete `NetworkPolicy` resources. - DNS, egress gateways, and policy implementations differ between clusters, so validate isolation with a small task first. -## Troubleshooting +## E2B Backend + +The E2B backend runs Work, Judge, and Game environments as E2B Sandboxes. +When `--judge-url` is omitted, SForge also creates a temporary managed Judge +Controller, so no separately deployed Judge service is required. + +### Docker Images and E2B Templates + +The pre-built EdgeBench images published under +[`seededge`](https://hub.docker.com/u/seededge) can be used directly as source +images. You do not need to run `sforge build`, `sforge pull`, or `sforge push` +for an unchanged published task. + +A Docker image and an E2B Template are different runtime objects. +`Template.from_image()` imports the Docker image into an E2B Template, while +`Sandbox.create()` starts the resulting Template rather than accepting a +Docker image reference. SForge automates this conversion: + +```text +seededge Docker image + -> sforge e2b-template + -> E2B Template in the user's E2B team + -> sforge run --backend e2b + -> E2B Sandbox +``` + +Template preparation is required once for a new or changed task image in an +E2B team. Matching Templates are reused; normal `sforge run` commands do not +rebuild them. + +### Prepare Templates + +```bash +uv sync --extra e2b +export E2B_API_KEY=... + +sforge fetch-tasks edgebench +sforge e2b-template \ + --task ad_placement_optimization \ + --source-registry seededge +``` + +Use `--all` instead of `--task` to prepare all downloaded tasks. SForge +automatically derives both image references, pins immutable digests, restores +non-sensitive OCI environment variables, snapshots the image's initial `/tmp`, +builds the Work and Judge Templates, waits for terminal status, retries +transient failures, writes versioned manifests, and reuses matching builds. +Users do not write an E2B Dockerfile or upload image layers manually. + +The current implementation requires a Linux amd64 host with a running local +Docker daemon, but does not rebuild the published images. The official task +images predate E2B and ship evaluator helpers under `/tmp` — a wrapper the +command names plus other files that wrapper reads at runtime. Docker and +Kubernetes boot from the image, so those files are simply present; E2B resets +`/tmp` on every Sandbox. When a role's command references `/tmp`, SForge snapshots +the whole image `/tmp` (so helpers reached only indirectly come along too), +stores it in the Template, and restores it on Sandbox start, so the evaluator +sees the same files as on the other backends. This never modifies the official +image, the evaluators, or scoring. The source registry must also be reachable by +the E2B build service. + +### Portable Image Fallback + +Some image versions contain an APT source that is inaccessible from the E2B +build network. For those images, provide a writable registry: + +```bash +export SFORGE_PORTABLE_REGISTRY_USERNAME=... +export SFORGE_PORTABLE_REGISTRY_PASSWORD=... + +sforge e2b-template \ + --task TASK \ + --source-registry seededge \ + --portable-registry registry.example.com/project +``` + +SForge creates a versioned portable derivative only after E2B reports an APT +provisioning failure. Registry push access is required for this fallback, but +the derivative is not rebuilt for every evaluation run. + +### Run a Task + +```bash +export SFORGE_AGENT_API_KEY=... +export SFORGE_AGENT_API_BASE_URL=... +export SFORGE_AGENT_MODEL=... + +sforge run \ + --backend e2b \ + --task ad_placement_optimization \ + --agent claude-code +``` + +SForge checks the Work/Judge Templates, creates a secured Controller, registers +the task, creates Work/Judge/Game Sandboxes as needed, propagates gateway +credentials, collects results, and removes the Sandboxes. Pass `--judge-url` +only when using a separately deployed Judge Server. + +Work Sandboxes reach a managed Controller through its secured E2B service +gateway. Host-side orchestration requests use the E2B command channel and the +Controller's loopback interface instead, avoiding an unnecessary round trip +through that public gateway. This transport choice is specific to E2B and does +not change Docker or Kubernetes Judge routing. + +Each E2B evaluation is scoped to one task. Supplying multiple task IDs only +batch-schedules several independent evaluations; it does not combine tasks into +one Work or Judge environment. The validation described here invoked one task +per `sforge run` command. + +### Plan Limits and Run Sizing + +E2B plan limits are part of the runtime contract. Template CPU and memory are +fixed when the Template is built, and requests above the team's limits are +rejected during Template creation. Rebuild the Template with values supported +by the target team; runtime `--work-*` and `--judge-*` flags cannot resize an +existing E2B Template. + +Keep the agent timeout comfortably below the team's maximum Sandbox lifetime. +The lifetime must also cover agent installation, archive extraction, pending +Judge evaluations, result collection, and cleanup. A managed Judge Controller +lives for the entire multi-task command, so the whole staggered or multi-wave +batch—not only one task—must fit within that Controller's permitted lifetime. +For example, a team limited to one-hour Sandboxes should use a shorter agent +budget such as 30 minutes and avoid scheduling enough waves to keep one managed +Controller alive for an hour. Alternatively, use a separately deployed Judge +Server. + +Check each task's `judge.eval_timeout` as well. A Judge Sandbox is subject to +the same team lifetime limit, so a task whose declared evaluator timeout is +longer than that limit cannot be guaranteed to finish when the evaluator uses +its full budget. Do not silently lower the evaluator timeout if leaderboard +comparability matters; use a plan with a sufficient lifetime instead. + +`--max-workers` limits concurrent Work Sandboxes, not total E2B usage. Budget +for one managed Controller, the Work Sandboxes, temporary Judge Sandboxes, and +active Game Sandboxes. Leave capacity for overlap while evaluations finish and +resources are cleaned up. + +The final score retains SForge's backend-independent semantics: it is the best +result among completed agent and auto-eval submissions. `final_archive.tar.gz` +is a recovery snapshot and is not submitted implicitly at timeout. For short +runs, choose an auto-eval interval that leaves enough time for at least one +evaluation to finish, or have the agent call `sforge-submit` before the run +ends. + +### Required User Input + +| Input | When required | +| --- | --- | +| `E2B_API_KEY` | Template preparation and E2B runs | +| Agent API key, base URL, and model | Agent runs | +| Linux amd64 host and local Docker daemon | Template preparation with the current implementation | +| `--source-registry seededge` | Published EdgeBench images | +| Source registry credentials | Private source registries only | +| `--portable-registry` and credentials | APT portability fallback only | +| CPU/memory overrides | Only when task defaults are unsuitable; E2B resources are fixed in the Template | + +Host `HTTP_PROXY` and `HTTPS_PROXY` variables are not copied into E2B Work, +Judge, or Game Sandboxes. A remote-reachable proxy is forwarded only when the +user explicitly configures `SFORGE_HTTP_PROXY` or `SFORGE_HTTPS_PROXY`; isolated +tasks remove proxy variables entirely. See [Network Isolation](/en/features/network-isolation). + +### E2B Troubleshooting | Symptom | Likely cause | Fix | -|---------|--------------|-----| -| `kubectl cluster-info failed` | Wrong kubeconfig, unreachable cluster, or wrong namespace context | Check `kubectl -n cluster-info`; set `SFORGE_K8S_KUBECONFIG` and `SFORGE_K8S_NAMESPACE` if needed | -| Pod never becomes Running | Image pull failure, scheduling failure, or insufficient resources | Run `kubectl -n describe pod ` and check events | -| Pod cannot pull image | Image was not pushed to a registry reachable by the cluster, or registry credentials are missing | Run `sforge push`; verify `SFORGE_K8S_IMAGE_REGISTRY`; configure image pull secrets for the namespace | -| Work pod cannot submit evaluations | `--judge-url` is not reachable from pods, or the Judge server is not listening externally | Start `sforge serve` with `--host 0.0.0.0` and pass a pod-reachable IP/Service URL with `--judge-url` | -| `--disable-internet` has no effect | CNI does not enforce NetworkPolicy, or permissions are insufficient | Confirm NetworkPolicy support and verify the current identity can create NetworkPolicies | -| Pod stays Pending after setting a node selector | No nodes match the selector | Check `SFORGE_K8S_NODE_SELECTOR` and node labels | +| --- | --- | --- | +| Template not found | The image has not been converted in this E2B team, or its inputs changed | Run `sforge e2b-template` for the task | +| E2B cannot pull a `seededge` image | Registry/network failure or the exact tag is unavailable | Confirm the task definition and retry | +| Template provisioning cannot reach an APT source | The source image contains an inaccessible mirror | Supply `--portable-registry` and credentials | +| A Template build reports an internal error or remains `building` past the client timeout | Transient E2B build-service failure; the SDK has no per-build cancellation API | Check whether the requested tag became launchable before retrying, then retry the specific role; avoid starting many cold builds at once | +| A runtime evaluator under `/tmp` is missing | The Template was built outside SForge, so the image's `/tmp` snapshot is absent | Rebuild with `sforge e2b-template`; SForge snapshots the whole image `/tmp` and restores it on Sandbox start | +| A secured Judge returns 403 | Its traffic access token was not provided | Use managed mode, or set `SFORGE_JUDGE_ACCESS_TOKEN` for an external secured Judge | +| Agent-side Judge requests intermittently time out | The Work Sandbox could not establish a connection through the E2B public gateway | Retry the agent submission, reduce simultaneous submission bursts, or use an external Judge Server; host-side managed-controller calls already use E2B command-channel loopback | +| A run ends at the configured task timeout with `Sandbox not found` | The team-level maximum Sandbox lifetime was reached before final extraction and cleanup | Shorten the agent timeout, reduce multi-wave batch duration, or use a plan with a longer Sandbox lifetime | +| CPU or memory is rejected while preparing a Template | The requested Template resources exceed the E2B team's limits | Rebuild at supported values; note that this is not equivalent to leaderboard resource settings when those require more resources | +| A run leaves a Sandbox | Cleanup failed or the runner was terminated abruptly | Inspect the run log and wait for the configured E2B TTL as the final safeguard | + +## Troubleshooting + +| Backend | Symptom | Likely cause | Fix | +| --- | --- | --- | --- | +| Kubernetes | `kubectl cluster-info failed` | Wrong kubeconfig, unreachable cluster, or wrong namespace context | Check `kubectl -n cluster-info`; set `SFORGE_K8S_KUBECONFIG` and `SFORGE_K8S_NAMESPACE` if needed | +| Kubernetes | Pod never becomes Running | Image pull failure, scheduling failure, or insufficient resources | Run `kubectl -n describe pod ` and check events | +| Kubernetes | Pod cannot pull an image | The image is unavailable to cluster nodes, or registry credentials are missing | Run `sforge push`; verify `SFORGE_K8S_IMAGE_REGISTRY`; configure image pull secrets for the namespace | +| Kubernetes | Work cannot reach Judge | The Judge URL is not pod-reachable, or the server is not listening externally | Start `sforge serve` with `--host 0.0.0.0` and pass a pod-reachable IP or Service URL | +| Kubernetes | `--disable-internet` has no effect | CNI does not enforce NetworkPolicy, or permissions are insufficient | Confirm NetworkPolicy support and permissions | +| Kubernetes | Pod remains Pending with a node selector | No nodes match the selector | Check `SFORGE_K8S_NODE_SELECTOR` and node labels | diff --git a/docs/en/configuration/environment-variables.md b/docs/en/configuration/environment-variables.md index c4db0d1..2c26ee5 100644 --- a/docs/en/configuration/environment-variables.md +++ b/docs/en/configuration/environment-variables.md @@ -59,6 +59,11 @@ Used when running agents via `sforge run`. These are injected as container envir Directly injecting `SFORGE_HTTP_PROXY` / `SFORGE_HTTPS_PROXY` into the agent container gives that container proxy-mediated network access. This is useful only for exceptional run-time dependency downloads and is **not recommended** for LLM API access. It is also incompatible with network isolation (`--disable-internet`). If the agent needs to call an LLM API through a corporate proxy while network isolation is enabled, run `sforge proxy` on the host and point `SFORGE_AGENT_API_BASE_URL` to `http://host.docker.internal:` instead. + +For E2B, generic host `HTTP_PROXY` / `HTTPS_PROXY` variables are not forwarded +to Work Sandboxes. Set the `SFORGE_` variants only when the proxy is reachable +from E2B; isolated E2B runs always remove proxy variables and use the native +host allowlist directly. ::: ## Judge Variables @@ -66,6 +71,8 @@ It is also incompatible with network isolation (`--disable-internet`). If the ag | Variable | Purpose | |----------|---------| | `SFORGE_JUDGE_EXTRA_ENV` | Extra env vars for judge containers, format: `"KEY1=VAL1,KEY2=VAL2"` | +| `SFORGE_ADMIN_SECRET` | Trusted-host credential for Judge session registration and auto-eval. If omitted, SForge creates a mode-`0600` secret under `SFORGE_LOG_DIR`. Never expose it to Work containers. | +| `SFORGE_JUDGE_ACCESS_TOKEN` | E2B traffic access token for an explicitly configured secured Judge URL. Managed mode sets this automatically. | ## Resource Limit Variables @@ -80,11 +87,15 @@ Judge CPU/memory limits are currently set through CLI flags (`--judge-cpu-limit` | Variable | Default | Purpose | |----------|---------|---------| -| `SFORGE_BACKEND` | `docker` | Container backend: `docker` or `k8s` | +| `SFORGE_BACKEND` | `docker` | Container backend: `docker`, `k8s`, or official `e2b` | | `SFORGE_K8S_NAMESPACE` | `default` | Kubernetes namespace | | `SFORGE_K8S_IMAGE_REGISTRY` | --- | Container registry for K8s image pulls | | `SFORGE_K8S_KUBECONFIG` | --- | Path to kubeconfig file | | `SFORGE_K8S_NODE_SELECTOR` | --- | Node selector for K8s pods, format: `"key1=val1,key2=val2"` | +| `SFORGE_E2B_TEMPLATE_MAP` | --- | Explicit E2B image-key to template mapping | +| `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL in seconds | +| `SFORGE_PORTABLE_REGISTRY_USERNAME` | --- | Registry username for portable E2B derivative images | +| `SFORGE_PORTABLE_REGISTRY_PASSWORD` | --- | Registry password for portable E2B derivative images | ## Path Variables @@ -93,6 +104,8 @@ Judge CPU/memory limits are currently set through CLI flags (`--judge-cpu-limit` | `SFORGE_LOG_DIR` | `logs/` | Override log output directory | | `SFORGE_TASKS_DIR` | `tasks/` | Override task definitions directory | | `SFORGE_REGISTRY` | --- | Default Docker registry URL for `pull`/`push` commands | +| `SFORGE_REGISTRY_USERNAME` | --- | Source-registry username for E2B Template builds | +| `SFORGE_REGISTRY_PASSWORD` | --- | Source-registry password for E2B Template builds | ## Proxy Fallback Chain diff --git a/docs/en/configuration/experiments.md b/docs/en/configuration/experiments.md index 5f0c9c9..2fe0bb8 100644 --- a/docs/en/configuration/experiments.md +++ b/docs/en/configuration/experiments.md @@ -63,7 +63,7 @@ Default settings applied to all tasks unless overridden per-task. | `disable_auto_resume` | `bool` | `false` | Disable auto-resume on abnormal agent exit | | `internet` | `bool` | --- | Enable/disable internet access | | `extra_env` | `dict` | --- | Extra environment variables injected into the agent container | -| `backend` | `string` | --- | Container backend (`docker` or `k8s`) | +| `backend` | `string` | --- | Container backend (`docker`, `k8s`, or `e2b`) | | `judge_url` | `string` | --- | Judge server URL override | | `max_submissions` | `int` | --- | Maximum number of agent submissions per run | | `submission_cooldown` | `int` | --- | Minimum seconds between agent submissions | diff --git a/docs/en/features/network-isolation.md b/docs/en/features/network-isolation.md index 6fd39c1..374fbf2 100644 --- a/docs/en/features/network-isolation.md +++ b/docs/en/features/network-isolation.md @@ -25,6 +25,7 @@ The implementation of `--disable-internet` depends on the container backend: |---------|----------------| | Docker | Host-side iptables/ip6tables rules | | Kubernetes | Kubernetes NetworkPolicy | +| E2B | E2B `update_network` allow/deny rules | ## Allowed Endpoints @@ -97,6 +98,16 @@ sforge run \ Whether NetworkPolicy actually blocks traffic depends on the cluster CNI. Some clusters allow NetworkPolicy objects to be created but do not enforce them. Validate isolation with a small task before relying on it. ::: +## E2B Backend + +The E2B backend applies the allowlist with the official `update_network` API. +E2B rules operate at host level rather than port level, so allowing a hostname +allows all ports on that host. The E2B API currently accepts the +IPv4 all-traffic selector (`0.0.0.0/0`) and rejects `::/0`; SForge therefore +uses the provider-supported selector rather than sending an invalid IPv6 rule. +The deny rule remains active until the Sandbox is deleted. Generic host proxy +variables are not forwarded to E2B Sandboxes. + ## Using with Proxy: `sforge proxy` Directly injecting `SFORGE_HTTP_PROXY` / `SFORGE_HTTPS_PROXY` into the Work container is incompatible with network isolation. If the AI API must be reached through a corporate proxy, run `sforge proxy` on the host and allow the Work container to reach only that local proxy. diff --git a/docs/en/reference/cli.md b/docs/en/reference/cli.md index b62c05c..c8cb481 100644 --- a/docs/en/reference/cli.md +++ b/docs/en/reference/cli.md @@ -114,7 +114,9 @@ The server exposes the [Judge HTTP API](/en/reference/judge-api) and handles bot ## sforge run -Run an agent on one or more tasks. This is the primary command for evaluation. +Run an agent on one task, or batch-schedule independent evaluations for several +tasks. Each task always has its own Work environment, Judge session, results, +and cleanup lifecycle. ### Basic usage @@ -122,7 +124,7 @@ Run an agent on one or more tasks. This is the primary command for evaluation. # Standard agent mode sforge run --task ad_placement_optimization --agent claude-code -# Multiple tasks in parallel +# Batch-schedule independent task runs sforge run --task ad_placement_optimization gitlet rookiedb --agent claude-code # Experiment config mode @@ -133,7 +135,7 @@ sforge run --experiment experiment.yaml | Flag | Default | Description | |------|---------|-------------| -| `--task` | required* | One or more task IDs (space-separated). Multiple tasks run fully in parallel. | +| `--task` | required* | One or more task IDs (space-separated). Multiple IDs batch-schedule independent task runs; they are not combined into one evaluation. | | `--agent` | required* | Agent name (e.g., `claude-code`, `codex`). Required unless `--experiment` is specified. | | `--experiment` | --- | Path to experiment YAML config file. If `--task` is omitted, all YAML tasks run; if `--task` is provided, only that subset runs with the experiment settings. | | `--model` | --- | Model override (e.g., `claude-opus-4-8`) | @@ -141,8 +143,9 @@ sforge run --experiment experiment.yaml | `--eval-interval` | `300` | Auto-eval daemon interval in seconds | | `--run-id` | random | Run identifier for tracking and log organization | | `--judge-url` | `http://host.docker.internal:8080` | Judge server URL as seen from inside the container | -| `--backend` | `docker` | Container backend (`docker` or `k8s`) | +| `--backend` | `docker` | Container backend (`docker`, `k8s`, or `e2b`) | | `--stagger` | --- | Spread task launches evenly over N seconds (e.g., `--stagger 300`) | +| `--max-workers` | backend-specific | Maximum number of independent task runs scheduled at once. Defaults to all selected tasks for Docker/Kubernetes and 4 for E2B. | | `--max-submissions` | --- | Maximum number of agent submissions per run | | `--submission-cooldown` | --- | Minimum seconds between agent submissions | | `--work-cpu-limit` | --- | Number of CPUs for work containers | @@ -192,7 +195,7 @@ sforge eval --task ad_placement_optimization --archive - < solution.tar.gz | `--run-id` | Custom run ID for log organization | | `--timeout` | Evaluation timeout in seconds | | `--json` | Also output the full JSON report | -| `--backend` | Container backend (`docker` or `k8s`) | +| `--backend` | Container backend (`docker`, `k8s`, or `e2b`) | | `--judge-cpu-limit` | Number of CPUs for judge container | | `--judge-mem-limit` | Memory limit for judge container (e.g., `'4g'`) | diff --git a/docs/en/reference/judge-api.md b/docs/en/reference/judge-api.md index 6b29b12..e0b89e1 100644 --- a/docs/en/reference/judge-api.md +++ b/docs/en/reference/judge-api.md @@ -16,7 +16,11 @@ All endpoints are prefixed with `/api/v1`. ## Authentication -The Judge API uses a **token-based session model**. Before submitting, clients must register a session to obtain a token. The token encodes the task ID, run ID, and auto-incrementing round counters. +The Judge API uses a **token-based session model**. The trusted host registers +a session with the `X-SForge-Admin-Secret` header; the returned session token is then used +for submissions, result polling, history, and Game routes. SForge generates a +mode-`0600` local secret when the variable is not set. Do not expose the admin +secret to Work Sandboxes or agents. ## Endpoints @@ -68,16 +72,15 @@ Submit a code archive for evaluation. The server resolves the task ID and run ID ```http POST /api/v1/submit +Authorization: Bearer Content-Type: multipart/form-data -token: a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4 archive: @solution.tar.gz kind: agent ``` | Field | Type | Required | Description | |-------|------|----------|-------------| -| `token` | string | yes | Session token from `/register` | | `archive` | file | yes | `.tar.gz` archive of the solution | | `kind` | string | no | `"agent"` (default) or `"auto"`. Determines which round counter is incremented. | @@ -99,6 +102,7 @@ Poll for the result of a submission. ```http GET /api/v1/result/{submission_id} +Authorization: Bearer ``` **Response (queued/running):** @@ -155,18 +159,13 @@ GET /api/v1/result/{submission_id} ### Run History -Retrieve the full submission history for a run session, including the best score selection. - -**By token:** - -```http -GET /api/v1/history?token= -``` - -**By run ID:** +Retrieve the agent-visible submission history for a run session, including +the best score selection. Host-side SForge requests add the admin secret to +include auto-eval entries. ```http -GET /api/v1/runs/{run_id}/history?task_id= +GET /api/v1/history +Authorization: Bearer ``` **Response:** @@ -202,12 +201,16 @@ The `best_*` fields are computed using the task's configured selection policy (s ### Game Endpoints -For tasks with `game_mode: true`, the Judge server manages interactive game sessions in dedicated containers. +For tasks with `game_mode: true`, the Judge server manages interactive game +sessions in dedicated containers. Every Game request requires +`Authorization: Bearer ` and the token must match the URL's +run and task. #### Start New Game ```http POST /api/v1/game/{run_id}/{task_id}/new +Authorization: Bearer Content-Type: application/json {} ``` @@ -230,6 +233,7 @@ Content-Type: application/json ```http POST /api/v1/game/{run_id}/{task_id}/{session_id}/step +Authorization: Bearer Content-Type: application/json {"action": "go north"} @@ -253,6 +257,7 @@ Content-Type: application/json ```http GET /api/v1/game/{run_id}/{task_id}/{session_id}/status +Authorization: Bearer ``` **Response:** @@ -272,6 +277,7 @@ GET /api/v1/game/{run_id}/{task_id}/{session_id}/status ```http POST /api/v1/game/{run_id}/{task_id}/{session_id}/close +Authorization: Bearer ``` **Response:** @@ -292,6 +298,7 @@ Close all active game sessions for a given run and task. ```http POST /api/v1/game/{run_id}/{task_id}/close-all +Authorization: Bearer ``` **Response:** @@ -308,16 +315,18 @@ Game sessions have a 10-minute idle timeout. Sessions are automatically archived ```bash # 1. Register a session +export SFORGE_ADMIN_SECRET=... TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/register \ -H "Content-Type: application/json" \ - -d '{"task_id": "ad_placement_optimization", "run_id": "run-001"}' \ + -H "X-SForge-Admin-Secret: $SFORGE_ADMIN_SECRET" \ + -d '{"task_id":"ad_placement_optimization","run_id":"run-001"}' \ | jq -r '.token') echo "Token: $TOKEN" # 2. Submit an archive SUBMISSION_ID=$(curl -s -X POST http://localhost:8080/api/v1/submit \ - -F "token=$TOKEN" \ + -H "Authorization: Bearer $TOKEN" \ -F "archive=@solution.tar.gz" \ -F "kind=agent" \ | jq -r '.submission_id') @@ -325,20 +334,24 @@ SUBMISSION_ID=$(curl -s -X POST http://localhost:8080/api/v1/submit \ echo "Submission: $SUBMISSION_ID" # 3. Poll for result (repeat until status is "completed" or "error") -curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq . +curl -s -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq . # 4. Check run history -curl -s "http://localhost:8080/api/v1/history?token=$TOKEN" | jq . +curl -s -H "Authorization: Bearer $TOKEN" \ + http://localhost:8080/api/v1/history | jq . ``` ### Poll loop ```bash while true; do - STATUS=$(curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq -r '.status') + STATUS=$(curl -s -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq -r '.status') echo "Status: $STATUS" if [ "$STATUS" = "completed" ] || [ "$STATUS" = "error" ]; then - curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq . + curl -s -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq . break fi sleep 5 diff --git a/docs/zh/configuration/container-backends.md b/docs/zh/configuration/container-backends.md index 5986bac..023dc17 100644 --- a/docs/zh/configuration/container-backends.md +++ b/docs/zh/configuration/container-backends.md @@ -4,7 +4,8 @@ title: 容器后端 # 容器后端 -SForge 通过容器后端创建 Work 容器和 Judge 容器。默认后端是本机 Docker；当需要把运行负载放到 Kubernetes 集群中时，可以切换到 `k8s` 后端。 +SForge 通过容器后端创建 Work 和 Judge 环境。默认后端是本机 Docker；已有 Kubernetes +集群时可以使用 `k8s`，需要 E2B 托管远端运行环境时可以使用 `e2b`。 ## 如何选择后端 @@ -12,8 +13,10 @@ SForge 通过容器后端创建 Work 容器和 Judge 容器。默认后端是本 |------|----------|----------| | `docker` | 本地开发、调试任务、单机评测、小规模实验 | 本机 Docker daemon 可用 | | `k8s` | 共享集群、批量并发评测、希望让 Work/Judge Pod 在集群中运行 | `kubectl` 可访问集群，镜像在集群中可拉取，Judge URL 可从 Pod 访问 | +| `e2b` | 不自行维护集群或 Judge broker 的远端 Work/Judge 运行 | E2B API 与 E2B Template；详见 [E2B 后端](#e2b-后端) | 如果只是本地试跑任务，优先使用默认的 Docker 后端。Kubernetes 后端更适合已经有集群和镜像仓库的团队环境。 +E2B 后端在首次运行新增或发生变化的任务镜像前，需要先准备 Template。 ::: warning Docker 后端不适合大批量运行 每个任务会占用一个 Work 容器外加临时 Judge 容器，各自有独立的 CPU/内存限额。单机并发运行大量任务（约 **20 个以上**）即使是高性能服务器也会出现严重的资源争抢。大批量运行请使用 `k8s` 后端。 @@ -29,15 +32,13 @@ sforge run \ --task ad_placement_optimization \ --agent claude-code \ --backend k8s \ - --judge-url http://10.0.0.12:8080 --backend k8s + --judge-url http://10.0.0.12:8080 # 环境变量覆盖 export SFORGE_BACKEND=k8s sforge run \ --task ad_placement_optimization \ - --agent claude-code \ - --backend k8s \ - --judge-url http://10.0.0.12:8080 + --agent claude-code ``` 实验配置中也可以设置默认后端： @@ -51,7 +52,7 @@ tasks: ad_placement_optimization: {} ``` -## Docker backend +## Docker 后端 Docker 是默认后端，不需要显式配置： @@ -62,9 +63,7 @@ sforge serve export SFORGE_AGENT_API_KEY="sk-..." sforge run \ --task ad_placement_optimization \ - --agent claude-code \ - --backend k8s \ - --judge-url http://10.0.0.12:8080 + --agent claude-code ``` Docker 后端会： @@ -86,7 +85,7 @@ sforge run \ --judge-mem-limit 4g ``` -## Kubernetes backend +## Kubernetes 后端 Kubernetes 后端通过 `kubectl` 创建 Pod。每个 SForge 容器会对应到集群中的一个 Pod，容器名为 `work`，Pod 使用标签 `app=sforge` 和 `sforge-pod=`。 @@ -222,13 +221,164 @@ sforge run \ - 当前 kubeconfig 需要有创建和删除 `NetworkPolicy` 的权限。 - 不同集群的默认 DNS、出口网关和策略实现可能不同，建议先用小任务验证隔离效果。 -## 常见问题 +## E2B 后端 + +E2B 后端使用 E2B Sandbox 运行 Work、Judge 和 Game 环境。不传 `--judge-url` 时， +SForge 还会创建临时的 managed Judge Controller，用户不需要单独部署 Judge 服务。 + +### Docker 镜像与 E2B Template + +[`seededge`](https://hub.docker.com/u/seededge) 已经发布 EdgeBench 的预构建镜像。这些镜像 +可以直接作为 E2B Template 的源镜像；对于没有修改过的公开任务，不需要执行 +`sforge build`、`sforge pull` 或 `sforge push`。 + +Docker 镜像和 E2B Template 是两种不同的运行对象。`Template.from_image()` 将 Docker +镜像导入 E2B Template；`Sandbox.create()` 使用生成的 Template 名称或 ID 启动 Sandbox， +不能直接传入 Docker image 引用。SForge 会自动完成转换： + +```text +seededge Docker 镜像 + -> sforge e2b-template + -> 用户 E2B team 中的 Template + -> sforge run --backend e2b + -> E2B Sandbox +``` + +每个任务在一个 E2B team 中首次使用或镜像发生变化时需要准备 Template。输入一致时会 +复用已有 Template；日常 `sforge run` 不会重新构建。 + +### 准备 Template + +```bash +uv sync --extra e2b +export E2B_API_KEY=... + +sforge fetch-tasks edgebench +sforge e2b-template \ + --task ad_placement_optimization \ + --source-registry seededge +``` + +准备所有已下载任务时，将 `--task` 换成 `--all`。SForge 会自动推导两个镜像引用、固定 +不可变 digest、恢复非敏感 OCI 环境变量、快照镜像初始的 `/tmp`、构建 Work/Judge Template、 +等待明确终态、重试暂态错误、记录版本化 manifest，并复用匹配的已有构建。用户不需要 +编写 E2B Dockerfile，也不需要手工上传镜像层。 + +当前实现要求在 Linux amd64 主机上运行，并且本机 Docker daemon 可用，但不会重新构建 +已发布镜像。官方任务镜像早于 E2B，会在 `/tmp` 下预置评测辅助文件：命令直接引用的包装 +脚本，以及包装脚本运行时读取的其他文件。Docker 与 Kubernetes 直接从镜像启动，这些文件 +天然存在；而 E2B 每个 Sandbox 都会重置 `/tmp`。因此当某个 role 的命令引用 `/tmp` 时， +SForge 会快照整个镜像 `/tmp`（连同仅被间接引用的辅助文件一起保存），存入 Template，并在 +Sandbox 启动时恢复，使评测器看到与其他后端一致的文件。此过程不修改官方镜像、评测器或 +评分逻辑。源 Registry 还需要能被 E2B 构建服务访问。 + +### 可移植镜像回退 + +少数版本的镜像包含 E2B 构建网络无法访问的 APT 源。这时需要提供一个可写 Registry： + +```bash +export SFORGE_PORTABLE_REGISTRY_USERNAME=... +export SFORGE_PORTABLE_REGISTRY_PASSWORD=... + +sforge e2b-template \ + --task TASK \ + --source-registry seededge \ + --portable-registry registry.example.com/project +``` + +只有 E2B 返回 APT provisioning 失败时，SForge 才会创建版本化 portable derivative。 +该回退需要 Registry 写权限，但不会在每次评测时重复构建。 + +### 运行任务 + +```bash +export SFORGE_AGENT_API_KEY=... +export SFORGE_AGENT_API_BASE_URL=... +export SFORGE_AGENT_MODEL=... + +sforge run \ + --backend e2b \ + --task ad_placement_optimization \ + --agent claude-code +``` + +SForge 会预检 Work/Judge Template，创建 secured Controller，注册 task，按需创建 +Work/Judge/Game Sandbox，传递 gateway 凭证，收集结果并清理 Sandbox。只有使用独立部署的 +Judge Server 时才需要传 `--judge-url`。 + +Work Sandbox 通过受保护的 E2B service gateway 访问 managed Controller；host 侧编排请求 +则通过 E2B command channel 访问 Controller 的 loopback 地址，避免不必要地绕行公网 +gateway。这个传输选择只用于 E2B，不改变 Docker 或 Kubernetes 的 Judge 路由。 + +每个 E2B 评测实例只对应一个 task。传入多个 task ID 时，CLI 只是批量调度多个相互独立的 +评测，不会把多个 task 合并进同一个 Work 或 Judge 环境。本文验证采用每条 +`sforge run` 命令运行一个 task 的方式。 + +### 套餐限制与运行规模 + +E2B 套餐限制是运行契约的一部分。Template 的 CPU 和内存会在构建时固化；超过当前 team +上限的请求会在创建 Template 时被拒绝。应使用目标 team 支持的规格重新构建 Template； +运行时的 `--work-*` 和 `--judge-*` 参数不能调整已有 E2B Template 的资源。 + +Agent timeout 应明显短于当前套餐允许的最大 Sandbox 生命周期。Sandbox 生命周期还要覆盖 +Agent 安装、归档提取、等待 Judge 评测、读取结果和资源清理。managed Judge Controller 会在 +整条多任务命令期间持续运行，因此整个错峰或多波次批次都必须落在 Controller 允许的生命周期 +内，而不只是单个 task。比如套餐只允许一小时 Sandbox 时，可使用 30 分钟等更短的 Agent +预算，并避免让同一个 managed Controller 因多波调度运行满一小时；也可以改用独立部署的 +Judge Server。 + +还需要检查每个任务的 `judge.eval_timeout`。Judge Sandbox 同样受 team 最大生命周期限制； +如果任务声明的评测超时长于该限制，那么当 evaluator 实际使用完整预算时，就无法保证评测 +完成。如果要求与榜单口径可比，不应静默缩短 evaluator timeout，而应使用生命周期足够长的 +套餐。 + +`--max-workers` 只限制并行 Work Sandbox 数量，不等于 E2B 总实例数。容量规划还要计入一个 +managed Controller、临时 Judge Sandbox 和活跃 Game Sandbox，并为评测完成和资源清理的 +重叠阶段留出余量。 + +最终分数沿用 SForge 与后端无关的既有语义：只在已经完成的 agent submission 和 +auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快照，timeout 时不会被 +隐式提交。短任务应选择能为至少一次完整评分留出时间的 auto-eval 间隔，或者让 Agent 在 +结束前主动执行 `sforge-submit`。 + +### 用户需要提供的内容 + +| 输入 | 需要场景 | +| --- | --- | +| `E2B_API_KEY` | Template 准备和 E2B 运行 | +| Agent API key、base URL 和模型 | 运行 Agent | +| Linux amd64 主机和本机 Docker daemon | 使用当前实现准备 Template | +| `--source-registry seededge` | 使用已发布的 EdgeBench 镜像 | +| 源 Registry 凭证 | 仅使用私有源 Registry 时 | +| `--portable-registry` 及其凭证 | 仅需要 APT 可移植性回退时 | +| CPU/内存覆盖参数 | 仅 task 默认值不合适时；E2B 资源固化在 Template 中 | + +宿主机的 `HTTP_PROXY` 和 `HTTPS_PROXY` 不会复制到 E2B Work、Judge 或 Game Sandbox。 +只有用户显式配置 `SFORGE_HTTP_PROXY` 或 `SFORGE_HTTPS_PROXY` 时，远端可访问的代理才会 +被转发；网络隔离任务会移除全部代理变量。详见[网络隔离](/zh/features/network-isolation)。 + +### E2B 常见问题 | 现象 | 可能原因 | 处理方式 | -|------|----------|----------| -| `kubectl cluster-info failed` | kubeconfig 不正确、集群不可达或 namespace 参数有误 | 检查 `kubectl -n cluster-info`；必要时设置 `SFORGE_K8S_KUBECONFIG` 和 `SFORGE_K8S_NAMESPACE` | -| Pod 一直无法 Running | 镜像拉取失败、调度失败或资源不足 | 使用 `kubectl -n describe pod ` 查看事件 | -| Pod 拉不到镜像 | 镜像没有推送到集群可访问的 registry，或 registry 凭证未配置 | 先 `sforge push`；确认 `SFORGE_K8S_IMAGE_REGISTRY` 正确；为 namespace 配置 image pull secret | -| Work Pod 无法提交评测 | `--judge-url` 不是 Pod 可访问地址，或 Judge server 没有监听外部地址 | 用 `--host 0.0.0.0` 启动 `sforge serve`，并通过 `--judge-url` 设置 Pod 可访问的 IP/Service URL | -| `--disable-internet` 没有效果 | 集群 CNI 不支持 NetworkPolicy 或策略权限不足 | 确认 CNI 支持 NetworkPolicy，并检查当前身份是否能创建 NetworkPolicy | -| node selector 后 Pod Pending | 没有节点匹配 selector | 检查 `SFORGE_K8S_NODE_SELECTOR` 和节点标签 | +| --- | --- | --- | +| 找不到 Template | 当前 E2B team 尚未转换该镜像，或输入已经变化 | 对该 task 执行 `sforge e2b-template` | +| E2B 无法拉取 `seededge` 镜像 | Registry 网络故障或精确 tag 不存在 | 核对 task 定义后重试 | +| Template provisioning 无法访问 APT 源 | 源镜像包含不可达 mirror | 提供 `--portable-registry` 和凭证 | +| Template 构建返回 internal error，或超过客户端等待时间后仍为 `building` | E2B 构建服务暂态故障；SDK 没有单次构建取消 API | 重试前先检查目标 tag 是否已经可启动，再只重试失败的 role；避免同时触发大量冷构建 | +| `/tmp` 下的 evaluator 缺失 | Template 未通过 SForge 构建，缺少镜像 `/tmp` 快照 | 用 `sforge e2b-template` 重建；SForge 会快照整个镜像 `/tmp` 并在 Sandbox 启动时恢复 | +| secured Judge 返回 403 | 缺少 traffic access token | 使用 managed 模式，或为外部 secured Judge 配置 `SFORGE_JUDGE_ACCESS_TOKEN` | +| Agent 侧 Judge 请求偶发超时 | Work Sandbox 无法通过 E2B 公网 gateway 建立连接 | 让 Agent 重试提交、降低同时提交的突发并发，或使用独立 Judge Server；host 侧 managed-controller 请求已通过 E2B command channel 走 loopback | +| task timeout 到达时出现 `Sandbox not found` | 在最终归档和清理前达到了 team 级 Sandbox 最大生命周期 | 缩短 Agent timeout、减少多波次批次总时长，或使用支持更长 Sandbox 生命周期的套餐 | +| 准备 Template 时 CPU 或内存被拒绝 | 请求规格超过当前 E2B team 上限 | 使用支持的规格重建；若榜单配置要求更高资源，则两者不具备严格可比性 | +| 运行结束后仍有 Sandbox | cleanup 失败或 Runner 被强制终止 | 检查 run 日志；平台最终按配置的 E2B TTL 回收 | + +## 常见问题 + +| 后端 | 现象 | 可能原因 | 处理方式 | +| --- | --- | --- | --- | +| Kubernetes | `kubectl cluster-info failed` | kubeconfig 不正确、集群不可达或 namespace 参数有误 | 检查 `kubectl -n cluster-info`；必要时设置 `SFORGE_K8S_KUBECONFIG` 和 `SFORGE_K8S_NAMESPACE` | +| Kubernetes | Pod 一直无法 Running | 镜像拉取失败、调度失败或资源不足 | 使用 `kubectl -n describe pod ` 查看事件 | +| Kubernetes | Pod 拉不到镜像 | 镜像对集群节点不可见，或 registry 凭证未配置 | 先 `sforge push`；确认 `SFORGE_K8S_IMAGE_REGISTRY`；为 namespace 配置 image pull secret | +| Kubernetes | Work Pod 无法提交评测 | Judge URL 对 Pod 不可达，或服务没有监听外部地址 | 用 `--host 0.0.0.0` 启动 `sforge serve`，并设置 Pod 可访问的 IP 或 Service URL | +| Kubernetes | `--disable-internet` 没有效果 | CNI 不执行 NetworkPolicy，或权限不足 | 确认 CNI 支持 NetworkPolicy，并检查当前身份的权限 | +| Kubernetes | 配置 node selector 后 Pod Pending | 没有节点匹配 selector | 检查 `SFORGE_K8S_NODE_SELECTOR` 和节点标签 | diff --git a/docs/zh/configuration/environment-variables.md b/docs/zh/configuration/environment-variables.md index 1a7fff6..8970c90 100644 --- a/docs/zh/configuration/environment-variables.md +++ b/docs/zh/configuration/environment-variables.md @@ -59,6 +59,10 @@ SForge 大部分配置可通过 `SFORGE_*` 环境变量注入；仅 CLI/YAML 支 把 `SFORGE_HTTP_PROXY` / `SFORGE_HTTPS_PROXY` 直接注入 Agent 容器，会让容器通过代理获得网络访问能力。这个方式只适合极少数运行时下载依赖的场景，**不推荐**用于 LLM API 访问。 它也不兼容网络隔离模式（`--disable-internet`）。如果需要在网络隔离下通过企业代理访问 LLM API，请在宿主机运行 `sforge proxy`，并把 `SFORGE_AGENT_API_BASE_URL` 指向 `http://host.docker.internal:`。 + +对于 E2B，宿主机通用的 `HTTP_PROXY` / `HTTPS_PROXY` 不会自动传入 Work Sandbox。只有 +显式设置 `SFORGE_` 代理变量且远端能够访问该代理时才应转发；隔离模式始终移除代理并由 +E2B 原生域名白名单直接放行。 ::: ## Judge 变量 @@ -66,6 +70,8 @@ SForge 大部分配置可通过 `SFORGE_*` 环境变量注入；仅 CLI/YAML 支 | 变量 | 用途 | |------|------| | `SFORGE_JUDGE_EXTRA_ENV` | Judge 容器额外环境变量，格式：`"KEY1=VAL1,KEY2=VAL2"` | +| `SFORGE_ADMIN_SECRET` | 可信 Host 用于注册 Judge 会话和 auto-eval 的凭证；未设置时在 `SFORGE_LOG_DIR` 下生成权限为 `0600` 的 secret 文件，不得注入 Work 容器。 | +| `SFORGE_JUDGE_ACCESS_TOKEN` | 显式配置 secured Judge URL 时使用的 E2B traffic access token；managed 模式自动设置。 | ## 资源限制变量 @@ -80,11 +86,15 @@ Judge CPU/内存限制目前通过 CLI 参数（`--judge-cpu-limit`、`--judge-m | 变量 | 默认值 | 用途 | |------|--------|------| -| `SFORGE_BACKEND` | `docker` | 容器后端：`docker` 或 `k8s` | +| `SFORGE_BACKEND` | `docker` | 容器后端：`docker`、`k8s` 或官方 `e2b` | | `SFORGE_K8S_NAMESPACE` | `default` | Kubernetes 命名空间 | | `SFORGE_K8S_IMAGE_REGISTRY` | --- | K8s 镜像拉取使用的容器镜像仓库 | | `SFORGE_K8S_KUBECONFIG` | --- | kubeconfig 文件路径 | | `SFORGE_K8S_NODE_SELECTOR` | --- | K8s Pod 的节点选择器，格式：`"key1=val1,key2=val2"` | +| `SFORGE_E2B_TEMPLATE_MAP` | --- | E2B image key 到 Template 的显式映射 | +| `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL（秒） | +| `SFORGE_PORTABLE_REGISTRY_USERNAME` | --- | E2B 可移植派生镜像 Registry 用户名 | +| `SFORGE_PORTABLE_REGISTRY_PASSWORD` | --- | E2B 可移植派生镜像 Registry 密码 | ## 路径变量 @@ -93,6 +103,8 @@ Judge CPU/内存限制目前通过 CLI 参数（`--judge-cpu-limit`、`--judge-m | `SFORGE_LOG_DIR` | `logs/` | 覆盖日志输出目录 | | `SFORGE_TASKS_DIR` | `tasks/` | 覆盖任务定义目录 | | `SFORGE_REGISTRY` | --- | Docker Registry URL，用于 `pull`/`push` 命令 | +| `SFORGE_REGISTRY_USERNAME` | --- | E2B Template 源镜像 Registry 用户名 | +| `SFORGE_REGISTRY_PASSWORD` | --- | E2B Template 源镜像 Registry 密码 | ## 代理回退链 diff --git a/docs/zh/configuration/experiments.md b/docs/zh/configuration/experiments.md index b74318c..8a9358c 100644 --- a/docs/zh/configuration/experiments.md +++ b/docs/zh/configuration/experiments.md @@ -63,7 +63,7 @@ tasks: | `disable_auto_resume` | `bool` | `false` | 禁用 Agent 异常退出时的自动恢复 | | `internet` | `bool` | --- | 启用/禁用网络访问 | | `extra_env` | `dict` | --- | 注入 Agent 容器的额外环境变量 | -| `backend` | `string` | --- | 容器后端（`docker` 或 `k8s`） | +| `backend` | `string` | --- | 容器后端（`docker`、`k8s` 或 `e2b`） | | `judge_url` | `string` | --- | Judge 服务器 URL 覆盖 | | `max_submissions` | `int` | --- | 每次运行中 Agent 最大提交次数 | | `submission_cooldown` | `int` | --- | Agent 两次提交之间的最小间隔（秒） | diff --git a/docs/zh/features/network-isolation.md b/docs/zh/features/network-isolation.md index 8b84225..00b6ef1 100644 --- a/docs/zh/features/network-isolation.md +++ b/docs/zh/features/network-isolation.md @@ -25,6 +25,7 @@ sforge run --task --agent claude-code --enable-internet |---------|----------| | Docker | 宿主机 iptables/ip6tables 规则 | | Kubernetes | Kubernetes NetworkPolicy | +| E2B | E2B `update_network` allow/deny 规则 | ## 允许访问的端点 @@ -97,6 +98,14 @@ sforge run \ NetworkPolicy 是否真正生效取决于集群 CNI。部分集群即使成功创建 NetworkPolicy，也可能不会实际拦截流量。首次使用时建议用小任务验证隔离效果。 ::: +## E2B backend + +E2B backend 通过官方 `update_network` API 下发白名单。E2B 规则按 Host +生效，不支持端口粒度；允许某个域名意味着允许访问该域名的所有端口。E2B API +当前接受 IPv4 全流量选择器（`0.0.0.0/0`），但会拒绝 `::/0`，因此 SForge 使用平台 +支持的选择器，不发送无效的 IPv6 规则。deny 规则会一直保留到 Sandbox 被删除，宿主机 +通用代理变量也不会自动传入 E2B Sandbox。 + ## 配合代理使用：`sforge proxy` 直接把 `SFORGE_HTTP_PROXY` / `SFORGE_HTTPS_PROXY` 注入 Work 容器与网络隔离不兼容。如果 AI API 必须通过企业代理访问，推荐在宿主机上启动 `sforge proxy`，然后只允许 Work 容器访问这个本地代理。 diff --git a/docs/zh/reference/cli.md b/docs/zh/reference/cli.md index c379220..a420c80 100644 --- a/docs/zh/reference/cli.md +++ b/docs/zh/reference/cli.md @@ -114,7 +114,8 @@ sforge serve --host 0.0.0.0 --port 9090 ## sforge run -在一个或多个任务上运行 Agent。这是评测的核心命令。 +在一个 task 上运行 Agent，或批量调度多个相互独立的 task 评测。每个 task 始终拥有各自的 +Work 环境、Judge session、结果和清理生命周期。 ### 基本用法 @@ -122,7 +123,7 @@ sforge serve --host 0.0.0.0 --port 9090 # 标准 Agent 模式 sforge run --task ad_placement_optimization --agent claude-code -# 多任务并行运行 +# 批量调度多个独立 task sforge run --task ad_placement_optimization gitlet rookiedb --agent claude-code # 实验配置模式 @@ -133,7 +134,7 @@ sforge run --experiment experiment.yaml | 选项 | 默认值 | 说明 | |------|--------|------| -| `--task` | 必填* | 一个或多个任务 ID（空格分隔）。多任务时完全并行运行。 | +| `--task` | 必填* | 一个或多个任务 ID（空格分隔）。传入多个 ID 时，CLI 批量调度多个独立 task，不会把它们合并为一次评测。 | | `--agent` | 必填* | Agent 名称（如 `claude-code`、`codex`）。除非指定 `--experiment`，否则必填。 | | `--experiment` | --- | 实验 YAML 配置文件路径。如果未指定 `--task`，运行 YAML 中的全部任务；如果指定了 `--task`，则只运行该子集并套用实验配置。 | | `--model` | --- | 模型覆盖（如 `claude-opus-4-8`） | @@ -141,8 +142,9 @@ sforge run --experiment experiment.yaml | `--eval-interval` | `300` | 自动评测守护进程的间隔时间（秒） | | `--run-id` | 随机生成 | 运行标识符，用于跟踪和日志组织 | | `--judge-url` | `http://host.docker.internal:8080` | 容器内部看到的 Judge 服务器 URL | -| `--backend` | `docker` | 容器后端（`docker` 或 `k8s`） | +| `--backend` | `docker` | 容器后端（`docker`、`k8s` 或 `e2b`） | | `--stagger` | --- | 将任务启动均匀分散在 N 秒内（如 `--stagger 300`） | +| `--max-workers` | 取决于后端 | 同时调度的独立 task 数量上限。Docker/Kubernetes 默认为全部所选任务，E2B 默认为 4。 | | `--max-submissions` | --- | 每次运行的最大 Agent 提交次数 | | `--submission-cooldown` | --- | Agent 两次提交之间的最小间隔（秒） | | `--work-cpu-limit` | --- | Work 容器的 CPU 数量限制 | @@ -192,7 +194,7 @@ sforge eval --task ad_placement_optimization --archive - < solution.tar.gz | `--run-id` | 自定义运行 ID，用于日志组织 | | `--timeout` | 评测超时时间（秒） | | `--json` | 同时输出完整的 JSON 报告 | -| `--backend` | 容器后端（`docker` 或 `k8s`） | +| `--backend` | 容器后端（`docker`、`k8s` 或 `e2b`） | | `--judge-cpu-limit` | Judge 容器的 CPU 数量限制 | | `--judge-mem-limit` | Judge 容器的内存限制（如 `'4g'`） | diff --git a/docs/zh/reference/judge-api.md b/docs/zh/reference/judge-api.md index 3b7ed64..7f315f8 100644 --- a/docs/zh/reference/judge-api.md +++ b/docs/zh/reference/judge-api.md @@ -16,7 +16,10 @@ http://localhost:8080 ## 认证机制 -Judge API 使用**基于 Token 的会话模型**。在提交之前，客户端必须先注册会话以获取 Token。Token 编码了任务 ID、运行 ID 和自增的轮次计数器。 +Judge API 使用**基于 Token 的会话模型**。可信 Host 使用 +`X-SForge-Admin-Secret` Header 注册会话，返回的会话 Token 用于提交、结果轮询、历史和 Game +接口。未显式配置时，SForge 会在本地生成权限为 `0600` 的 secret 文件。不得把 admin +secret 暴露给 Work Sandbox 或 Agent。 ## 接口列表 @@ -68,16 +71,15 @@ Token 是一个 32 字符的十六进制字符串，内部维护任务 ID、运 ```http POST /api/v1/submit +Authorization: Bearer Content-Type: multipart/form-data -token: a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4 archive: @solution.tar.gz kind: agent ``` | 字段 | 类型 | 必填 | 说明 | |------|------|------|------| -| `token` | string | 是 | 从 `/register` 获取的会话 Token | | `archive` | file | 是 | `.tar.gz` 格式的解决方案归档 | | `kind` | string | 否 | `"agent"`（默认）或 `"auto"`。决定递增哪个轮次计数器。 | @@ -99,6 +101,7 @@ kind: agent ```http GET /api/v1/result/{submission_id} +Authorization: Bearer ``` **响应（排队中/运行中）：** @@ -155,18 +158,12 @@ GET /api/v1/result/{submission_id} ### 运行历史 -获取某次运行会话的完整提交历史，包括最佳分数的选定结果。 - -**通过 Token 查询：** - -```http -GET /api/v1/history?token= -``` - -**通过运行 ID 查询：** +获取某次运行会话中 Agent 可见的提交历史和最佳分数。SForge 的 Host 请求会额外携带 +admin secret，以包含 auto-eval 记录。 ```http -GET /api/v1/runs/{run_id}/history?task_id= +GET /api/v1/history +Authorization: Bearer ``` **响应：** @@ -202,12 +199,15 @@ GET /api/v1/runs/{run_id}/history?task_id= ### 游戏接口 -对于 `game_mode: true` 的任务，Judge 服务器在专用容器中管理交互式游戏会话。 +对于 `game_mode: true` 的任务，Judge 服务器在专用容器中管理交互式游戏会话。所有 +Game 请求都必须携带 `Authorization: Bearer `，且 Token 必须与 URL 中 +的 run 和 task 匹配。 #### 创建新游戏 ```http POST /api/v1/game/{run_id}/{task_id}/new +Authorization: Bearer Content-Type: application/json {} ``` @@ -230,6 +230,7 @@ Content-Type: application/json ```http POST /api/v1/game/{run_id}/{task_id}/{session_id}/step +Authorization: Bearer Content-Type: application/json {"action": "go north"} @@ -253,6 +254,7 @@ Content-Type: application/json ```http GET /api/v1/game/{run_id}/{task_id}/{session_id}/status +Authorization: Bearer ``` **响应：** @@ -272,6 +274,7 @@ GET /api/v1/game/{run_id}/{task_id}/{session_id}/status ```http POST /api/v1/game/{run_id}/{task_id}/{session_id}/close +Authorization: Bearer ``` **响应：** @@ -292,6 +295,7 @@ POST /api/v1/game/{run_id}/{task_id}/{session_id}/close ```http POST /api/v1/game/{run_id}/{task_id}/close-all +Authorization: Bearer ``` **响应：** @@ -308,16 +312,18 @@ POST /api/v1/game/{run_id}/{task_id}/close-all ```bash # 1. 注册会话 +export SFORGE_ADMIN_SECRET=... TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/register \ -H "Content-Type: application/json" \ - -d '{"task_id": "ad_placement_optimization", "run_id": "run-001"}' \ + -H "X-SForge-Admin-Secret: $SFORGE_ADMIN_SECRET" \ + -d '{"task_id":"ad_placement_optimization","run_id":"run-001"}' \ | jq -r '.token') echo "Token: $TOKEN" # 2. 提交归档 SUBMISSION_ID=$(curl -s -X POST http://localhost:8080/api/v1/submit \ - -F "token=$TOKEN" \ + -H "Authorization: Bearer $TOKEN" \ -F "archive=@solution.tar.gz" \ -F "kind=agent" \ | jq -r '.submission_id') @@ -325,20 +331,24 @@ SUBMISSION_ID=$(curl -s -X POST http://localhost:8080/api/v1/submit \ echo "Submission: $SUBMISSION_ID" # 3. 轮询结果（重复直到 status 为 "completed" 或 "error"） -curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq . +curl -s -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq . # 4. 查看运行历史 -curl -s "http://localhost:8080/api/v1/history?token=$TOKEN" | jq . +curl -s -H "Authorization: Bearer $TOKEN" \ + http://localhost:8080/api/v1/history | jq . ``` ### 轮询循环 ```bash while true; do - STATUS=$(curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq -r '.status') + STATUS=$(curl -s -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq -r '.status') echo "Status: $STATUS" if [ "$STATUS" = "completed" ] || [ "$STATUS" = "error" ]; then - curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq . + curl -s -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq . break fi sleep 5 From b83e0dcfb394d62350891d6f931aae7a58bc8a6d Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 25 Aug 2026 06:41:42 -0400 Subject: [PATCH 03/17] Remove the e2b-template build step: official Templates are pre-built and published by maintainers --- docs/en/configuration/container-backends.md | 93 +- .../en/configuration/environment-variables.md | 4 - docs/zh/configuration/container-backends.md | 80 +- .../zh/configuration/environment-variables.md | 4 - sforge/cli.py | 178 --- sforge/harness/e2b_templates.py | 1248 ----------------- sforge/harness/run_agent.py | 7 +- 7 files changed, 26 insertions(+), 1588 deletions(-) delete mode 100644 sforge/harness/e2b_templates.py diff --git a/docs/en/configuration/container-backends.md b/docs/en/configuration/container-backends.md index 7e4eb7d..28212c9 100644 --- a/docs/en/configuration/container-backends.md +++ b/docs/en/configuration/container-backends.md @@ -231,81 +231,22 @@ Controller, so no separately deployed Judge service is required. ### Docker Images and E2B Templates -The pre-built EdgeBench images published under -[`seededge`](https://hub.docker.com/u/seededge) can be used directly as source -images. You do not need to run `sforge build`, `sforge pull`, or `sforge push` -for an unchanged published task. +A Docker image and an E2B Template are different runtime objects: E2B +Sandboxes start from Templates, not from Docker image references. Official +E2B Templates for the published EdgeBench tasks are pre-built and published +by the SForge maintainers, so no template preparation step is required. +`sforge run --backend e2b` derives each task image's Template reference +automatically (`edgebench.work.foo_bar:abc123` -> +`edgebench-work-foo-bar:abc123`). Set `SFORGE_E2B_TEMPLATE_MAP` only to +override this mapping, for example to run self-built Templates for a +modified task. -A Docker image and an E2B Template are different runtime objects. -`Template.from_image()` imports the Docker image into an E2B Template, while -`Sandbox.create()` starts the resulting Template rather than accepting a -Docker image reference. SForge automates this conversion: - -```text -seededge Docker image - -> sforge e2b-template - -> E2B Template in the user's E2B team - -> sforge run --backend e2b - -> E2B Sandbox -``` - -Template preparation is required once for a new or changed task image in an -E2B team. Matching Templates are reused; normal `sforge run` commands do not -rebuild them. - -### Prepare Templates +### Run a Task ```bash uv sync --extra e2b export E2B_API_KEY=... -sforge fetch-tasks edgebench -sforge e2b-template \ - --task ad_placement_optimization \ - --source-registry seededge -``` - -Use `--all` instead of `--task` to prepare all downloaded tasks. SForge -automatically derives both image references, pins immutable digests, restores -non-sensitive OCI environment variables, snapshots the image's initial `/tmp`, -builds the Work and Judge Templates, waits for terminal status, retries -transient failures, writes versioned manifests, and reuses matching builds. -Users do not write an E2B Dockerfile or upload image layers manually. - -The current implementation requires a Linux amd64 host with a running local -Docker daemon, but does not rebuild the published images. The official task -images predate E2B and ship evaluator helpers under `/tmp` — a wrapper the -command names plus other files that wrapper reads at runtime. Docker and -Kubernetes boot from the image, so those files are simply present; E2B resets -`/tmp` on every Sandbox. When a role's command references `/tmp`, SForge snapshots -the whole image `/tmp` (so helpers reached only indirectly come along too), -stores it in the Template, and restores it on Sandbox start, so the evaluator -sees the same files as on the other backends. This never modifies the official -image, the evaluators, or scoring. The source registry must also be reachable by -the E2B build service. - -### Portable Image Fallback - -Some image versions contain an APT source that is inaccessible from the E2B -build network. For those images, provide a writable registry: - -```bash -export SFORGE_PORTABLE_REGISTRY_USERNAME=... -export SFORGE_PORTABLE_REGISTRY_PASSWORD=... - -sforge e2b-template \ - --task TASK \ - --source-registry seededge \ - --portable-registry registry.example.com/project -``` - -SForge creates a versioned portable derivative only after E2B reports an APT -provisioning failure. Registry push access is required for this fallback, but -the derivative is not rebuilt for every evaluation run. - -### Run a Task - -```bash export SFORGE_AGENT_API_KEY=... export SFORGE_AGENT_API_BASE_URL=... export SFORGE_AGENT_MODEL=... @@ -372,13 +313,8 @@ ends. | Input | When required | | --- | --- | -| `E2B_API_KEY` | Template preparation and E2B runs | +| `E2B_API_KEY` | E2B runs | | Agent API key, base URL, and model | Agent runs | -| Linux amd64 host and local Docker daemon | Template preparation with the current implementation | -| `--source-registry seededge` | Published EdgeBench images | -| Source registry credentials | Private source registries only | -| `--portable-registry` and credentials | APT portability fallback only | -| CPU/memory overrides | Only when task defaults are unsuitable; E2B resources are fixed in the Template | Host `HTTP_PROXY` and `HTTPS_PROXY` variables are not copied into E2B Work, Judge, or Game Sandboxes. A remote-reachable proxy is forwarded only when the @@ -389,15 +325,10 @@ tasks remove proxy variables entirely. See [Network Isolation](/en/features/netw | Symptom | Likely cause | Fix | | --- | --- | --- | -| Template not found | The image has not been converted in this E2B team, or its inputs changed | Run `sforge e2b-template` for the task | -| E2B cannot pull a `seededge` image | Registry/network failure or the exact tag is unavailable | Confirm the task definition and retry | -| Template provisioning cannot reach an APT source | The source image contains an inaccessible mirror | Supply `--portable-registry` and credentials | -| A Template build reports an internal error or remains `building` past the client timeout | Transient E2B build-service failure; the SDK has no per-build cancellation API | Check whether the requested tag became launchable before retrying, then retry the specific role; avoid starting many cold builds at once | -| A runtime evaluator under `/tmp` is missing | The Template was built outside SForge, so the image's `/tmp` snapshot is absent | Rebuild with `sforge e2b-template`; SForge snapshots the whole image `/tmp` and restores it on Sandbox start | +| Template not found | The task image version has no published official Template, or the task uses a modified image | Verify the task uses a published image version, or point `SFORGE_E2B_TEMPLATE_MAP` at a self-built Template | | A secured Judge returns 403 | Its traffic access token was not provided | Use managed mode, or set `SFORGE_JUDGE_ACCESS_TOKEN` for an external secured Judge | | Agent-side Judge requests intermittently time out | The Work Sandbox could not establish a connection through the E2B public gateway | Retry the agent submission, reduce simultaneous submission bursts, or use an external Judge Server; host-side managed-controller calls already use E2B command-channel loopback | | A run ends at the configured task timeout with `Sandbox not found` | The team-level maximum Sandbox lifetime was reached before final extraction and cleanup | Shorten the agent timeout, reduce multi-wave batch duration, or use a plan with a longer Sandbox lifetime | -| CPU or memory is rejected while preparing a Template | The requested Template resources exceed the E2B team's limits | Rebuild at supported values; note that this is not equivalent to leaderboard resource settings when those require more resources | | A run leaves a Sandbox | Cleanup failed or the runner was terminated abruptly | Inspect the run log and wait for the configured E2B TTL as the final safeguard | ## Troubleshooting diff --git a/docs/en/configuration/environment-variables.md b/docs/en/configuration/environment-variables.md index 2c26ee5..2d5738f 100644 --- a/docs/en/configuration/environment-variables.md +++ b/docs/en/configuration/environment-variables.md @@ -94,8 +94,6 @@ Judge CPU/memory limits are currently set through CLI flags (`--judge-cpu-limit` | `SFORGE_K8S_NODE_SELECTOR` | --- | Node selector for K8s pods, format: `"key1=val1,key2=val2"` | | `SFORGE_E2B_TEMPLATE_MAP` | --- | Explicit E2B image-key to template mapping | | `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL in seconds | -| `SFORGE_PORTABLE_REGISTRY_USERNAME` | --- | Registry username for portable E2B derivative images | -| `SFORGE_PORTABLE_REGISTRY_PASSWORD` | --- | Registry password for portable E2B derivative images | ## Path Variables @@ -104,8 +102,6 @@ Judge CPU/memory limits are currently set through CLI flags (`--judge-cpu-limit` | `SFORGE_LOG_DIR` | `logs/` | Override log output directory | | `SFORGE_TASKS_DIR` | `tasks/` | Override task definitions directory | | `SFORGE_REGISTRY` | --- | Default Docker registry URL for `pull`/`push` commands | -| `SFORGE_REGISTRY_USERNAME` | --- | Source-registry username for E2B Template builds | -| `SFORGE_REGISTRY_PASSWORD` | --- | Source-registry password for E2B Template builds | ## Proxy Fallback Chain diff --git a/docs/zh/configuration/container-backends.md b/docs/zh/configuration/container-backends.md index 023dc17..85247da 100644 --- a/docs/zh/configuration/container-backends.md +++ b/docs/zh/configuration/container-backends.md @@ -228,70 +228,20 @@ SForge 还会创建临时的 managed Judge Controller，用户不需要单独部 ### Docker 镜像与 E2B Template -[`seededge`](https://hub.docker.com/u/seededge) 已经发布 EdgeBench 的预构建镜像。这些镜像 -可以直接作为 E2B Template 的源镜像；对于没有修改过的公开任务，不需要执行 -`sforge build`、`sforge pull` 或 `sforge push`。 +Docker 镜像和 E2B Template 是两种不同的运行对象：E2B Sandbox 从 Template 启动， +不能直接传入 Docker image 引用。已发布 EdgeBench 任务的官方 E2B Template 由 +SForge 维护者预先构建并发布，用户不需要执行任何 Template 准备步骤。 +`sforge run --backend e2b` 会自动推导每个任务镜像对应的 Template 引用 +（`edgebench.work.foo_bar:abc123` -> `edgebench-work-foo-bar:abc123`）。只有需要 +覆盖该映射时（例如为修改过的任务运行自建 Template）才设置 +`SFORGE_E2B_TEMPLATE_MAP`。 -Docker 镜像和 E2B Template 是两种不同的运行对象。`Template.from_image()` 将 Docker -镜像导入 E2B Template；`Sandbox.create()` 使用生成的 Template 名称或 ID 启动 Sandbox， -不能直接传入 Docker image 引用。SForge 会自动完成转换： - -```text -seededge Docker 镜像 - -> sforge e2b-template - -> 用户 E2B team 中的 Template - -> sforge run --backend e2b - -> E2B Sandbox -``` - -每个任务在一个 E2B team 中首次使用或镜像发生变化时需要准备 Template。输入一致时会 -复用已有 Template；日常 `sforge run` 不会重新构建。 - -### 准备 Template +### 运行任务 ```bash uv sync --extra e2b export E2B_API_KEY=... -sforge fetch-tasks edgebench -sforge e2b-template \ - --task ad_placement_optimization \ - --source-registry seededge -``` - -准备所有已下载任务时，将 `--task` 换成 `--all`。SForge 会自动推导两个镜像引用、固定 -不可变 digest、恢复非敏感 OCI 环境变量、快照镜像初始的 `/tmp`、构建 Work/Judge Template、 -等待明确终态、重试暂态错误、记录版本化 manifest，并复用匹配的已有构建。用户不需要 -编写 E2B Dockerfile，也不需要手工上传镜像层。 - -当前实现要求在 Linux amd64 主机上运行，并且本机 Docker daemon 可用，但不会重新构建 -已发布镜像。官方任务镜像早于 E2B，会在 `/tmp` 下预置评测辅助文件：命令直接引用的包装 -脚本，以及包装脚本运行时读取的其他文件。Docker 与 Kubernetes 直接从镜像启动，这些文件 -天然存在；而 E2B 每个 Sandbox 都会重置 `/tmp`。因此当某个 role 的命令引用 `/tmp` 时， -SForge 会快照整个镜像 `/tmp`（连同仅被间接引用的辅助文件一起保存），存入 Template，并在 -Sandbox 启动时恢复，使评测器看到与其他后端一致的文件。此过程不修改官方镜像、评测器或 -评分逻辑。源 Registry 还需要能被 E2B 构建服务访问。 - -### 可移植镜像回退 - -少数版本的镜像包含 E2B 构建网络无法访问的 APT 源。这时需要提供一个可写 Registry： - -```bash -export SFORGE_PORTABLE_REGISTRY_USERNAME=... -export SFORGE_PORTABLE_REGISTRY_PASSWORD=... - -sforge e2b-template \ - --task TASK \ - --source-registry seededge \ - --portable-registry registry.example.com/project -``` - -只有 E2B 返回 APT provisioning 失败时，SForge 才会创建版本化 portable derivative。 -该回退需要 Registry 写权限，但不会在每次评测时重复构建。 - -### 运行任务 - -```bash export SFORGE_AGENT_API_KEY=... export SFORGE_AGENT_API_BASE_URL=... export SFORGE_AGENT_MODEL=... @@ -345,13 +295,8 @@ auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快 | 输入 | 需要场景 | | --- | --- | -| `E2B_API_KEY` | Template 准备和 E2B 运行 | +| `E2B_API_KEY` | E2B 运行 | | Agent API key、base URL 和模型 | 运行 Agent | -| Linux amd64 主机和本机 Docker daemon | 使用当前实现准备 Template | -| `--source-registry seededge` | 使用已发布的 EdgeBench 镜像 | -| 源 Registry 凭证 | 仅使用私有源 Registry 时 | -| `--portable-registry` 及其凭证 | 仅需要 APT 可移植性回退时 | -| CPU/内存覆盖参数 | 仅 task 默认值不合适时；E2B 资源固化在 Template 中 | 宿主机的 `HTTP_PROXY` 和 `HTTPS_PROXY` 不会复制到 E2B Work、Judge 或 Game Sandbox。 只有用户显式配置 `SFORGE_HTTP_PROXY` 或 `SFORGE_HTTPS_PROXY` 时，远端可访问的代理才会 @@ -361,15 +306,10 @@ auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快 | 现象 | 可能原因 | 处理方式 | | --- | --- | --- | -| 找不到 Template | 当前 E2B team 尚未转换该镜像，或输入已经变化 | 对该 task 执行 `sforge e2b-template` | -| E2B 无法拉取 `seededge` 镜像 | Registry 网络故障或精确 tag 不存在 | 核对 task 定义后重试 | -| Template provisioning 无法访问 APT 源 | 源镜像包含不可达 mirror | 提供 `--portable-registry` 和凭证 | -| Template 构建返回 internal error，或超过客户端等待时间后仍为 `building` | E2B 构建服务暂态故障；SDK 没有单次构建取消 API | 重试前先检查目标 tag 是否已经可启动，再只重试失败的 role；避免同时触发大量冷构建 | -| `/tmp` 下的 evaluator 缺失 | Template 未通过 SForge 构建，缺少镜像 `/tmp` 快照 | 用 `sforge e2b-template` 重建；SForge 会快照整个镜像 `/tmp` 并在 Sandbox 启动时恢复 | +| 找不到 Template | 该任务镜像版本没有已发布的官方 Template，或任务使用了修改过的镜像 | 确认任务使用已发布的镜像版本，或用 `SFORGE_E2B_TEMPLATE_MAP` 指向自建 Template | | secured Judge 返回 403 | 缺少 traffic access token | 使用 managed 模式，或为外部 secured Judge 配置 `SFORGE_JUDGE_ACCESS_TOKEN` | | Agent 侧 Judge 请求偶发超时 | Work Sandbox 无法通过 E2B 公网 gateway 建立连接 | 让 Agent 重试提交、降低同时提交的突发并发，或使用独立 Judge Server；host 侧 managed-controller 请求已通过 E2B command channel 走 loopback | | task timeout 到达时出现 `Sandbox not found` | 在最终归档和清理前达到了 team 级 Sandbox 最大生命周期 | 缩短 Agent timeout、减少多波次批次总时长，或使用支持更长 Sandbox 生命周期的套餐 | -| 准备 Template 时 CPU 或内存被拒绝 | 请求规格超过当前 E2B team 上限 | 使用支持的规格重建；若榜单配置要求更高资源，则两者不具备严格可比性 | | 运行结束后仍有 Sandbox | cleanup 失败或 Runner 被强制终止 | 检查 run 日志；平台最终按配置的 E2B TTL 回收 | ## 常见问题 diff --git a/docs/zh/configuration/environment-variables.md b/docs/zh/configuration/environment-variables.md index 8970c90..9649dab 100644 --- a/docs/zh/configuration/environment-variables.md +++ b/docs/zh/configuration/environment-variables.md @@ -93,8 +93,6 @@ Judge CPU/内存限制目前通过 CLI 参数（`--judge-cpu-limit`、`--judge-m | `SFORGE_K8S_NODE_SELECTOR` | --- | K8s Pod 的节点选择器，格式：`"key1=val1,key2=val2"` | | `SFORGE_E2B_TEMPLATE_MAP` | --- | E2B image key 到 Template 的显式映射 | | `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL（秒） | -| `SFORGE_PORTABLE_REGISTRY_USERNAME` | --- | E2B 可移植派生镜像 Registry 用户名 | -| `SFORGE_PORTABLE_REGISTRY_PASSWORD` | --- | E2B 可移植派生镜像 Registry 密码 | ## 路径变量 @@ -103,8 +101,6 @@ Judge CPU/内存限制目前通过 CLI 参数（`--judge-cpu-limit`、`--judge-m | `SFORGE_LOG_DIR` | `logs/` | 覆盖日志输出目录 | | `SFORGE_TASKS_DIR` | `tasks/` | 覆盖任务定义目录 | | `SFORGE_REGISTRY` | --- | Docker Registry URL，用于 `pull`/`push` 命令 | -| `SFORGE_REGISTRY_USERNAME` | --- | E2B Template 源镜像 Registry 用户名 | -| `SFORGE_REGISTRY_PASSWORD` | --- | E2B Template 源镜像 Registry 密码 | ## 代理回退链 diff --git a/sforge/cli.py b/sforge/cli.py index ba0ec1c..15613bb 100644 --- a/sforge/cli.py +++ b/sforge/cli.py @@ -108,99 +108,6 @@ def _run_max_workers( return task_count -def _e2b_registry_credentials(portable: bool) -> tuple[str | None, str | None]: - prefix = "SFORGE_PORTABLE_REGISTRY" if portable else "SFORGE_REGISTRY" - return os.environ.get(f"{prefix}_USERNAME"), os.environ.get( - f"{prefix}_PASSWORD" - ) - - -def _build_e2b_task_template( - task_spec: TaskSpec, - role: str, - source_registry: str, - docker_client, - args, -) -> tuple[object, dict]: - """Build one task role, retrying through a portable source if needed.""" - import tempfile - - from e2b import default_build_logger - - from sforge.harness.e2b_templates import ( - AptSourceError, - build_e2b_template, - make_template_manifest, - stage_runtime_artifacts, - ) - - cpu = args.work_cpu if role == "work" else args.judge_cpu - memory = args.work_memory_mb if role == "work" else args.judge_memory_mb - force_portable = False - - for stage_attempt in range(2): - manifest = make_template_manifest( - task_spec, - role, - source_registry, - cpu_count=cpu, - memory_mb=memory, - ) - with tempfile.TemporaryDirectory( - prefix=f"sforge-e2b-{task_spec.task_id}-{role}-" - ) as context_dir: - context_path = Path(context_dir) - source_username, source_password = _e2b_registry_credentials(False) - portable_username, portable_password = _e2b_registry_credentials(True) - manifest, runtime_sources, portable = stage_runtime_artifacts( - manifest, - docker_client, - context_path, - pull_attempts=args.attempts, - portable_registry=args.portable_registry, - source_registry_username=source_username, - source_registry_password=source_password, - portable_registry_username=portable_username, - portable_registry_password=portable_password, - force_portable=force_portable, - ) - print( - f"[{task_spec.task_id}/{role}] {manifest.source_image} " - f"-> {manifest.reference}" - ) - username, password = ( - (portable_username, portable_password) - if portable - else (source_username, source_password) - ) - try: - result = build_e2b_template( - manifest, - attempts=args.attempts, - build_timeout=args.build_timeout, - force=args.force, - on_build_logs=( - None if args.silent else default_build_logger() - ), - registry_username=username, - registry_password=password, - context_dir=context_path, - runtime_sources=runtime_sources, - ) - except AptSourceError: - if stage_attempt or not args.portable_registry or portable: - raise - print( - f"[{task_spec.task_id}/{role}] retrying through a portable " - "source image after E2B could not reach its APT source" - ) - force_portable = True - continue - return manifest, result - - raise RuntimeError("E2B Template build exhausted portable-image retries") - - # --- Commands --- @@ -249,57 +156,6 @@ def _build_one(ts: TaskSpec): print("Done.") -def cmd_e2b_template(args): - """Build deterministic official E2B Templates from task images.""" - config = _make_config(args) - task_specs = _resolve_tasks(args, config) - source_registry = args.source_registry or config.registry - if not source_registry: - print( - "Error: --source-registry or SFORGE_REGISTRY is required; E2B " - "must be able to pull immutable Work/Judge images", - file=sys.stderr, - ) - sys.exit(1) - - from sforge.harness.e2b_templates import write_template_manifest - - docker_client = docker.from_env() - roles = [args.role] if args.role != "both" else ["work", "judge"] - failures = [] - for task_spec in task_specs: - task_failed = False - for role in roles: - if task_failed: - print( - f"[{task_spec.task_id}/{role}] SKIPPED: an earlier role " - "failed for this task", - file=sys.stderr, - ) - continue - try: - manifest, result = _build_e2b_task_template( - task_spec, role, source_registry, docker_client, args, - ) - path = write_template_manifest( - manifest, result, - config.log_dir / "e2b_templates" / task_spec.task_id, - ) - print( - f" {result['status']}: build={result.get('build_id')} " - f"manifest={path}" - ) - except Exception as exc: - failures.append((task_spec.task_id, role, str(exc))) - task_failed = True - print( - f"[{task_spec.task_id}/{role}] FAILED: {exc}", - file=sys.stderr, - ) - if failures: - raise SystemExit(1) - - def cmd_pull(args): """Pull pre-built images from a remote registry (parallel).""" config = _make_config(args) @@ -1137,40 +993,6 @@ def main(): help="Force rebuild ALL images including base") p_build.set_defaults(func=cmd_build) - # e2b-template - p_e2b_template = subparsers.add_parser( - "e2b-template", - help="Build official E2B Templates from immutable task images", - ) - p_e2b_group = p_e2b_template.add_mutually_exclusive_group(required=True) - p_e2b_group.add_argument("--task", nargs="+") - p_e2b_group.add_argument("--all", action="store_true", default=False) - p_e2b_template.add_argument( - "--source-registry", default=None, - help="Registry prefix containing the Work/Judge images", - ) - p_e2b_template.add_argument( - "--portable-registry", default=None, - help=( - "Registry prefix for sanitized derivatives of images that contain " - "APT sources unreachable from the official E2B build network" - ), - ) - p_e2b_template.add_argument( - "--role", choices=["work", "judge", "both"], default="both", - ) - p_e2b_template.add_argument("--work-cpu", type=_positive_int, default=None) - p_e2b_template.add_argument("--work-memory-mb", type=_positive_int, default=None) - p_e2b_template.add_argument("--judge-cpu", type=_positive_int, default=None) - p_e2b_template.add_argument("--judge-memory-mb", type=_positive_int, default=None) - p_e2b_template.add_argument("--attempts", type=_positive_int, default=3) - p_e2b_template.add_argument( - "--build-timeout", type=_positive_int, default=600, - help="Maximum seconds to wait for each E2B Template build", - ) - p_e2b_template.add_argument("--force", action="store_true", default=False) - p_e2b_template.set_defaults(func=cmd_e2b_template) - # pull p_pull = subparsers.add_parser("pull", help="Pull pre-built images from remote registry") p_pull_group = p_pull.add_mutually_exclusive_group(required=True) diff --git a/sforge/harness/e2b_templates.py b/sforge/harness/e2b_templates.py deleted file mode 100644 index 5a548ed..0000000 --- a/sforge/harness/e2b_templates.py +++ /dev/null @@ -1,1248 +0,0 @@ -# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Deterministic E2B Template supply chain for SForge task images.""" - -from __future__ import annotations - -import hashlib -import io -import json -import re -import shlex -import stat -import tarfile -import time -from dataclasses import asdict, dataclass, replace -from pathlib import Path - -import docker.errors -import requests - -from sforge.harness.task_spec import TaskSpec - -# schema_version stays 4: the manifest gains a snapshot_roots field, but that -# field is excluded from the fingerprint, and schema_version itself is part of -# every existing fingerprint. Keeping it at 4 lets clean tasks and tasks whose -# image /tmp is empty reuse their current E2B builds; only tasks that actually -# ship files under /tmp get a new fingerprint and rebuild. -MANIFEST_SCHEMA_VERSION = 4 -PORTABLE_IMAGE_RECIPE_VERSION = 2 -WORK_FILESYSTEM_RECIPE_VERSION = 2 -DEFAULT_WORK_MEMORY_MB = 4096 -DEFAULT_JUDGE_MEMORY_MB = 8192 - -# The official task images predate E2B and ship evaluator helpers under /tmp: a -# wrapper the command names plus other files that wrapper reads at runtime. -# Docker and Kubernetes boot from the image, so those files are simply present; -# E2B resets /tmp on every Sandbox, so any helper the command does not name on -# its own line silently disappears. SForge therefore snapshots the image's -# initial /tmp and restores it on Sandbox start, restoring parity with the other -# backends. This is a backend-compatibility shim: it never rewrites the official -# images, the evaluators, or any scoring behavior. -_TMP_SNAPSHOT_ROOT = "/tmp" -_TMP_SNAPSHOT_PERSISTENT = "/opt/sforge/runtime/tmp" -TMP_SNAPSHOT_RECIPE_VERSION = 1 -# A /tmp path named anywhere in the role's command decides *whether* to attempt -# a snapshot; the snapshot itself always captures the whole image /tmp, so -# helpers the command reaches only indirectly come along too. Capturing all of -# /tmp (rather than the named paths) is what makes this robust, and keying the -# decision on a /tmp reference preserves the no-pull fast path for the many -# tasks — including multi-gigabyte ones — that never touch /tmp. -_TMP_PATH = re.compile(r"(? bool: - if isinstance(exc, (docker.errors.ImageNotFound, docker.errors.NotFound)): - return True - # Some Registry implementations surface a missing manifest through the - # Docker Engine distribution endpoint as HTTP 500 instead of 404. Only - # accept its explicit not-found wording; other server errors remain fatal. - message = str(exc).lower() - return "artifact " in message and " not found" in message - - -def _manifest_fingerprint(data: dict, role: str) -> str: - fingerprint_data = dict(data) - if role == "work": - fingerprint_data["filesystem_recipe_version"] = WORK_FILESYSTEM_RECIPE_VERSION - # snapshot_roots records the build-time intent to snapshot /tmp; it is not - # part of the built template's content, so it never enters the fingerprint. - # Only a non-empty captured payload (runtime_artifacts) changes what the - # template ships, so the snapshot recipe version is folded in only then. - # This keeps clean tasks byte-identical to the pre-snapshot fingerprint and - # avoids rebuilding every template. - fingerprint_data.pop("snapshot_roots", None) - if fingerprint_data.get("runtime_artifacts"): - fingerprint_data["tmp_snapshot_recipe_version"] = TMP_SNAPSHOT_RECIPE_VERSION - return hashlib.sha256( - json.dumps(fingerprint_data, sort_keys=True).encode() - ).hexdigest() - - -def template_name(image_key: str) -> str: - repo, _, _ = image_key.partition(":") - return repo.lower().replace(".", "-").replace("_", "-") - - -def template_tag(image_key: str) -> str: - _, sep, tag = image_key.rpartition(":") - if not sep or not tag: - raise ValueError(f"Image key has no immutable tag: {image_key!r}") - return tag - - -def parse_memory_mb(value: str | None, default: int = 4096) -> int: - if value is None: - return default - raw = str(value).strip().lower() - match = re.fullmatch(r"(\d+)([kmgt]i?b?|)", raw) - if not match: - raise ValueError(f"Unsupported memory value: {value!r}") - amount = int(match.group(1)) - if amount <= 0: - raise ValueError("Memory must be greater than zero") - unit = match.group(2) - if unit in ("", "m", "mb", "mib"): - return amount - if unit in ("k", "kb", "kib"): - return max(1, amount // 1024) - if unit in ("g", "gb", "gib"): - return amount * 1024 - if unit in ("t", "tb", "tib"): - return amount * 1024 * 1024 - raise ValueError(f"Unsupported memory unit: {unit!r}") - - -@dataclass(frozen=True) -class RuntimeArtifact: - source: str - persistent: str - digest: str = "" - - -@dataclass(frozen=True) -class TemplateManifest: - schema_version: int - task_id: str - role: str - source_image: str - source_image_digest: str - image_key: str - name: str - tag: str - user: str - workdir: str - cpu_count: int - memory_mb: int - environment: tuple[tuple[str, str], ...] - runtime_artifacts: tuple[RuntimeArtifact, ...] - snapshot_roots: tuple[str, ...] - fingerprint: str - - @property - def reference(self) -> str: - return f"{self.name}:{self.tag}" - - def to_dict(self) -> dict: - return asdict(self) - - -def _command_references_tmp(task_spec: TaskSpec, role: str) -> bool: - """Whether the role's command touches /tmp, so its image /tmp is snapshotted. - - This only decides *whether* to snapshot. The snapshot itself always captures - the entire image /tmp, so evaluator helpers the command reaches only - indirectly (a wrapper that reads a sibling file) are preserved too. Keying - the decision on a /tmp reference keeps the no-pull fast path for the many - tasks — including multi-gigabyte ones — whose commands never touch /tmp. - """ - if role == "work": - command = task_spec.work.agent_query or "" - else: - command = " ".join( - filter( - None, - ( - task_spec.judge.eval_cmd, - task_spec.judge.game_server_cmd, - ), - ) - ) - return bool(_TMP_PATH.search(command)) - - -def make_template_manifest( - task_spec: TaskSpec, - role: str, - source_registry: str, - *, - cpu_count: int | None = None, - memory_mb: int | None = None, -) -> TemplateManifest: - if role not in ("work", "judge"): - raise ValueError("Template role must be 'work' or 'judge'") - image_key = ( - task_spec.work_image_key if role == "work" else task_spec.judge_image_key - ) - spec = task_spec.work if role == "work" else task_spec.judge - snapshot_roots = ( - (_TMP_SNAPSHOT_ROOT,) - if _command_references_tmp(task_spec, role) - else () - ) - resolved_cpu = cpu_count if cpu_count is not None else spec.cpu_limit or 4 - resolved_memory = ( - memory_mb - if memory_mb is not None - else parse_memory_mb( - spec.mem_limit, - default=( - DEFAULT_WORK_MEMORY_MB - if role == "work" - else DEFAULT_JUDGE_MEMORY_MB - ), - ) - ) - if resolved_cpu <= 0: - raise ValueError("Template CPU count must be greater than zero") - if resolved_memory <= 0: - raise ValueError("Template memory must be greater than zero") - core = { - "schema_version": MANIFEST_SCHEMA_VERSION, - "task_id": task_spec.task_id, - "role": role, - "source_image": f"{source_registry.rstrip('/')}/{image_key}", - "source_image_digest": "", - "image_key": image_key, - "name": template_name(image_key), - "tag": template_tag(image_key), - "user": "agent" if role == "work" else "root", - "workdir": task_spec.cwd, - "cpu_count": resolved_cpu, - "memory_mb": resolved_memory, - "environment": [], - "runtime_artifacts": [], - "snapshot_roots": list(snapshot_roots), - } - fingerprint = _manifest_fingerprint(core, role) - return TemplateManifest( - **{ - **core, - "runtime_artifacts": (), - "snapshot_roots": snapshot_roots, - }, - fingerprint=fingerprint, - ) - - -def _restore_command(manifest: TemplateManifest) -> str | None: - if not manifest.runtime_artifacts: - return None - commands = [] - for item in manifest.runtime_artifacts: - parent = str(Path(item.source).parent) - # Merge each captured entry back into place without touching the - # directory root: /tmp keeps its E2B-provisioned runtime files and its - # sticky bit. `cp -aT` treats the destination as the entry itself, so a - # directory that already exists is overwritten in place rather than - # nested under itself, making restore idempotent across restarts. The - # && chain means any failed copy aborts start-up, so a broken restore - # surfaces as a Sandbox that never becomes ready instead of an evaluator - # silently missing a helper. - commands.append( - f"mkdir -p {shlex.quote(parent)} && " - f"cp -aT {shlex.quote(item.persistent)} {shlex.quote(item.source)}" - ) - return " && ".join(commands) + " && sleep infinity" - - -def _replace_runtime_artifacts( - manifest: TemplateManifest, - artifacts: tuple[RuntimeArtifact, ...], -) -> TemplateManifest: - provisional = replace(manifest, runtime_artifacts=artifacts, fingerprint="") - data = asdict(provisional) - data.pop("fingerprint") - fingerprint = _manifest_fingerprint(data, manifest.role) - return replace(provisional, fingerprint=fingerprint) - - -def _replace_image_metadata( - manifest: TemplateManifest, - source_image: str, - environment: tuple[tuple[str, str], ...], -) -> TemplateManifest: - provisional = replace( - manifest, - source_image=source_image, - source_image_digest=source_image.split("@", 1)[1], - environment=environment, - fingerprint="", - ) - data = asdict(provisional) - data.pop("fingerprint") - return replace( - provisional, fingerprint=_manifest_fingerprint(data, manifest.role), - ) - - -def _image_environment(image) -> tuple[tuple[str, str], ...]: - values = {} - for entry in (image.attrs.get("Config") or {}).get("Env") or []: - key, separator, value = entry.partition("=") - normalized = key.lower() - sensitive = any( - marker in normalized - for marker in ( - "api_key", "access_key", "auth", "credential", - "password", "passwd", "private_key", "secret", "token", - ) - ) - host_specific = normalized in { - "http_proxy", "https_proxy", "no_proxy", - } - if ( - separator - and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key) - and not sensitive - and not host_specific - ): - values[key] = value - return tuple(sorted(values.items())) - - -_MANIFEST_ACCEPT = ", ".join(( - "application/vnd.oci.image.index.v1+json", - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.docker.distribution.manifest.v2+json", -)) - - -def _registry_location(image_ref: str) -> tuple[str, str, str]: - name, separator, identifier = image_ref.partition("@") - if not separator: - name, separator, identifier = image_ref.rpartition(":") - if not separator or "/" in identifier: - name, identifier = image_ref, "latest" - parts = name.split("/") - if len(parts) > 1 and ("." in parts[0] or ":" in parts[0] or parts[0] == "localhost"): - registry, repository = parts[0], "/".join(parts[1:]) - else: - registry = "registry-1.docker.io" - repository = name if "/" in name else f"library/{name}" - return registry, repository, identifier - - -def _registry_image_environment( - image_ref: str, - *, - attempts: int = 3, - username: str | None = None, - password: str | None = None, -) -> tuple[tuple[str, str], ...]: - """Read OCI Config.Env without downloading image layers.""" - if bool(username) != bool(password): - raise ValueError("Both registry username and password are required") - registry, repository, identifier = _registry_location(image_ref) - base_url = f"https://{registry}/v2/{repository}" - headers = {"Accept": _MANIFEST_ACCEPT} - auth = (username, password) if username and password else None - - def request(path: str): - response = requests.get( - base_url + path, headers=headers, auth=auth, timeout=30, - ) - if response.status_code != 401: - response.raise_for_status() - return response - scheme, _, parameters = response.headers.get("WWW-Authenticate", "").partition(" ") - if scheme.lower() != "bearer": - response.raise_for_status() - challenge = requests.utils.parse_dict_header(parameters) - token_response = requests.get( - challenge["realm"], - params={ - key: challenge[key] - for key in ("service", "scope") - if key in challenge - }, - auth=auth, timeout=30, - ) - token_response.raise_for_status() - token_data = token_response.json() - token = token_data.get("token") or token_data.get("access_token") - if not token: - raise RuntimeError(f"Registry token response for {image_ref!r} had no token") - headers["Authorization"] = f"Bearer {token}" - response = requests.get(base_url + path, headers=headers, timeout=30) - response.raise_for_status() - return response - - error = None - for attempt in range(1, max(1, attempts) + 1): - try: - manifest = request(f"/manifests/{identifier}").json() - if "manifests" in manifest: - descriptor = next( - ( - item for item in manifest["manifests"] - if (item.get("platform") or {}).get("os") == "linux" - and (item.get("platform") or {}).get("architecture") == "amd64" - ), - None, - ) - if descriptor is None: - raise RuntimeError( - f"Image {image_ref!r} has no linux/amd64 manifest" - ) - manifest = request( - f"/manifests/{descriptor['digest']}" - ).json() - config = manifest.get("config") or {} - digest = config.get("digest") - if not isinstance(digest, str) or not digest.startswith("sha256:"): - raise RuntimeError( - f"Image {image_ref!r} has no valid OCI config digest" - ) - raw = request(f"/blobs/{digest}").content - if hashlib.sha256(raw).hexdigest() != digest.removeprefix("sha256:"): - raise RuntimeError(f"OCI config digest mismatch for {image_ref!r}") - config_data = json.loads(raw) - image = type("RegistryImage", (), { - "attrs": {"Config": config_data.get("config") or {}}, - })() - return _image_environment(image) - except Exception as exc: - error = exc - if attempt < max(1, attempts): - time.sleep(min(2 ** (attempt - 1), 8)) - raise RuntimeError( - f"Failed to read OCI config for {image_ref!r} after {attempts} " - f"attempts: {error}" - ) from error - - -def _environment_profile_command( - environment: tuple[tuple[str, str], ...], -) -> str | None: - if not environment: - return None - content = "".join( - f"export {key}={shlex.quote(value)}\n" for key, value in environment - ) - return ( - "mkdir -p /etc/profile.d && printf %s " - f"{shlex.quote(content)} > /etc/profile.d/sforge-image-env.sh && " - "chmod 0644 /etc/profile.d/sforge-image-env.sh" - ) - - -def _runtime_artifact_digest(path: Path) -> str: - """Hash content plus filesystem semantics preserved by cp -a.""" - digest = hashlib.sha256() - members = [path] - if path.is_dir(): - members.extend(sorted(path.rglob("*"))) - for member in members: - relative = "." if member == path else str(member.relative_to(path)) - kind = "dir" if member.is_dir() else "file" - mode = stat.S_IMODE(member.stat().st_mode) - digest.update(f"{relative}\0{kind}\0{mode:o}\0".encode()) - if member.is_file(): - digest.update(member.read_bytes()) - digest.update(b"\0") - return digest.hexdigest() - - -def _capture_tmp_snapshot( - container, - context_dir: Path, -) -> tuple[tuple[RuntimeArtifact, ...], dict[str, str]]: - """Snapshot the image's initial /tmp into staged per-entry artifacts. - - The whole directory is captured (not just command-named paths) so evaluator - helpers reached indirectly survive. Each top-level entry becomes its own - artifact so the Sandbox restore merges them back into /tmp without recreating - or clobbering the directory root. Anything unusual — a path escape, a symlink - or special file, or a payload past the ceilings — fails the build loudly - rather than being silently dropped. - """ - try: - stream, _ = container.get_archive(_TMP_SNAPSHOT_ROOT) - archive = b"".join(stream) - except docker.errors.NotFound: - return (), {} - - staging = context_dir / "tmp-snapshot" - staging.mkdir(parents=True, exist_ok=True) - staging_root = staging.resolve() - with tarfile.open(fileobj=io.BytesIO(archive)) as bundle: - members = bundle.getmembers() - if not members: - return (), {} - prefix = members[0].name.split("/", 1)[0] - entry_count = 0 - total_bytes = 0 - keep = [] - for member in members: - relative = member.name[len(prefix):].lstrip("/") - if not relative: - # The /tmp directory itself: E2B provides it, so never recreate - # or re-permission the root; only its contents are restored. - continue - top = relative.split("/", 1)[0] - if top in _SNAPSHOT_EXCLUDE_NAMES: - continue - target = (staging / member.name).resolve() - if staging_root not in target.parents and target != staging_root: - raise ValueError( - f"Unsafe /tmp snapshot path: {member.name!r}" - ) - if member.issym() or member.islnk(): - raise ValueError( - "/tmp snapshot symlinks are not supported: " - f"{member.name!r}" - ) - if not (member.isfile() or member.isdir()): - raise ValueError( - "/tmp snapshot special files are not supported: " - f"{member.name!r}" - ) - entry_count += 1 - if member.isfile(): - total_bytes += member.size - if entry_count > MAX_SNAPSHOT_ENTRIES: - raise ValueError( - f"/tmp snapshot exceeds {MAX_SNAPSHOT_ENTRIES} entries" - ) - if total_bytes > MAX_SNAPSHOT_BYTES: - raise ValueError( - f"/tmp snapshot exceeds {MAX_SNAPSHOT_BYTES} bytes" - ) - keep.append(member) - if not keep: - return (), {} - # Preserve the image's exact modes so the restored /tmp matches what - # Docker and Kubernetes see. The loop above already rejected symlinks, - # hardlinks, special files, and path escapes, so "fully_trusted" adds no - # new exposure here; it also extracts without the Python 3.14 - # extraction-filter deprecation. Extraction runs as the unprivileged - # build user, so setuid/setgid bits cannot escalate on the host. - bundle.extractall(staging, members=keep, filter="fully_trusted") - - source_root = staging / prefix - artifacts = [] - sources = {} - for entry in sorted(source_root.iterdir()): - name = entry.name - source = f"{_TMP_SNAPSHOT_ROOT}/{name}" - artifacts.append( - RuntimeArtifact( - source=source, - persistent=f"{_TMP_SNAPSHOT_PERSISTENT}/{name}", - digest=_runtime_artifact_digest(entry), - ) - ) - sources[source] = str(entry.relative_to(context_dir)) - return tuple(artifacts), sources - - -def stage_runtime_artifacts( - manifest: TemplateManifest, - docker_client, - context_dir: Path, - *, - pull_attempts: int = 3, - portable_registry: str | None = None, - source_registry_username: str | None = None, - source_registry_password: str | None = None, - portable_registry_username: str | None = None, - portable_registry_password: str | None = None, - force_portable: bool = False, -) -> tuple[TemplateManifest, dict[str, str], bool]: - """Resolve the immutable source image and snapshot its initial /tmp. - - Tasks whose command never touches /tmp keep the no-pull fast path: only the - registry digest and OCI environment are resolved, with no image download. - """ - if not manifest.snapshot_roots and not force_portable: - try: - image = docker_client.images.get(manifest.source_image) - except Exception as exc: - if not _is_missing_image_error(exc): - raise - immutable_image = _resolve_registry_digest( - docker_client, - manifest.source_image, - attempts=pull_attempts, - username=source_registry_username, - password=source_registry_password, - ) - environment = _registry_image_environment( - immutable_image, attempts=pull_attempts, - username=source_registry_username, - password=source_registry_password, - ) - resolved = _replace_image_metadata( - manifest, immutable_image, environment, - ) - return _replace_runtime_artifacts(resolved, ()), {}, False - - image = _get_or_pull_image( - docker_client, - manifest.source_image, - attempts=pull_attempts, - username=source_registry_username, - password=source_registry_password, - ) - immutable_source = _matching_repo_digest(image, manifest.source_image) - used_portable_image = force_portable - if used_portable_image: - if not portable_registry: - raise AptSourceError( - f"Source image {manifest.source_image!r} requires APT source " - "normalization. Pass --portable-registry to publish a portable " - "derivative before building the E2B Template." - ) - portable_ref = _portable_image_ref( - portable_registry, - manifest.image_key, - immutable_source.split("@", 1)[1], - ) - immutable_image = _optional_registry_digest( - docker_client, - portable_ref, - attempts=pull_attempts, - username=portable_registry_username, - password=portable_registry_password, - ) - if immutable_image is None: - image, immutable_image = _build_and_push_portable_image( - docker_client, - image, - manifest.source_image, - portable_ref, - context_dir, - username=portable_registry_username, - password=portable_registry_password, - ) - else: - # The Registry manifest is authoritative. A tag may still exist in - # the local Docker cache after its remote manifest was deleted, so - # never derive an E2B source digest from the local tag alone. - image = _get_or_pull_image( - docker_client, - immutable_image, - attempts=pull_attempts, - username=portable_registry_username, - password=portable_registry_password, - ) - else: - immutable_image = immutable_source - manifest = _replace_image_metadata( - manifest, immutable_image, _image_environment(image), - ) - if not manifest.snapshot_roots: - return _replace_runtime_artifacts(manifest, ()), {}, used_portable_image - - container = docker_client.containers.create(manifest.source_image) - try: - artifacts, sources = _capture_tmp_snapshot(container, context_dir) - finally: - container.remove(force=True) - return ( - _replace_runtime_artifacts(manifest, artifacts), - sources, - used_portable_image, - ) - - -def _registry_auth( - username: str | None, - password: str | None, -) -> dict[str, dict[str, str]]: - if not username and not password: - return {} - if not username or not password: - raise ValueError("Both registry username and password are required") - return {"auth_config": {"username": username, "password": password}} - - -def _get_or_pull_image( - docker_client, - image_ref: str, - *, - attempts: int = 3, - username: str | None = None, - password: str | None = None, -): - """Resolve an image locally, retrying transient registry failures.""" - try: - return docker_client.images.get(image_ref) - except Exception as exc: - if not _is_missing_image_error(exc): - raise - - attempts = max(1, attempts) - auth = _registry_auth(username, password) - error = None - for attempt in range(1, attempts + 1): - try: - return docker_client.images.pull(image_ref, **auth) - except Exception as exc: - error = exc - # A failed pull can still leave a complete image in the daemon. - try: - return docker_client.images.get(image_ref) - except Exception as lookup_exc: - if not _is_missing_image_error(lookup_exc): - raise - if attempt < attempts: - time.sleep(min(2 ** (attempt - 1), 8)) - raise RuntimeError( - f"Failed to pull source image {image_ref!r} after {attempts} attempts: {error}" - ) from error - - -def _resolve_registry_digest( - docker_client, - image_ref: str, - *, - attempts: int = 3, - username: str | None = None, - password: str | None = None, -) -> str: - """Resolve an immutable registry digest without downloading layers.""" - attempts = max(1, attempts) - auth = _registry_auth(username, password) - error = None - for attempt in range(1, attempts + 1): - try: - data = docker_client.images.get_registry_data(image_ref, **auth) - digest = data.id - if not digest.startswith("sha256:"): - raise ValueError( - f"Registry returned an invalid digest for {image_ref!r}: {digest!r}" - ) - repository, separator, _ = image_ref.rpartition(":") - if not separator: - repository = image_ref - return f"{repository}@{digest}" - except Exception as exc: - error = exc - if attempt < attempts: - time.sleep(min(2 ** (attempt - 1), 8)) - raise RuntimeError( - f"Failed to resolve source image {image_ref!r} after {attempts} " - f"attempts: {error}" - ) from error - - -def _optional_registry_digest( - docker_client, - image_ref: str, - *, - attempts: int = 3, - username: str | None = None, - password: str | None = None, -) -> str | None: - """Resolve a remote image digest, returning None only for NotFound. - - A local Docker tag is not proof that the corresponding remote manifest - still exists. Authentication and transport failures remain hard errors so - they cannot silently trigger a rebuild or overwrite an inaccessible tag. - """ - attempts = max(1, attempts) - auth = _registry_auth(username, password) - for attempt in range(1, attempts + 1): - try: - data = docker_client.images.get_registry_data(image_ref, **auth) - digest = data.id - if not digest.startswith("sha256:"): - raise ValueError( - f"Registry returned an invalid digest for {image_ref!r}: " - f"{digest!r}" - ) - repository, separator, _ = image_ref.rpartition(":") - if not separator: - repository = image_ref - return f"{repository}@{digest}" - except Exception as exc: - if _is_missing_image_error(exc): - return None - if attempt == attempts: - raise RuntimeError( - f"Failed to inspect portable image {image_ref!r} after " - f"{attempts} attempts: {exc}" - ) from exc - time.sleep(min(2 ** (attempt - 1), 8)) - return None - - -def _portable_image_ref( - portable_registry: str, - image_key: str, - source_digest: str, -) -> str: - repo, sep, tag = image_key.rpartition(":") - if not sep: - raise ValueError(f"Image key has no immutable tag: {image_key!r}") - digest_suffix = source_digest.removeprefix("sha256:")[:12] - if not digest_suffix: - raise ValueError("Source image digest is required for a portable image") - return ( - f"{portable_registry.rstrip('/')}/{repo}:" - f"{tag}-{digest_suffix}-e2b-v{PORTABLE_IMAGE_RECIPE_VERSION}" - ) - - -def _matching_repo_digest(image, image_ref: str) -> str: - repo_digests = image.attrs.get("RepoDigests") or [] - source_repo, separator, _ = image_ref.rpartition(":") - if not separator: - source_repo = image_ref - matching = [ - item - for item in repo_digests - if "@" in item and item.split("@", 1)[0] == source_repo - ] - if not matching: - raise ValueError( - f"Source image {image_ref!r} has no matching registry digest " - f"(available: {repo_digests}); push it to that registry before " - "building an E2B Template" - ) - return matching[0] - - -def _portable_apt_sources_command() -> str: - """Return a distro-aware command that restores public APT sources.""" - debian_components = "main" - ubuntu_components = "main universe" - debian_sources = " ".join( - ( - f'"deb http://deb.debian.org/debian $codename {debian_components}"', - f'"deb http://deb.debian.org/debian $codename-updates {debian_components}"', - f'"deb http://security.debian.org/debian-security ' - f'$codename-security {debian_components}"', - ) - ) - ubuntu_sources = " ".join( - ( - f'"deb http://archive.ubuntu.com/ubuntu $codename {ubuntu_components}"', - f'"deb http://archive.ubuntu.com/ubuntu $codename-updates ' - f'{ubuntu_components}"', - f'"deb http://security.ubuntu.com/ubuntu $codename-security ' - f'{ubuntu_components}"', - ) - ) - return ( - "set -eu; . /etc/os-release; " - "codename=${VERSION_CODENAME:-${UBUNTU_CODENAME:-}}; " - 'test -n "$codename"; ' - "rm -f /etc/apt/sources.list; " - "rm -rf /etc/apt/sources.list.d; " - "mkdir -p /etc/apt/sources.list.d; " - 'case "$ID" in ' - f"debian) printf '%s\\n' {debian_sources} " - "> /etc/apt/sources.list ;; " - f"ubuntu) printf '%s\\n' {ubuntu_sources} " - "> /etc/apt/sources.list ;; " - '*) echo "Unsupported APT distribution: $ID" >&2; exit 1 ;; ' - "esac" - ) - - -def _build_and_push_portable_image( - docker_client, - source_image, - source_ref: str, - portable_ref: str, - context_dir: Path, - *, - username: str | None = None, - password: str | None = None, -) -> tuple[object, str]: - """Publish a source-equivalent image with public APT sources. - - E2B provisions its own packages before user Template instructions run, so - source-list repair must already be present in the imported image. - """ - immutable_source = _matching_repo_digest(source_image, source_ref) - config = source_image.attrs.get("Config") or {} - original_user = config.get("User") or "" - original_workdir = config.get("WorkingDir") or "/" - dockerfile = [ - f"FROM {immutable_source}", - "USER root", - "RUN " + _portable_apt_sources_command(), - ] - if original_user: - dockerfile.append(f"USER {original_user}") - dockerfile.append(f"WORKDIR {original_workdir}") - dockerfile_path = context_dir / "Portable.Dockerfile" - dockerfile_path.write_text("\n".join(dockerfile) + "\n") - - image, logs = docker_client.images.build( - path=str(context_dir), - dockerfile=dockerfile_path.name, - tag=portable_ref, - rm=True, - forcerm=True, - ) - for item in logs: - if item.get("error") or item.get("errorDetail"): - raise RuntimeError( - f"Failed to build portable source image {portable_ref!r}: " - f"{item.get('error') or item['errorDetail'].get('message')}" - ) - - repository, tag = portable_ref.rsplit(":", 1) - pushed_digest = None - push_kwargs = { - "tag": tag, - "stream": True, - "decode": True, - **_registry_auth(username, password), - } - for item in docker_client.images.push(repository, **push_kwargs): - if item.get("error") or item.get("errorDetail"): - raise RuntimeError( - f"Failed to push portable source image {portable_ref!r}: " - f"{item.get('error') or item['errorDetail'].get('message')}" - ) - pushed_digest = (item.get("aux") or {}).get("Digest") or pushed_digest - image.reload() - immutable_portable = ( - f"{repository}@{pushed_digest}" - if pushed_digest - else _matching_repo_digest(image, portable_ref) - ) - return image, immutable_portable - - -def build_template_definition( - manifest: TemplateManifest, - template_cls, - *, - registry_username: str | None = None, - registry_password: str | None = None, - context_dir: Path | None = None, - runtime_sources: dict[str, str] | None = None, -): - template = ( - template_cls(file_context_path=context_dir) if context_dir else template_cls() - ) - auth = {} - if registry_username or registry_password: - if not registry_username or not registry_password: - raise ValueError("Both registry username and password are required") - auth = { - "username": registry_username, - "password": registry_password, - } - builder = template.from_image(manifest.source_image, **auth) - environment = dict(manifest.environment) - if environment: - builder = builder.set_envs(environment) - builder = builder.run_cmd( - _environment_profile_command(manifest.environment), user="root", - ) - runtime_sources = runtime_sources or {} - for item in manifest.runtime_artifacts: - source = runtime_sources.get(item.source) - if source is None: - raise ValueError( - f"Runtime artifact {item.source!r} was not staged from the image" - ) - builder = builder.copy( - source, - item.persistent, - user="root", - force_upload=True, - ) - if manifest.runtime_artifacts: - # The copies above land root-owned, but the Sandbox start command runs - # as manifest.user (agent for work). A non-root user cannot `cp -a` a - # root-owned directory back into /tmp (preserving times/ownership is - # denied), so hand the staged tree to the run user. Modes are preserved, - # so the restored files keep the image's permissions; only ownership - # normalizes to the run identity, which is exactly who reads them. - builder = builder.run_cmd( - f"chown -R {shlex.quote(manifest.user)}:{shlex.quote(manifest.user)} " - f"{shlex.quote(_TMP_SNAPSHOT_PERSISTENT)}", - user="root", - ) - if manifest.role == "work": - # Match the Docker work-image contract. Imported images can preserve - # root-owned subdirectories even when their default user is `agent`. - builder = builder.run_cmd( - f"chown -R agent:agent {shlex.quote(manifest.workdir)} && " - f"chmod -R u+rwX {shlex.quote(manifest.workdir)}", - user="root", - ) - builder = builder.set_user(manifest.user).set_workdir(manifest.workdir) - restore = _restore_command(manifest) - if restore: - ready = " && ".join( - f"test -e {shlex.quote(item.source)}" for item in manifest.runtime_artifacts - ) - builder = builder.set_start_cmd(restore, ready) - return builder - - -def _matching_build_id( - template_cls, name: str, tag: str, fingerprint_tag: str -) -> str | None: - """Return the build shared by the image and manifest aliases.""" - try: - if not template_cls.exists(f"{name}:{tag}"): - return None - if not template_cls.exists(f"{name}:{fingerprint_tag}"): - return None - tags = template_cls.get_tags(name) - except Exception as exc: - raise RuntimeError( - f"Could not inspect E2B Template aliases for {name!r}: {exc}" - ) from exc - builds = {item.tag: item.build_id for item in tags} - image_build = builds.get(tag) - manifest_build = builds.get(fingerprint_tag) - if not image_build or image_build != manifest_build: - return None - return image_build - - -def _is_retryable_build_error(exc: Exception) -> bool: - text = " ".join( - str(item).lower() - for item in (exc, getattr(exc, "__cause__", None)) - if item is not None - ) - deterministic = ( - "provisioning output", - "could not resolve", - "no installation candidate", - ) - if any(marker in text for marker in deterministic): - return False - transient = ( - "timeout", - "timed out", - "connection", - "eof", - "tls", - "temporarily unavailable", - "internal error", - "502", - "503", - "504", - ) - return any(marker in text for marker in transient) - - -def _is_apt_source_error(message: str) -> bool: - """Recognize E2B provisioning failures caused by unreachable APT repos.""" - lowered = message.lower() - return "provisioning output" in lowered and any( - marker in lowered - for marker in ( - "apt", - "could not resolve", - "temporary failure resolving", - "no installation candidate", - ) - ) - - -def _build_failure_message(status) -> str: - reason = getattr(status, "reason", None) - parts = [] - message = getattr(reason, "message", None) - if message: - parts.append(message) - for entry in getattr(reason, "log_entries", None) or []: - text = getattr(entry, "message", None) - if text: - parts.append(text) - parts.extend(str(item) for item in (getattr(status, "logs", None) or [])) - return "\n".join(parts)[-12000:] or "Build failed" - - -def build_e2b_template( - manifest: TemplateManifest, - *, - template_cls=None, - attempts: int = 3, - build_timeout: int = 600, - force: bool = False, - on_build_logs=None, - registry_username: str | None = None, - registry_password: str | None = None, - context_dir: Path | None = None, - runtime_sources: dict[str, str] | None = None, -): - """Build idempotently, reconciling ambiguous API failures by tag.""" - if attempts <= 0: - raise ValueError("Template build attempts must be greater than zero") - if build_timeout <= 0: - raise ValueError("Template build timeout must be greater than zero") - if template_cls is None: - from e2b import Template as template_cls - fingerprint_tag = f"sforge-{manifest.fingerprint[:12]}" - previous_ready_build = _matching_build_id( - template_cls, - manifest.name, - manifest.tag, - fingerprint_tag, - ) - if previous_ready_build and not force: - return {"status": "existing", "build_id": previous_ready_build} - definition = build_template_definition( - manifest, - template_cls, - registry_username=registry_username, - registry_password=registry_password, - context_dir=context_dir, - runtime_sources=runtime_sources, - ) - error = None - used_attempts = 0 - for attempt in range(1, attempts + 1): - used_attempts = attempt - try: - info = template_cls.build_in_background( - definition, - manifest.name, - tags=[manifest.tag, fingerprint_tag], - cpu_count=manifest.cpu_count, - memory_mb=manifest.memory_mb, - skip_cache=force, - on_build_logs=on_build_logs, - ) - except Exception as exc: - if _is_apt_source_error(str(exc)): - raise AptSourceError(str(exc)) from exc - error = exc - accepted = _matching_build_id( - template_cls, - manifest.name, - manifest.tag, - fingerprint_tag, - ) - if accepted and accepted != previous_ready_build: - return {"status": "reconciled", "build_id": accepted} - if attempt < attempts and _is_retryable_build_error(exc): - time.sleep(min(2 ** (attempt - 1), 8)) - continue - break - - deadline = time.monotonic() + build_timeout - logs_offset = 0 - last_status_error = None - while time.monotonic() < deadline: - try: - status = template_cls.get_build_status( - info, - logs_offset=logs_offset, - ) - last_status_error = None - except Exception as exc: - last_status_error = exc - if not _is_retryable_build_error(exc): - raise RuntimeError( - f"Could not inspect E2B template build " - f"{info.template_id}/{info.build_id}: {exc}" - ) from exc - time.sleep(1) - continue - logs_offset += len(status.log_entries) - if on_build_logs is not None: - for entry in status.log_entries: - on_build_logs(entry) - state = getattr(status.status, "value", status.status) - if state == "ready": - return { - "status": "built", - "template_id": info.template_id, - "build_id": info.build_id, - } - if state == "error": - message = _build_failure_message(status) - if _is_apt_source_error(message): - raise AptSourceError(message) - error = RuntimeError(message) - accepted = _matching_build_id( - template_cls, manifest.name, manifest.tag, fingerprint_tag, - ) - if accepted and accepted != previous_ready_build: - return {"status": "reconciled", "build_id": accepted} - if attempt < attempts and _is_retryable_build_error(error): - time.sleep(min(2 ** (attempt - 1), 8)) - break - raise error - time.sleep(1) - else: - accepted = _matching_build_id( - template_cls, - manifest.name, - manifest.tag, - fingerprint_tag, - ) - if accepted == info.build_id: - return {"status": "reconciled", "build_id": accepted} - detail = ( - f": last status error: {last_status_error}" - if last_status_error else "" - ) - raise TimeoutError( - f"E2B template build timed out after {build_timeout}s " - f"(template_id={info.template_id}, build_id={info.build_id}); " - f"the E2B SDK has no per-build cancellation API{detail}" - ) - raise RuntimeError( - f"Failed to build E2B template {manifest.reference!r} after " - f"{used_attempts} attempts: {error}" - ) from error - - -def write_template_manifest( - manifest: TemplateManifest, - result: dict, - output_dir: Path, -) -> Path: - output_dir.mkdir(parents=True, exist_ok=True) - path = output_dir / f"{manifest.role}.json" - path.write_text( - json.dumps( - {**manifest.to_dict(), "result": result}, - indent=2, - ensure_ascii=False, - ) - ) - return path diff --git a/sforge/harness/run_agent.py b/sforge/harness/run_agent.py index 4c36465..4c8d04d 100644 --- a/sforge/harness/run_agent.py +++ b/sforge/harness/run_agent.py @@ -481,9 +481,10 @@ def run_agent( if not backend.image_exists(image_key): if backend.backend_name == "e2b": raise RuntimeError( - f"E2B Template for {image_key!r} was not found. Build it with: " - f"sforge e2b-template --task {task_spec.task_id} " - "--source-registry " + f"E2B Template for {image_key!r} was not found. Make sure " + "the task uses a published image version with an official " + "Template, or point SFORGE_E2B_TEMPLATE_MAP at a " + "self-built Template." ) raise RuntimeError( f"Image '{image_key}' not found. Run `sforge pull --task {task_spec.task_id}` to fetch from registry, or `sforge build --task {task_spec.task_id}` to build locally." From c462f8a1e23fd8ce0c426546c0e136443ffc9d96 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 25 Aug 2026 08:54:49 -0400 Subject: [PATCH 04/17] Pin work container exec user to agent (e2b execs default to the SDK user account) --- sforge/harness/run_agent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sforge/harness/run_agent.py b/sforge/harness/run_agent.py index 4c8d04d..0e040ef 100644 --- a/sforge/harness/run_agent.py +++ b/sforge/harness/run_agent.py @@ -629,6 +629,8 @@ def run_agent( cap_drop=container_cap_drop or None, cpu_limit=cpu, mem_limit=mem, + # e2b execs default to the SDK's "user" account, not the image USER + user="agent", ) backend.start_container(handle) logger.info(f"Container started: {container_name} (judge_url={judge_url})") From e52ea7e7488279cd26f9c8c6a3fd96f26ff30957 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 25 Aug 2026 14:01:10 -0400 Subject: [PATCH 05/17] Remove --max-workers; tasks run fully in parallel as on main --- docs/en/configuration/container-backends.md | 8 ++-- docs/en/reference/cli.md | 1 - docs/zh/configuration/container-backends.md | 5 +-- docs/zh/reference/cli.md | 1 - sforge/cli.py | 41 ++------------------- 5 files changed, 9 insertions(+), 47 deletions(-) diff --git a/docs/en/configuration/container-backends.md b/docs/en/configuration/container-backends.md index 28212c9..2f9645a 100644 --- a/docs/en/configuration/container-backends.md +++ b/docs/en/configuration/container-backends.md @@ -297,10 +297,10 @@ longer than that limit cannot be guaranteed to finish when the evaluator uses its full budget. Do not silently lower the evaluator timeout if leaderboard comparability matters; use a plan with a sufficient lifetime instead. -`--max-workers` limits concurrent Work Sandboxes, not total E2B usage. Budget -for one managed Controller, the Work Sandboxes, temporary Judge Sandboxes, and -active Game Sandboxes. Leave capacity for overlap while evaluations finish and -resources are cleaned up. +Tasks run fully in parallel, so total E2B usage scales with the number of +selected tasks. Budget for one managed Controller, the Work Sandboxes, +temporary Judge Sandboxes, and active Game Sandboxes, and leave capacity for +overlap while evaluations finish and resources are cleaned up. The final score retains SForge's backend-independent semantics: it is the best result among completed agent and auto-eval submissions. `final_archive.tar.gz` diff --git a/docs/en/reference/cli.md b/docs/en/reference/cli.md index c8cb481..cabaf66 100644 --- a/docs/en/reference/cli.md +++ b/docs/en/reference/cli.md @@ -145,7 +145,6 @@ sforge run --experiment experiment.yaml | `--judge-url` | `http://host.docker.internal:8080` | Judge server URL as seen from inside the container | | `--backend` | `docker` | Container backend (`docker`, `k8s`, or `e2b`) | | `--stagger` | --- | Spread task launches evenly over N seconds (e.g., `--stagger 300`) | -| `--max-workers` | backend-specific | Maximum number of independent task runs scheduled at once. Defaults to all selected tasks for Docker/Kubernetes and 4 for E2B. | | `--max-submissions` | --- | Maximum number of agent submissions per run | | `--submission-cooldown` | --- | Minimum seconds between agent submissions | | `--work-cpu-limit` | --- | Number of CPUs for work containers | diff --git a/docs/zh/configuration/container-backends.md b/docs/zh/configuration/container-backends.md index 85247da..cdbfdfc 100644 --- a/docs/zh/configuration/container-backends.md +++ b/docs/zh/configuration/container-backends.md @@ -282,9 +282,8 @@ Judge Server。 完成。如果要求与榜单口径可比，不应静默缩短 evaluator timeout，而应使用生命周期足够长的 套餐。 -`--max-workers` 只限制并行 Work Sandbox 数量，不等于 E2B 总实例数。容量规划还要计入一个 -managed Controller、临时 Judge Sandbox 和活跃 Game Sandbox，并为评测完成和资源清理的 -重叠阶段留出余量。 +任务是全并行运行的，E2B 总用量随所选任务数增长。容量规划要计入一个 managed Controller、 +临时 Judge Sandbox 和活跃 Game Sandbox，并为评测完成和资源清理的重叠阶段留出余量。 最终分数沿用 SForge 与后端无关的既有语义：只在已经完成的 agent submission 和 auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快照，timeout 时不会被 diff --git a/docs/zh/reference/cli.md b/docs/zh/reference/cli.md index a420c80..17391ee 100644 --- a/docs/zh/reference/cli.md +++ b/docs/zh/reference/cli.md @@ -144,7 +144,6 @@ sforge run --experiment experiment.yaml | `--judge-url` | `http://host.docker.internal:8080` | 容器内部看到的 Judge 服务器 URL | | `--backend` | `docker` | 容器后端（`docker`、`k8s` 或 `e2b`） | | `--stagger` | --- | 将任务启动均匀分散在 N 秒内（如 `--stagger 300`） | -| `--max-workers` | 取决于后端 | 同时调度的独立 task 数量上限。Docker/Kubernetes 默认为全部所选任务，E2B 默认为 4。 | | `--max-submissions` | --- | 每次运行的最大 Agent 提交次数 | | `--submission-cooldown` | --- | Agent 两次提交之间的最小间隔（秒） | | `--work-cpu-limit` | --- | Work 容器的 CPU 数量限制 | diff --git a/sforge/cli.py b/sforge/cli.py index 15613bb..2ba2ad2 100644 --- a/sforge/cli.py +++ b/sforge/cli.py @@ -40,7 +40,6 @@ from sforge.harness.run_evaluation import judge_submission from sforge.harness.task_spec import TaskSpec, load_all_tasks, make_task_spec -E2B_DEFAULT_MAX_WORKERS = 4 def _resolve_task(args, config: SForgeConfig) -> TaskSpec: @@ -91,23 +90,6 @@ def _make_config(args) -> SForgeConfig: return load_config(overrides) -def _positive_int(value: str) -> int: - parsed = int(value) - if parsed <= 0: - raise argparse.ArgumentTypeError("must be greater than zero") - return parsed - - -def _run_max_workers( - backend_name: str, task_count: int, requested: int | None, -) -> int: - if requested is not None: - return min(task_count, requested) - if backend_name == "e2b": - return min(task_count, E2B_DEFAULT_MAX_WORKERS) - return task_count - - # --- Commands --- @@ -608,12 +590,7 @@ def cmd_run(args): if stagger: print(f" Tasks: {', '.join(t.task_id for t in task_specs)} (staggered over {stagger}s, {stagger_delay:.1f}s apart)") else: - print(f" Tasks: {', '.join(t.task_id for t in task_specs)} (parallel)") - max_workers = _run_max_workers( - backend.backend_name, len(task_runs), - getattr(args, "max_workers", None), - ) - print(f" Workers: {max_workers}") + print(f" Tasks: {', '.join(t.task_id for t in task_specs)} (all in parallel)") print() shutdown_event = threading.Event() @@ -683,11 +660,7 @@ def _invoke(ts: TaskSpec, task_config: SForgeConfig, task_args): ) if multi: - max_workers = _run_max_workers( - backend.backend_name, len(task_runs), - getattr(args, "max_workers", None), - ) - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex: + with concurrent.futures.ThreadPoolExecutor(max_workers=len(task_runs)) as ex: futures = [] for i, (ts, tc, ta) in enumerate(task_runs): if i > 0 and stagger_delay > 0: @@ -1021,8 +994,7 @@ def main(): help="Container backend (default from SFORGE_BACKEND or 'docker')") p_run.add_argument("--task", default=None, nargs="+", help="One or more task IDs (e.g. --task ahc056 ahc057). " - "Multiple tasks are run in parallel, subject to " - "--max-workers.") + "Multiple tasks are run fully in parallel.") p_run.add_argument("--experiment", default=None, help="Path to experiment YAML config file (model config + per-task overrides)") p_run.add_argument("--agent", default=None, help="Agent name (claude-code, aider, codex)") @@ -1041,13 +1013,6 @@ def main(): help="Minimum seconds between agent submissions (default: no cooldown)") p_run.add_argument("--stagger", type=int, default=None, dest="stagger", help="Spread task launches evenly over N seconds (e.g. --stagger 300)") - p_run.add_argument( - "--max-workers", type=_positive_int, default=None, - help=( - "Maximum parallel tasks (default: all tasks for Docker/Kubernetes; " - f"{E2B_DEFAULT_MAX_WORKERS} for E2B)" - ), - ) p_run.add_argument("--judge-url", default="http://host.docker.internal:8080", help="Judge server URL") p_run.add_argument("--run-id", default=None, help="Run ID for tracking") net_group = p_run.add_mutually_exclusive_group() From a421d51083bd37840e8d3702ae97cc506b620378 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 25 Aug 2026 14:16:36 -0400 Subject: [PATCH 06/17] Deploy the judge server on the host as on main; drop the managed e2b controller --- docs/en/configuration/container-backends.md | 56 +-- .../en/configuration/environment-variables.md | 1 - docs/en/reference/judge-api.md | 53 +-- docs/zh/configuration/container-backends.md | 46 ++- .../zh/configuration/environment-variables.md | 1 - docs/zh/reference/judge-api.md | 50 +-- sforge/cli.py | 50 +-- sforge/harness/backend/e2b_backend.py | 7 +- sforge/harness/config.py | 2 - sforge/harness/e2b_controller.py | 391 ------------------ sforge/harness/evolve_scripts.py | 20 +- sforge/harness/judge_client.py | 102 ----- sforge/harness/judge_server.py | 112 +---- sforge/harness/run_agent.py | 306 ++++---------- 14 files changed, 201 insertions(+), 996 deletions(-) delete mode 100644 sforge/harness/e2b_controller.py delete mode 100644 sforge/harness/judge_client.py diff --git a/docs/en/configuration/container-backends.md b/docs/en/configuration/container-backends.md index 2f9645a..f2cabf2 100644 --- a/docs/en/configuration/container-backends.md +++ b/docs/en/configuration/container-backends.md @@ -14,7 +14,7 @@ or `e2b` for E2B-hosted environments. |---------|----------|-------------------| | `docker` | Local development, task debugging, single-machine evaluation, small experiments | A working local Docker daemon | | `k8s` | Shared clusters, high-concurrency batch evaluation, running Work/Judge pods in a cluster | `kubectl` access, images pullable by the cluster, and a Judge URL reachable from pods | -| `e2b` | Managed remote Work/Judge Sandboxes without operating a cluster or Judge broker | E2B API access and E2B Templates; see [E2B Backend](#e2b-backend) | +| `e2b` | Remote Work/Judge Sandboxes without operating a cluster | E2B API access, E2B Templates, and a routable Judge Server; see [E2B Backend](#e2b-backend) | If you are trying a task locally, use the default Docker backend first. The Kubernetes backend is intended for team environments that already have a cluster and container registry. The E2B backend has a one-time Template preparation step before a new or @@ -226,8 +226,9 @@ Notes: ## E2B Backend The E2B backend runs Work, Judge, and Game environments as E2B Sandboxes. -When `--judge-url` is omitted, SForge also creates a temporary managed Judge -Controller, so no separately deployed Judge service is required. +The Judge Server itself runs on a host you control, exactly as with the other +backends: start `sforge serve` on a machine the Sandboxes can reach and pass +its address via `--judge-url`. ### Docker Images and E2B Templates @@ -243,10 +244,22 @@ modified task. ### Run a Task +Start the Judge Server on a host with a publicly routable address (agents +inside E2B Sandboxes submit to it over the internet): + ```bash uv sync --extra e2b export E2B_API_KEY=... +export SFORGE_ADMIN_SECRET=... # same value on the serve and run hosts + +sforge serve --host 0.0.0.0 --port 8080 +``` + +Then run the agent, pointing `--judge-url` at that host: +```bash +export E2B_API_KEY=... +export SFORGE_ADMIN_SECRET=... # same value as on the serve host export SFORGE_AGENT_API_KEY=... export SFORGE_AGENT_API_BASE_URL=... export SFORGE_AGENT_MODEL=... @@ -254,19 +267,14 @@ export SFORGE_AGENT_MODEL=... sforge run \ --backend e2b \ --task ad_placement_optimization \ - --agent claude-code + --agent claude-code \ + --judge-url http://YOUR_HOST:8080 ``` -SForge checks the Work/Judge Templates, creates a secured Controller, registers -the task, creates Work/Judge/Game Sandboxes as needed, propagates gateway -credentials, collects results, and removes the Sandboxes. Pass `--judge-url` -only when using a separately deployed Judge Server. - -Work Sandboxes reach a managed Controller through its secured E2B service -gateway. Host-side orchestration requests use the E2B command channel and the -Controller's loopback interface instead, avoiding an unnecessary round trip -through that public gateway. This transport choice is specific to E2B and does -not change Docker or Kubernetes Judge routing. +The Judge Server needs `E2B_API_KEY` too: registrations from an e2b run make +it create Judge and Game Sandboxes on E2B (mirroring how a k8s run makes it +schedule judge pods). Work Sandboxes submit to the Judge Server's public URL; +the Judge Server reaches Game Sandboxes through the E2B service gateway. Each E2B evaluation is scoped to one task. Supplying multiple task IDs only batch-schedules several independent evaluations; it does not combine tasks into @@ -283,13 +291,7 @@ existing E2B Template. Keep the agent timeout comfortably below the team's maximum Sandbox lifetime. The lifetime must also cover agent installation, archive extraction, pending -Judge evaluations, result collection, and cleanup. A managed Judge Controller -lives for the entire multi-task command, so the whole staggered or multi-wave -batch—not only one task—must fit within that Controller's permitted lifetime. -For example, a team limited to one-hour Sandboxes should use a shorter agent -budget such as 30 minutes and avoid scheduling enough waves to keep one managed -Controller alive for an hour. Alternatively, use a separately deployed Judge -Server. +Judge evaluations, result collection, and cleanup. Check each task's `judge.eval_timeout` as well. A Judge Sandbox is subject to the same team lifetime limit, so a task whose declared evaluator timeout is @@ -298,9 +300,9 @@ its full budget. Do not silently lower the evaluator timeout if leaderboard comparability matters; use a plan with a sufficient lifetime instead. Tasks run fully in parallel, so total E2B usage scales with the number of -selected tasks. Budget for one managed Controller, the Work Sandboxes, -temporary Judge Sandboxes, and active Game Sandboxes, and leave capacity for -overlap while evaluations finish and resources are cleaned up. +selected tasks. Budget for the Work Sandboxes, temporary Judge Sandboxes, and +active Game Sandboxes, and leave capacity for overlap while evaluations finish +and resources are cleaned up. The final score retains SForge's backend-independent semantics: it is the best result among completed agent and auto-eval submissions. `final_archive.tar.gz` @@ -313,7 +315,8 @@ ends. | Input | When required | | --- | --- | -| `E2B_API_KEY` | E2B runs | +| `E2B_API_KEY` | E2B runs, on both the `sforge run` host and the Judge Server host | +| A publicly routable `--judge-url` | All E2B runs | | Agent API key, base URL, and model | Agent runs | Host `HTTP_PROXY` and `HTTPS_PROXY` variables are not copied into E2B Work, @@ -326,8 +329,7 @@ tasks remove proxy variables entirely. See [Network Isolation](/en/features/netw | Symptom | Likely cause | Fix | | --- | --- | --- | | Template not found | The task image version has no published official Template, or the task uses a modified image | Verify the task uses a published image version, or point `SFORGE_E2B_TEMPLATE_MAP` at a self-built Template | -| A secured Judge returns 403 | Its traffic access token was not provided | Use managed mode, or set `SFORGE_JUDGE_ACCESS_TOKEN` for an external secured Judge | -| Agent-side Judge requests intermittently time out | The Work Sandbox could not establish a connection through the E2B public gateway | Retry the agent submission, reduce simultaneous submission bursts, or use an external Judge Server; host-side managed-controller calls already use E2B command-channel loopback | +| Agent-side Judge requests intermittently time out | The Work Sandbox could not reach the Judge Server over the internet | Check that the Judge host and port are publicly reachable, then retry the agent submission | | A run ends at the configured task timeout with `Sandbox not found` | The team-level maximum Sandbox lifetime was reached before final extraction and cleanup | Shorten the agent timeout, reduce multi-wave batch duration, or use a plan with a longer Sandbox lifetime | | A run leaves a Sandbox | Cleanup failed or the runner was terminated abruptly | Inspect the run log and wait for the configured E2B TTL as the final safeguard | diff --git a/docs/en/configuration/environment-variables.md b/docs/en/configuration/environment-variables.md index 2d5738f..3083648 100644 --- a/docs/en/configuration/environment-variables.md +++ b/docs/en/configuration/environment-variables.md @@ -72,7 +72,6 @@ host allowlist directly. |----------|---------| | `SFORGE_JUDGE_EXTRA_ENV` | Extra env vars for judge containers, format: `"KEY1=VAL1,KEY2=VAL2"` | | `SFORGE_ADMIN_SECRET` | Trusted-host credential for Judge session registration and auto-eval. If omitted, SForge creates a mode-`0600` secret under `SFORGE_LOG_DIR`. Never expose it to Work containers. | -| `SFORGE_JUDGE_ACCESS_TOKEN` | E2B traffic access token for an explicitly configured secured Judge URL. Managed mode sets this automatically. | ## Resource Limit Variables diff --git a/docs/en/reference/judge-api.md b/docs/en/reference/judge-api.md index e0b89e1..6b29b12 100644 --- a/docs/en/reference/judge-api.md +++ b/docs/en/reference/judge-api.md @@ -16,11 +16,7 @@ All endpoints are prefixed with `/api/v1`. ## Authentication -The Judge API uses a **token-based session model**. The trusted host registers -a session with the `X-SForge-Admin-Secret` header; the returned session token is then used -for submissions, result polling, history, and Game routes. SForge generates a -mode-`0600` local secret when the variable is not set. Do not expose the admin -secret to Work Sandboxes or agents. +The Judge API uses a **token-based session model**. Before submitting, clients must register a session to obtain a token. The token encodes the task ID, run ID, and auto-incrementing round counters. ## Endpoints @@ -72,15 +68,16 @@ Submit a code archive for evaluation. The server resolves the task ID and run ID ```http POST /api/v1/submit -Authorization: Bearer Content-Type: multipart/form-data +token: a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4 archive: @solution.tar.gz kind: agent ``` | Field | Type | Required | Description | |-------|------|----------|-------------| +| `token` | string | yes | Session token from `/register` | | `archive` | file | yes | `.tar.gz` archive of the solution | | `kind` | string | no | `"agent"` (default) or `"auto"`. Determines which round counter is incremented. | @@ -102,7 +99,6 @@ Poll for the result of a submission. ```http GET /api/v1/result/{submission_id} -Authorization: Bearer ``` **Response (queued/running):** @@ -159,13 +155,18 @@ Authorization: Bearer ### Run History -Retrieve the agent-visible submission history for a run session, including -the best score selection. Host-side SForge requests add the admin secret to -include auto-eval entries. +Retrieve the full submission history for a run session, including the best score selection. + +**By token:** + +```http +GET /api/v1/history?token= +``` + +**By run ID:** ```http -GET /api/v1/history -Authorization: Bearer +GET /api/v1/runs/{run_id}/history?task_id= ``` **Response:** @@ -201,16 +202,12 @@ The `best_*` fields are computed using the task's configured selection policy (s ### Game Endpoints -For tasks with `game_mode: true`, the Judge server manages interactive game -sessions in dedicated containers. Every Game request requires -`Authorization: Bearer ` and the token must match the URL's -run and task. +For tasks with `game_mode: true`, the Judge server manages interactive game sessions in dedicated containers. #### Start New Game ```http POST /api/v1/game/{run_id}/{task_id}/new -Authorization: Bearer Content-Type: application/json {} ``` @@ -233,7 +230,6 @@ Content-Type: application/json ```http POST /api/v1/game/{run_id}/{task_id}/{session_id}/step -Authorization: Bearer Content-Type: application/json {"action": "go north"} @@ -257,7 +253,6 @@ Content-Type: application/json ```http GET /api/v1/game/{run_id}/{task_id}/{session_id}/status -Authorization: Bearer ``` **Response:** @@ -277,7 +272,6 @@ Authorization: Bearer ```http POST /api/v1/game/{run_id}/{task_id}/{session_id}/close -Authorization: Bearer ``` **Response:** @@ -298,7 +292,6 @@ Close all active game sessions for a given run and task. ```http POST /api/v1/game/{run_id}/{task_id}/close-all -Authorization: Bearer ``` **Response:** @@ -315,18 +308,16 @@ Game sessions have a 10-minute idle timeout. Sessions are automatically archived ```bash # 1. Register a session -export SFORGE_ADMIN_SECRET=... TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/register \ -H "Content-Type: application/json" \ - -H "X-SForge-Admin-Secret: $SFORGE_ADMIN_SECRET" \ - -d '{"task_id":"ad_placement_optimization","run_id":"run-001"}' \ + -d '{"task_id": "ad_placement_optimization", "run_id": "run-001"}' \ | jq -r '.token') echo "Token: $TOKEN" # 2. Submit an archive SUBMISSION_ID=$(curl -s -X POST http://localhost:8080/api/v1/submit \ - -H "Authorization: Bearer $TOKEN" \ + -F "token=$TOKEN" \ -F "archive=@solution.tar.gz" \ -F "kind=agent" \ | jq -r '.submission_id') @@ -334,24 +325,20 @@ SUBMISSION_ID=$(curl -s -X POST http://localhost:8080/api/v1/submit \ echo "Submission: $SUBMISSION_ID" # 3. Poll for result (repeat until status is "completed" or "error") -curl -s -H "Authorization: Bearer $TOKEN" \ - "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq . +curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq . # 4. Check run history -curl -s -H "Authorization: Bearer $TOKEN" \ - http://localhost:8080/api/v1/history | jq . +curl -s "http://localhost:8080/api/v1/history?token=$TOKEN" | jq . ``` ### Poll loop ```bash while true; do - STATUS=$(curl -s -H "Authorization: Bearer $TOKEN" \ - "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq -r '.status') + STATUS=$(curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq -r '.status') echo "Status: $STATUS" if [ "$STATUS" = "completed" ] || [ "$STATUS" = "error" ]; then - curl -s -H "Authorization: Bearer $TOKEN" \ - "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq . + curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq . break fi sleep 5 diff --git a/docs/zh/configuration/container-backends.md b/docs/zh/configuration/container-backends.md index cdbfdfc..6d95b48 100644 --- a/docs/zh/configuration/container-backends.md +++ b/docs/zh/configuration/container-backends.md @@ -223,8 +223,9 @@ sforge run \ ## E2B 后端 -E2B 后端使用 E2B Sandbox 运行 Work、Judge 和 Game 环境。不传 `--judge-url` 时， -SForge 还会创建临时的 managed Judge Controller，用户不需要单独部署 Judge 服务。 +E2B 后端使用 E2B Sandbox 运行 Work、Judge 和 Game 环境。Judge Server 本身与其他后端 +一样部署在你自己的主机上：在 Sandbox 可达的机器上启动 `sforge serve`，并通过 +`--judge-url` 传入其地址。 ### Docker 镜像与 E2B Template @@ -238,10 +239,21 @@ SForge 维护者预先构建并发布，用户不需要执行任何 Template 准 ### 运行任务 +在有公网可达地址的主机上启动 Judge Server（E2B Sandbox 里的 Agent 通过公网向它提交）： + ```bash uv sync --extra e2b export E2B_API_KEY=... +export SFORGE_ADMIN_SECRET=... # serve 和 run 两台主机需使用同一个值 + +sforge serve --host 0.0.0.0 --port 8080 +``` + +然后运行 Agent，`--judge-url` 指向该主机： +```bash +export E2B_API_KEY=... +export SFORGE_ADMIN_SECRET=... # 与 serve 主机一致 export SFORGE_AGENT_API_KEY=... export SFORGE_AGENT_API_BASE_URL=... export SFORGE_AGENT_MODEL=... @@ -249,16 +261,14 @@ export SFORGE_AGENT_MODEL=... sforge run \ --backend e2b \ --task ad_placement_optimization \ - --agent claude-code + --agent claude-code \ + --judge-url http://YOUR_HOST:8080 ``` -SForge 会预检 Work/Judge Template，创建 secured Controller，注册 task，按需创建 -Work/Judge/Game Sandbox，传递 gateway 凭证，收集结果并清理 Sandbox。只有使用独立部署的 -Judge Server 时才需要传 `--judge-url`。 - -Work Sandbox 通过受保护的 E2B service gateway 访问 managed Controller；host 侧编排请求 -则通过 E2B command channel 访问 Controller 的 loopback 地址，避免不必要地绕行公网 -gateway。这个传输选择只用于 E2B，不改变 Docker 或 Kubernetes 的 Judge 路由。 +Judge Server 主机同样需要 `E2B_API_KEY`：e2b 运行注册后，Judge Server 会在 E2B 上 +创建 Judge 和 Game Sandbox（与 k8s 运行让它调度 judge pod 的机制对称）。Work Sandbox +向 Judge Server 的公网地址提交；Judge Server 通过 E2B service gateway 访问 Game +Sandbox。 每个 E2B 评测实例只对应一个 task。传入多个 task ID 时，CLI 只是批量调度多个相互独立的 评测，不会把多个 task 合并进同一个 Work 或 Judge 环境。本文验证采用每条 @@ -271,19 +281,15 @@ E2B 套餐限制是运行契约的一部分。Template 的 CPU 和内存会在 运行时的 `--work-*` 和 `--judge-*` 参数不能调整已有 E2B Template 的资源。 Agent timeout 应明显短于当前套餐允许的最大 Sandbox 生命周期。Sandbox 生命周期还要覆盖 -Agent 安装、归档提取、等待 Judge 评测、读取结果和资源清理。managed Judge Controller 会在 -整条多任务命令期间持续运行，因此整个错峰或多波次批次都必须落在 Controller 允许的生命周期 -内，而不只是单个 task。比如套餐只允许一小时 Sandbox 时，可使用 30 分钟等更短的 Agent -预算，并避免让同一个 managed Controller 因多波调度运行满一小时；也可以改用独立部署的 -Judge Server。 +Agent 安装、归档提取、等待 Judge 评测、读取结果和资源清理。 还需要检查每个任务的 `judge.eval_timeout`。Judge Sandbox 同样受 team 最大生命周期限制； 如果任务声明的评测超时长于该限制，那么当 evaluator 实际使用完整预算时，就无法保证评测 完成。如果要求与榜单口径可比，不应静默缩短 evaluator timeout，而应使用生命周期足够长的 套餐。 -任务是全并行运行的，E2B 总用量随所选任务数增长。容量规划要计入一个 managed Controller、 -临时 Judge Sandbox 和活跃 Game Sandbox，并为评测完成和资源清理的重叠阶段留出余量。 +任务是全并行运行的，E2B 总用量随所选任务数增长。容量规划要计入 Work Sandbox、临时 +Judge Sandbox 和活跃 Game Sandbox，并为评测完成和资源清理的重叠阶段留出余量。 最终分数沿用 SForge 与后端无关的既有语义：只在已经完成的 agent submission 和 auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快照，timeout 时不会被 @@ -294,7 +300,8 @@ auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快 | 输入 | 需要场景 | | --- | --- | -| `E2B_API_KEY` | E2B 运行 | +| `E2B_API_KEY` | E2B 运行（`sforge run` 主机和 Judge Server 主机都需要） | +| 公网可达的 `--judge-url` | 所有 E2B 运行 | | Agent API key、base URL 和模型 | 运行 Agent | 宿主机的 `HTTP_PROXY` 和 `HTTPS_PROXY` 不会复制到 E2B Work、Judge 或 Game Sandbox。 @@ -306,8 +313,7 @@ auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快 | 现象 | 可能原因 | 处理方式 | | --- | --- | --- | | 找不到 Template | 该任务镜像版本没有已发布的官方 Template，或任务使用了修改过的镜像 | 确认任务使用已发布的镜像版本，或用 `SFORGE_E2B_TEMPLATE_MAP` 指向自建 Template | -| secured Judge 返回 403 | 缺少 traffic access token | 使用 managed 模式，或为外部 secured Judge 配置 `SFORGE_JUDGE_ACCESS_TOKEN` | -| Agent 侧 Judge 请求偶发超时 | Work Sandbox 无法通过 E2B 公网 gateway 建立连接 | 让 Agent 重试提交、降低同时提交的突发并发，或使用独立 Judge Server；host 侧 managed-controller 请求已通过 E2B command channel 走 loopback | +| Agent 侧 Judge 请求偶发超时 | Work Sandbox 无法通过公网访问 Judge Server | 确认 Judge 主机和端口公网可达后重试提交 | | task timeout 到达时出现 `Sandbox not found` | 在最终归档和清理前达到了 team 级 Sandbox 最大生命周期 | 缩短 Agent timeout、减少多波次批次总时长，或使用支持更长 Sandbox 生命周期的套餐 | | 运行结束后仍有 Sandbox | cleanup 失败或 Runner 被强制终止 | 检查 run 日志；平台最终按配置的 E2B TTL 回收 | diff --git a/docs/zh/configuration/environment-variables.md b/docs/zh/configuration/environment-variables.md index 9649dab..38edd27 100644 --- a/docs/zh/configuration/environment-variables.md +++ b/docs/zh/configuration/environment-variables.md @@ -71,7 +71,6 @@ E2B 原生域名白名单直接放行。 |------|------| | `SFORGE_JUDGE_EXTRA_ENV` | Judge 容器额外环境变量，格式：`"KEY1=VAL1,KEY2=VAL2"` | | `SFORGE_ADMIN_SECRET` | 可信 Host 用于注册 Judge 会话和 auto-eval 的凭证；未设置时在 `SFORGE_LOG_DIR` 下生成权限为 `0600` 的 secret 文件，不得注入 Work 容器。 | -| `SFORGE_JUDGE_ACCESS_TOKEN` | 显式配置 secured Judge URL 时使用的 E2B traffic access token；managed 模式自动设置。 | ## 资源限制变量 diff --git a/docs/zh/reference/judge-api.md b/docs/zh/reference/judge-api.md index 7f315f8..3b7ed64 100644 --- a/docs/zh/reference/judge-api.md +++ b/docs/zh/reference/judge-api.md @@ -16,10 +16,7 @@ http://localhost:8080 ## 认证机制 -Judge API 使用**基于 Token 的会话模型**。可信 Host 使用 -`X-SForge-Admin-Secret` Header 注册会话，返回的会话 Token 用于提交、结果轮询、历史和 Game -接口。未显式配置时，SForge 会在本地生成权限为 `0600` 的 secret 文件。不得把 admin -secret 暴露给 Work Sandbox 或 Agent。 +Judge API 使用**基于 Token 的会话模型**。在提交之前，客户端必须先注册会话以获取 Token。Token 编码了任务 ID、运行 ID 和自增的轮次计数器。 ## 接口列表 @@ -71,15 +68,16 @@ Token 是一个 32 字符的十六进制字符串，内部维护任务 ID、运 ```http POST /api/v1/submit -Authorization: Bearer Content-Type: multipart/form-data +token: a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4 archive: @solution.tar.gz kind: agent ``` | 字段 | 类型 | 必填 | 说明 | |------|------|------|------| +| `token` | string | 是 | 从 `/register` 获取的会话 Token | | `archive` | file | 是 | `.tar.gz` 格式的解决方案归档 | | `kind` | string | 否 | `"agent"`（默认）或 `"auto"`。决定递增哪个轮次计数器。 | @@ -101,7 +99,6 @@ kind: agent ```http GET /api/v1/result/{submission_id} -Authorization: Bearer ``` **响应（排队中/运行中）：** @@ -158,12 +155,18 @@ Authorization: Bearer ### 运行历史 -获取某次运行会话中 Agent 可见的提交历史和最佳分数。SForge 的 Host 请求会额外携带 -admin secret，以包含 auto-eval 记录。 +获取某次运行会话的完整提交历史，包括最佳分数的选定结果。 + +**通过 Token 查询：** + +```http +GET /api/v1/history?token= +``` + +**通过运行 ID 查询：** ```http -GET /api/v1/history -Authorization: Bearer +GET /api/v1/runs/{run_id}/history?task_id= ``` **响应：** @@ -199,15 +202,12 @@ Authorization: Bearer ### 游戏接口 -对于 `game_mode: true` 的任务，Judge 服务器在专用容器中管理交互式游戏会话。所有 -Game 请求都必须携带 `Authorization: Bearer `，且 Token 必须与 URL 中 -的 run 和 task 匹配。 +对于 `game_mode: true` 的任务，Judge 服务器在专用容器中管理交互式游戏会话。 #### 创建新游戏 ```http POST /api/v1/game/{run_id}/{task_id}/new -Authorization: Bearer Content-Type: application/json {} ``` @@ -230,7 +230,6 @@ Content-Type: application/json ```http POST /api/v1/game/{run_id}/{task_id}/{session_id}/step -Authorization: Bearer Content-Type: application/json {"action": "go north"} @@ -254,7 +253,6 @@ Content-Type: application/json ```http GET /api/v1/game/{run_id}/{task_id}/{session_id}/status -Authorization: Bearer ``` **响应：** @@ -274,7 +272,6 @@ Authorization: Bearer ```http POST /api/v1/game/{run_id}/{task_id}/{session_id}/close -Authorization: Bearer ``` **响应：** @@ -295,7 +292,6 @@ Authorization: Bearer ```http POST /api/v1/game/{run_id}/{task_id}/close-all -Authorization: Bearer ``` **响应：** @@ -312,18 +308,16 @@ Authorization: Bearer ```bash # 1. 注册会话 -export SFORGE_ADMIN_SECRET=... TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/register \ -H "Content-Type: application/json" \ - -H "X-SForge-Admin-Secret: $SFORGE_ADMIN_SECRET" \ - -d '{"task_id":"ad_placement_optimization","run_id":"run-001"}' \ + -d '{"task_id": "ad_placement_optimization", "run_id": "run-001"}' \ | jq -r '.token') echo "Token: $TOKEN" # 2. 提交归档 SUBMISSION_ID=$(curl -s -X POST http://localhost:8080/api/v1/submit \ - -H "Authorization: Bearer $TOKEN" \ + -F "token=$TOKEN" \ -F "archive=@solution.tar.gz" \ -F "kind=agent" \ | jq -r '.submission_id') @@ -331,24 +325,20 @@ SUBMISSION_ID=$(curl -s -X POST http://localhost:8080/api/v1/submit \ echo "Submission: $SUBMISSION_ID" # 3. 轮询结果（重复直到 status 为 "completed" 或 "error"） -curl -s -H "Authorization: Bearer $TOKEN" \ - "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq . +curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq . # 4. 查看运行历史 -curl -s -H "Authorization: Bearer $TOKEN" \ - http://localhost:8080/api/v1/history | jq . +curl -s "http://localhost:8080/api/v1/history?token=$TOKEN" | jq . ``` ### 轮询循环 ```bash while true; do - STATUS=$(curl -s -H "Authorization: Bearer $TOKEN" \ - "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq -r '.status') + STATUS=$(curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq -r '.status') echo "Status: $STATUS" if [ "$STATUS" = "completed" ] || [ "$STATUS" = "error" ]; then - curl -s -H "Authorization: Bearer $TOKEN" \ - "http://localhost:8080/api/v1/result/$SUBMISSION_ID" | jq . + curl -s http://localhost:8080/api/v1/result/$SUBMISSION_ID | jq . break fi sleep 5 diff --git a/sforge/cli.py b/sforge/cli.py index 2ba2ad2..1f713f7 100644 --- a/sforge/cli.py +++ b/sforge/cli.py @@ -26,6 +26,7 @@ import threading import time from pathlib import Path +from urllib.parse import urlparse import docker @@ -318,7 +319,6 @@ def _run_single_task( shutdown_event=shutdown_event, max_submissions=getattr(args, "max_submissions", None), submission_cooldown=getattr(args, "submission_cooldown", None), - host_judge_connection=getattr(args, "host_judge_connection", None), ) print(f"\nAgent completed in {result.runtime_seconds:.1f}s") @@ -546,7 +546,6 @@ def cmd_run(args): sys.exit(1) run_id = args.run_id or uuid.uuid4().hex[:12] - managed_controller = None multi = len(task_specs) > 1 verbose = not args.silent and not multi @@ -574,6 +573,19 @@ def cmd_run(args): ) base_config.backend = backend_names.pop() backend = create_backend_from_config(base_config) + if backend.backend_name == "e2b": + unroutable = [ + ta.judge_url for _, _, ta in task_runs + if urlparse(ta.judge_url).hostname == "host.docker.internal" + ] + if unroutable: + print( + "Error: the e2b backend needs a Judge Server the sandboxes " + "can reach. Start `sforge serve` on a routable host and pass " + "its address via --judge-url.", + file=sys.stderr, + ) + sys.exit(1) # Resolve stagger: CLI flag wins over experiment YAML stagger = args.stagger @@ -617,28 +629,6 @@ def _invoke(ts: TaskSpec, task_config: SForgeConfig, task_args): summaries: list[dict] = [] try: - managed_tasks = [ - (task_config, task_args) - for _, task_config, task_args in task_runs - if backend.backend_name == "e2b" - and task_args.judge_url == "http://host.docker.internal:8080" - ] - if managed_tasks: - from sforge.harness.e2b_controller import ManagedE2BController - - managed_controller = ManagedE2BController( - backend, base_config, run_id, - ) - connection = managed_controller.start() - access_token = connection.headers.get( - "e2b-traffic-access-token" - ) - for task_config, task_args in managed_tasks: - task_args.judge_url = connection.url - task_config.e2b_judge_access_token = access_token - task_args.host_judge_connection = managed_controller.host_connection - print(f"Managed E2B Judge controller: {connection.url}") - for ts, task_config, task_args in task_runs: cfg_dict = _effective_config_dict(ts, task_args, task_config) unified_tasks[ts.task_id] = cfg_dict @@ -688,19 +678,7 @@ def _invoke(ts: TaskSpec, task_config: SForgeConfig, task_args): signal.signal(signal.SIGINT, signal.SIG_IGN) print("\nShutting down — stopping containers...") finally: - primary_error = sys.exc_info()[1] signal.signal(signal.SIGINT, old_sigint) - if managed_controller is not None: - try: - managed_controller.close() - except Exception as cleanup_error: - if primary_error is None: - raise - print( - "Managed E2B controller cleanup failed while preserving " - f"the original run error: {cleanup_error}", - file=sys.stderr, - ) if multi: run_root = base_config.log_dir / "runs" / run_id diff --git a/sforge/harness/backend/e2b_backend.py b/sforge/harness/backend/e2b_backend.py index ea902dc..3bfbb7c 100644 --- a/sforge/harness/backend/e2b_backend.py +++ b/sforge/harness/backend/e2b_backend.py @@ -371,16 +371,15 @@ def cleanup_container( return except Exception as exc: if attempt == 3: + # Match the docker/k8s cleanup contract: log and rely on + # the on_timeout=kill lifecycle instead of raising. log.error( "Failed to kill sandbox %s after %d attempts: %s", handle.id, attempt, exc, ) - raise RuntimeError( - f"Failed to kill E2B Sandbox {handle.id} after " - f"{attempt} attempts: {exc}" - ) from exc + return time.sleep(attempt) def container_exists(self, name: str) -> bool: diff --git a/sforge/harness/config.py b/sforge/harness/config.py index 1eee502..42648fc 100644 --- a/sforge/harness/config.py +++ b/sforge/harness/config.py @@ -82,7 +82,6 @@ class SForgeConfig: # E2B_API_KEY / E2B_API_URL / E2B_DOMAIN env vars read by the e2b SDK) e2b_template_map: dict[str, str] = field(default_factory=dict) e2b_sandbox_ttl: int | None = None - e2b_judge_access_token: str | None = None # Paths log_dir: Path = field(default_factory=lambda: LOG_DIR) tasks_dir: Path = field(default_factory=lambda: TASKS_DIR) @@ -114,7 +113,6 @@ def load_config(cli_overrides: dict | None = None) -> SForgeConfig: "k8s_namespace": ["SFORGE_K8S_NAMESPACE"], "k8s_image_registry": ["SFORGE_K8S_IMAGE_REGISTRY"], "k8s_kubeconfig": ["SFORGE_K8S_KUBECONFIG"], - "e2b_judge_access_token": ["SFORGE_JUDGE_ACCESS_TOKEN"], "work_cpu_limit": ["SFORGE_WORK_CPU_LIMIT"], "work_mem_limit": ["SFORGE_WORK_MEM_LIMIT"], "log_dir": ["SFORGE_LOG_DIR"], diff --git a/sforge/harness/e2b_controller.py b/sforge/harness/e2b_controller.py deleted file mode 100644 index cf5a4cb..0000000 --- a/sforge/harness/e2b_controller.py +++ /dev/null @@ -1,391 +0,0 @@ -# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Managed Judge controller for brokerless official E2B runs.""" - -from __future__ import annotations - -import io -import json -import logging -import os -import shlex -import tarfile -import time -import uuid -from importlib.metadata import PackageNotFoundError, requires -from pathlib import Path, PurePosixPath - -import requests - -from sforge.harness.backend import ContainerBackend, ContainerHandle -from sforge.harness.config import SForgeConfig -from sforge.harness.constants import get_admin_secret -from sforge.harness.judge_client import JudgeConnection - -logger = logging.getLogger(__name__) - - -def _controller_requirements() -> tuple[str, ...]: - """Return this installed SForge version's controller dependencies.""" - try: - declared = requires("sforge") or [] - except PackageNotFoundError: - declared = [] - runtime = [item for item in declared if "extra ==" not in item] - e2b = [ - item.split(";", 1)[0].strip() - for item in declared - if "extra == 'e2b'" in item or 'extra == "e2b"' in item - ] - return tuple(runtime + e2b) - - -class ManagedE2BController: - """Create, expose and tear down a Judge Server in an E2B Sandbox.""" - - def __init__( - self, - backend: ContainerBackend, - config: SForgeConfig, - run_id: str, - *, - source_root: Path | None = None, - ) -> None: - self.backend = backend - self.config = config - self.run_id = run_id - self.source_root = source_root or Path(__file__).resolve().parents[2] - self.handle: ContainerHandle | None = None - self.connection: JudgeConnection | None = None - self.host_connection: JudgeConnection | None = None - - def _environment(self) -> dict[str, str]: - allowed = ("E2B_API_KEY", "E2B_API_URL", "E2B_DOMAIN") - env = {key: os.environ[key] for key in allowed if os.environ.get(key)} - env.update( - { - "PYTHONPATH": "/opt/sforge-controller", - "SFORGE_TASKS_DIR": "/opt/sforge-controller/tasks", - "SFORGE_LOG_DIR": "/tmp/sforge-controller/logs", - "SFORGE_BACKEND": "e2b", - "SFORGE_ADMIN_SECRET": get_admin_secret(self.config.log_dir), - "SFORGE_E2B_SANDBOX_TTL": str(self.config.e2b_sandbox_ttl or 3600), - } - ) - if self.config.judge_extra_env: - env["SFORGE_JUDGE_EXTRA_ENV"] = ",".join( - f"{key}={value}" for key, value in self.config.judge_extra_env.items() - ) - if self.config.e2b_template_map: - env["SFORGE_E2B_TEMPLATE_MAP"] = ",".join( - f"{key}={value}" for key, value in self.config.e2b_template_map.items() - ) - return env - - def start(self, timeout: int = 120) -> JudgeConnection: - if self.handle is not None: - raise RuntimeError("Managed E2B controller is already running") - env = self._environment() - self.handle = self.backend.create_container( - "base", - f"sforge.controller.{self.run_id}", - user="root", - ) - try: - self.backend.copy_to_container( - self.handle, - self.source_root / "sforge", - PurePosixPath("/opt/sforge-controller/sforge"), - ) - self.backend.copy_to_container( - self.handle, - self.config.tasks_dir, - PurePosixPath("/opt/sforge-controller/tasks"), - ) - requirements = _controller_requirements() - if not requirements: - raise RuntimeError( - "Could not determine managed controller dependencies from " - "the installed SForge package metadata" - ) - requirement_args = " ".join(shlex.quote(item) for item in requirements) - bootstrap = self.backend.exec_run_with_exit_code( - self.handle, - "python3 -m venv /tmp/sforge-controller-venv && " - "/tmp/sforge-controller-venv/bin/python -m pip install " - "--disable-pip-version-check --no-cache-dir --quiet " - + requirement_args, - timeout=300, - user="root", - ) - if bootstrap.exit_code != 0: - raise RuntimeError( - "Failed to prepare managed E2B Judge controller: " - + bootstrap.output[-8000:] - ) - command = ( - "cd /opt/sforge-controller && " - "/tmp/sforge-controller-venv/bin/python -m sforge.cli serve " - "--host 0.0.0.0 --port 8080 " - "> /tmp/sforge-judge.log 2>&1" - ) - self.backend.exec_run( - self.handle, - command, - detach=True, - user="root", - environment=env, - ) - endpoint = self.backend.get_service_endpoint(self.handle, 8080) - self.connection = JudgeConnection.from_e2b_traffic_token( - endpoint.url, - endpoint.headers.get("e2b-traffic-access-token"), - ) - self.host_connection = JudgeConnection( - url="http://127.0.0.1:8080", - request_handler=self._request_on_controller, - ) - deadline = time.time() + timeout - while time.time() < deadline: - try: - response = self.connection.request( - "GET", - "/openapi.json", - timeout=5, - ) - if response.status_code == 200: - return self.connection - except requests.RequestException: - pass - time.sleep(1) - log = self.backend.exec_run( - self.handle, - "tail -200 /tmp/sforge-judge.log", - user="root", - ) - raise RuntimeError( - "Managed E2B Judge controller did not become ready: " - + log.output[-8000:] - ) - except BaseException: - try: - self.close() - except Exception: - logger.exception( - "Failed to clean up managed E2B controller %s after startup error", - self.run_id, - ) - raise - - def close(self) -> None: - if self.handle is not None: - try: - self._export_run_logs() - except Exception as exc: - logger.warning( - "Could not export managed E2B controller logs for %s: %s", - self.run_id, - exc, - ) - self.backend.cleanup_container(self.handle) - self.handle = None - self.connection = None - self.host_connection = None - - def _request_on_controller( - self, method: str, path: str, **kwargs, - ) -> requests.Response: - """Send a host-side Judge request over the E2B command channel. - - Work Sandboxes still use the secured public gateway. Host orchestration - can avoid that extra hop by issuing the same HTTP request from inside - the Controller Sandbox against loopback. - """ - if self.handle is None: - raise RuntimeError("Managed E2B Judge controller is not running") - - request_id = uuid.uuid4().hex - prefix = f"/tmp/.sforge-host-request-{request_id}" - spec_path = f"{prefix}.json" - response_path = f"{prefix}.response" - files = kwargs.pop("files", None) or {} - timeout = kwargs.pop("timeout", 60) - spec = { - "method": method, - "url": f"http://127.0.0.1:8080/{path.lstrip('/')}", - "headers": kwargs.pop("headers", None) or {}, - "json": kwargs.pop("json", None), - "data": kwargs.pop("data", None), - "timeout": timeout, - "files": {}, - } - if kwargs: - raise TypeError( - "Unsupported managed Judge request arguments: " - + ", ".join(sorted(kwargs)) - ) - - raw = self.handle.raw - uploaded_paths: list[str] = [] - try: - for index, (field_name, file_spec) in enumerate(files.items()): - filename, content, content_type = file_spec - remote_path = f"{prefix}.upload-{index}" - raw.files.write( - remote_path, content, user="root", request_timeout=300, - ) - uploaded_paths.append(remote_path) - spec["files"][field_name] = { - "filename": filename, - "path": remote_path, - "content_type": content_type, - } - raw.files.write( - spec_path, json.dumps(spec), user="root", request_timeout=60, - ) - command = ( - "/tmp/sforge-controller-venv/bin/python -c " - + shlex.quote( - "import json,requests,sys; " - "s=json.load(open(sys.argv[1])); opened=[]; files={}; " - "[(opened.append(open(v['path'],'rb')), " - "files.__setitem__(k,(v['filename'],opened[-1],v['content_type']))) " - "for k,v in s['files'].items()]; " - "r=requests.request(s['method'],s['url'],headers=s['headers']," - "json=s['json'],data=s['data'],files=files or None," - "timeout=s['timeout']); " - "open(sys.argv[2],'wb').write(r.content); print(r.status_code)" - ) - + f" {shlex.quote(spec_path)} {shlex.quote(response_path)}" - ) - exec_timeout = ( - int(max(timeout)) if isinstance(timeout, tuple) else int(timeout) - ) + 30 - result = self.backend.exec_run_with_exit_code( - self.handle, command, timeout=exec_timeout, user="root", - ) - if result.timed_out: - raise requests.Timeout( - f"Managed Judge loopback request timed out: {method} {path}" - ) - if result.exit_code != 0: - raise requests.ConnectionError( - "Managed Judge loopback request failed: " - + result.output[-2000:] - ) - try: - status_code = int(result.output.strip().splitlines()[-1]) - except (ValueError, IndexError) as exc: - raise requests.ConnectionError( - "Managed Judge loopback request returned no HTTP status: " - + result.output[-2000:] - ) from exc - content = raw.files.read( - response_path, format="bytes", user="root", - request_timeout=300, - ) - response = requests.Response() - response.status_code = status_code - response._content = content - response.url = spec["url"] - response.encoding = "utf-8" - return response - finally: - targets = [spec_path, response_path, *uploaded_paths] - try: - self.backend.exec_run( - self.handle, - "rm -f -- " + " ".join(map(shlex.quote, targets)), - user="root", - ) - except Exception: - logger.debug( - "Could not clean up managed Judge request files", - exc_info=True, - ) - - def _export_run_logs(self) -> None: - """Merge Judge artifacts into the host run directory before cleanup.""" - if self.handle is None: - return - remote_run = f"/tmp/sforge-controller/logs/runs/{self.run_id}" - remote_archive = "/tmp/sforge-controller-run-logs.tar.gz" - result = self.backend.exec_run_with_exit_code( - self.handle, - f"test -d {shlex.quote(remote_run)} || exit 3; " - f"tar czf {remote_archive} " - "--exclude='submission.tar.gz' --exclude='final_archive.tar.gz' " - f"-C {shlex.quote(remote_run)} .", - timeout=120, - user="root", - ) - if result.timed_out: - raise TimeoutError("Timed out while exporting controller logs") - if result.exit_code == 3: - return - if result.exit_code != 0: - raise RuntimeError(result.output[-4000:]) - wrapped = self.backend.copy_from_container( - self.handle, - PurePosixPath(remote_archive), - ) - with tarfile.open(fileobj=io.BytesIO(wrapped)) as outer: - files = [member for member in outer.getmembers() if member.isfile()] - if len(files) != 1: - raise RuntimeError( - f"Unexpected controller log archive wrapper: {len(files)} files" - ) - source = outer.extractfile(files[0]) - if source is None: - raise RuntimeError("Could not read controller log archive") - archive = source.read() - _merge_log_archive( - archive, - self.config.log_dir / "runs" / self.run_id, - ) - - def __enter__(self) -> JudgeConnection: - return self.start() - - def __exit__(self, exc_type, exc, tb) -> None: - self.close() - - -def _merge_log_archive(archive: bytes, target_dir: Path) -> None: - """Safely merge a controller-created tar.gz into a host run tree.""" - target_root = target_dir.resolve() - total_size = 0 - with tarfile.open(fileobj=io.BytesIO(archive), mode="r:gz") as bundle: - for member in bundle.getmembers(): - target = (target_root / member.name).resolve() - if target != target_root and target_root not in target.parents: - raise ValueError(f"Unsafe controller log path: {member.name!r}") - if member.issym() or member.islnk(): - raise ValueError( - f"Controller log links are not supported: {member.name!r}" - ) - if member.isdir(): - target.mkdir(parents=True, exist_ok=True) - continue - if not member.isfile(): - continue - total_size += member.size - if member.size > 64 * 1024 * 1024 or total_size > 256 * 1024 * 1024: - raise ValueError("Controller log archive exceeds the size limit") - source = bundle.extractfile(member) - if source is None: - continue - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(source.read()) diff --git a/sforge/harness/evolve_scripts.py b/sforge/harness/evolve_scripts.py index bf1fd67..14d8f30 100644 --- a/sforge/harness/evolve_scripts.py +++ b/sforge/harness/evolve_scripts.py @@ -81,7 +81,6 @@ def _generate_submit_script() -> str: JUDGE_URL="${SFORGE_JUDGE_URL}" TOKEN="${SFORGE_TOKEN}" -ACCESS_TOKEN="${SFORGE_JUDGE_ACCESS_TOKEN:-}" PATCH_DIR="${SFORGE_PATCH_DIR:-$(pwd)}" STATE_FILE="/tmp/sforge_state.json" @@ -94,14 +93,8 @@ def _generate_submit_script() -> str: exit 1 fi -CURL_AUTH=() -if [ -n "$ACCESS_TOKEN" ]; then - CURL_AUTH=(-H "e2b-traffic-access-token: $ACCESS_TOKEN") -fi -CURL_SESSION=(-H "Authorization: Bearer $TOKEN") - if [ "$LIST_MODE" -eq 1 ]; then - HIST=$(curl -s -m 30 "${CURL_AUTH[@]}" "${CURL_SESSION[@]}" "$JUDGE_URL/api/v1/history") + HIST=$(curl -s -m 30 "$JUDGE_URL/api/v1/history?token=$TOKEN") if [ -z "$HIST" ]; then echo "ERROR: No response from judge server" >&2 exit 1 @@ -165,12 +158,10 @@ def _generate_submit_script() -> str: # ── Submit + poll ── HTTP_CODE=$(curl -s -o /tmp/_submit_resp.json -w '%{http_code}' -m 120 -X POST "$JUDGE_URL/api/v1/submit" \ - "${CURL_AUTH[@]}" \ - "${CURL_SESSION[@]}" \ + -F "token=$TOKEN" \ -F "archive=@$ARCHIVE_FILE") rm -f "$ARCHIVE_FILE" SUBMIT_RESP=$(cat /tmp/_submit_resp.json) -rm -f /tmp/_submit_resp.json if [ "$HTTP_CODE" = "429" ]; then DETAIL=$(echo "$SUBMIT_RESP" | jq -r '.detail // empty') @@ -204,7 +195,7 @@ def _generate_submit_script() -> str: for _ in $(seq 1 720); do sleep 10 - RESULT=$(curl -s -m 30 "${CURL_AUTH[@]}" "${CURL_SESSION[@]}" "$JUDGE_URL/api/v1/result/$SUBMISSION_ID" 2>/dev/null || true) + RESULT=$(curl -s -m 30 "$JUDGE_URL/api/v1/result/$SUBMISSION_ID" 2>/dev/null || true) STATUS=$(echo "$RESULT" | jq -r '.status // empty') if [ "$STATUS" = "completed" ] || [ "$STATUS" = "error" ]; then break @@ -428,10 +419,7 @@ def generate_game_prompt(original_query: str, internet: bool = True) -> str: "---\n\n" f"{network_note}" "## Game Server HTTP API\n\n" - "The server URL is available in the `GAME_SERVER_URL` environment variable. " - "Every request must include `Authorization: Bearer $SFORGE_TOKEN`; when " - "`SFORGE_JUDGE_ACCESS_TOKEN` is set, also include " - "`e2b-traffic-access-token: $SFORGE_JUDGE_ACCESS_TOKEN`.\n\n" + "The server URL is available in the `GAME_SERVER_URL` environment variable.\n\n" "### Start a new game\n" "```\n" "POST {GAME_SERVER_URL}/new\n" diff --git a/sforge/harness/judge_client.py b/sforge/harness/judge_client.py deleted file mode 100644 index aacd424..0000000 --- a/sforge/harness/judge_client.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""One transport contract for every SForge Judge HTTP request.""" - -from __future__ import annotations - -import logging -import time -from collections.abc import Callable -from dataclasses import dataclass, field - -import requests - -E2B_TRAFFIC_ACCESS_HEADER = "e2b-traffic-access-token" -SFORGE_ADMIN_SECRET_HEADER = "X-SForge-Admin-Secret" - -logger = logging.getLogger(__name__) - -E2B_CONNECT_ATTEMPTS = 3 -E2B_CONNECT_TIMEOUT = 10 - - -@dataclass(frozen=True) -class JudgeConnection: - url: str - headers: dict[str, str] = field(default_factory=dict) - connect_attempts: int = 1 - request_handler: Callable[..., requests.Response] | None = field( - default=None, repr=False, compare=False, - ) - - @classmethod - def from_e2b_traffic_token( - cls, - url: str, - token: str | None, - ) -> JudgeConnection: - """Create a Judge connection through an optional secured E2B gateway.""" - headers = {E2B_TRAFFIC_ACCESS_HEADER: token} if token else {} - return cls( - url=url.rstrip("/"), - headers=headers, - connect_attempts=E2B_CONNECT_ATTEMPTS if token else 1, - ) - - def request(self, method: str, path: str, **kwargs): - headers = {**self.headers, **(kwargs.pop("headers", None) or {})} - if self.request_handler is not None: - return self.request_handler( - method, path, headers=headers, **kwargs, - ) - url = f"{self.url}/{path.lstrip('/')}" - timeout = kwargs.get("timeout") - if self.connect_attempts > 1 and isinstance(timeout, (int, float)): - # A scalar applies to both connect and response reads. Keep the - # caller's response budget, but fail fast enough to retry a flaky - # E2B public-gateway connection. A ConnectTimeout happens before - # the request is sent, so retrying POST is safe here. - kwargs["timeout"] = (min(E2B_CONNECT_TIMEOUT, timeout), timeout) - - for attempt in range(1, self.connect_attempts + 1): - try: - return requests.request( - method, url, headers=headers, **kwargs, - ) - except requests.ConnectTimeout: - if attempt == self.connect_attempts: - raise - logger.warning( - "E2B Judge gateway connect timed out for %s %s " - "(attempt %d/%d); retrying", - method, path, attempt, self.connect_attempts, - ) - time.sleep(attempt) - raise AssertionError("unreachable") - - def agent_env(self) -> dict[str, str]: - env = {"SFORGE_JUDGE_URL": self.url} - token = self.headers.get(E2B_TRAFFIC_ACCESS_HEADER) - if token: - env["SFORGE_JUDGE_ACCESS_TOKEN"] = token - return env - - -def merge_no_proxy(current: str, hostname: str) -> str: - """Append one exact hostname without substring-based false matches.""" - entries = [item.strip() for item in current.split(",") if item.strip()] - if hostname and hostname not in entries: - entries.append(hostname) - return ",".join(entries) diff --git a/sforge/harness/judge_server.py b/sforge/harness/judge_server.py index 6594a2d..e1516fb 100644 --- a/sforge/harness/judge_server.py +++ b/sforge/harness/judge_server.py @@ -28,7 +28,7 @@ from typing import Any import requests -from fastapi import FastAPI, File, Form, Header, HTTPException, UploadFile +from fastapi import FastAPI, File, Form, HTTPException, Query, UploadFile from pydantic import BaseModel from sforge.harness.backend import ContainerBackend @@ -42,7 +42,6 @@ ) from sforge.harness.constants import get_admin_secret from sforge.harness.docker_build import BuildImageError, build_judge_image -from sforge.harness.judge_client import SFORGE_ADMIN_SECRET_HEADER from sforge.harness.run_evaluation import judge_submission from sforge.harness.selection import select_best from sforge.harness.task_spec import TaskSpec, load_all_tasks @@ -125,6 +124,7 @@ class GameCloseResponse(BaseModel): class RegisterRequest(BaseModel): task_id: str run_id: str + admin_secret: str = "" judge_cpu_limit: int | None = None judge_mem_limit: str | None = None backend: str | None = None @@ -341,15 +341,6 @@ def resolve_token(self, token: str) -> dict: raise KeyError("Invalid token") return dict(info) - def require_session( - self, token: str, run_id: str, task_id: str, - ) -> dict: - """Resolve a token and bind it to the route's run/task scope.""" - info = self.resolve_token(token) - if info["run_id"] != run_id or info["task_id"] != task_id: - raise KeyError("Session token does not match run/task") - return info - def consume_round(self, token: str, kind: str = "agent") -> tuple[str, str, str, int | None, str | None, int | None]: """Allocate the next round ID for a token. Returns (task_id, run_id, round_id, cpu, mem, remaining). @@ -866,52 +857,12 @@ def create_app(config: SForgeConfig | None = None) -> FastAPI: state = JudgeState(config) state.load_tasks() - def require_bearer_token(authorization: str | None) -> str: - scheme, _, token = (authorization or "").partition(" ") - if scheme.lower() != "bearer" or not token: - raise HTTPException(status_code=401, detail="Bearer token required") - return token - - def require_game_session( - authorization: str | None, run_id: str, task_id: str, - ) -> str: - token = require_bearer_token(authorization) - try: - state.require_session(token, run_id, task_id) - except KeyError: - raise HTTPException(status_code=401, detail="Invalid session token") - return token - - def require_game_scope( - session_id: str, run_id: str, task_id: str, - ) -> GameSessionState: - with state._game_lock: - sess = state.game_sessions.get(session_id) - if sess is None: - raise HTTPException(404, f"Game session not found: {session_id}") - if sess.run_id != run_id or sess.task_id != task_id: - raise HTTPException(404, f"Game session not found: {session_id}") - return sess - @app.get("/api/v1/result/{submission_id}") - def get_result( - submission_id: str, - authorization: str | None = Header(default=None), - ) -> ResultResponse: + def get_result(submission_id: str) -> ResultResponse: """Get result of an async submission.""" result = state.get_result(submission_id) if result is None: raise HTTPException(status_code=404, detail="Submission not found") - token = require_bearer_token(authorization) - try: - info = state.resolve_token(token) - except KeyError: - raise HTTPException(status_code=401, detail="Invalid token") - if ( - result.get("run_id") != info["run_id"] - or result.get("task_id") != info["task_id"] - ): - raise HTTPException(status_code=404, detail="Submission not found") return ResultResponse( submission_id=submission_id, status=result["status"], @@ -920,14 +871,9 @@ def get_result( ) @app.post("/api/v1/register") - def register( - req: RegisterRequest, - admin_secret: str | None = Header( - default=None, alias=SFORGE_ADMIN_SECRET_HEADER, - ), - ) -> RegisterResponse: + def register(req: RegisterRequest) -> RegisterResponse: """Register a session and get a token for submissions.""" - if admin_secret != state.admin_secret: + if req.admin_secret != state.admin_secret: raise HTTPException(status_code=403, detail="Invalid admin secret") try: token = state.register_session( @@ -950,19 +896,16 @@ def register( @app.post("/api/v1/submit") def submit( + token: str = Form(...), archive: UploadFile = File(...), kind: str = Form("agent"), - authorization: str | None = Header(default=None), - admin_secret: str | None = Header( - default=None, alias=SFORGE_ADMIN_SECRET_HEADER, - ), + admin_secret: str = Form(""), ) -> SubmitResponse: """Submit an archive using a session token. The server resolves task_id, run_id, and assigns a round_id. kind=auto requires a valid admin_secret (host-side only). """ - token = require_bearer_token(authorization) if kind not in ("agent", "auto"): raise HTTPException(status_code=400, detail="kind must be 'agent' or 'auto'") if kind == "auto" and admin_secret != state.admin_secret: @@ -990,18 +933,12 @@ def submit( raise HTTPException(status_code=404, detail=str(e)) @app.get("/api/v1/history") - def history( - authorization: str | None = Header(default=None), - admin_secret: str | None = Header( - default=None, alias=SFORGE_ADMIN_SECRET_HEADER, - ), - ) -> dict: + def history(token: str = Query(...), admin_secret: str = Query("")) -> dict: """Get run history using a session token. Without admin_secret: returns only agent submissions (agent-visible view). With valid admin_secret: returns all entries including auto-eval (host-side view). """ - token = require_bearer_token(authorization) try: info = state.resolve_token(token) except KeyError: @@ -1032,12 +969,8 @@ def history( # --- Game session routes --- @app.post("/api/v1/game/{run_id}/{task_id}/new") - def game_new( - run_id: str, task_id: str, req: GameNewRequest, - authorization: str | None = Header(default=None), - ) -> GameNewResponse: + def game_new(run_id: str, task_id: str, req: GameNewRequest) -> GameNewResponse: """Start a new game session in a dedicated container.""" - require_game_session(authorization, run_id, task_id) try: sess, data = state.create_game_session(run_id, task_id) except ValueError as e: @@ -1055,12 +988,7 @@ def game_new( ) @app.post("/api/v1/game/{run_id}/{task_id}/{session_id}/step") - def game_step( - run_id: str, task_id: str, session_id: str, req: GameStepRequest, - authorization: str | None = Header(default=None), - ) -> GameStepResponse: - require_game_session(authorization, run_id, task_id) - require_game_scope(session_id, run_id, task_id) + def game_step(run_id: str, task_id: str, session_id: str, req: GameStepRequest) -> GameStepResponse: try: data = state.game_step(session_id, req.action) except KeyError: @@ -1078,12 +1006,7 @@ def game_step( ) @app.get("/api/v1/game/{run_id}/{task_id}/{session_id}/status") - def game_status( - run_id: str, task_id: str, session_id: str, - authorization: str | None = Header(default=None), - ) -> GameStatusResponse: - require_game_session(authorization, run_id, task_id) - require_game_scope(session_id, run_id, task_id) + def game_status(run_id: str, task_id: str, session_id: str) -> GameStatusResponse: try: data = state.game_status(session_id) except KeyError: @@ -1098,12 +1021,7 @@ def game_status( ) @app.post("/api/v1/game/{run_id}/{task_id}/{session_id}/close") - def game_close( - run_id: str, task_id: str, session_id: str, - authorization: str | None = Header(default=None), - ) -> GameCloseResponse: - require_game_session(authorization, run_id, task_id) - require_game_scope(session_id, run_id, task_id) + def game_close(run_id: str, task_id: str, session_id: str) -> GameCloseResponse: try: data = state.close_game_session(session_id) except KeyError: @@ -1117,11 +1035,7 @@ def game_close( ) @app.post("/api/v1/game/{run_id}/{task_id}/close-all") - def game_close_all( - run_id: str, task_id: str, - authorization: str | None = Header(default=None), - ) -> dict: - require_game_session(authorization, run_id, task_id) + def game_close_all(run_id: str, task_id: str) -> dict: count = state.close_all_game_sessions(run_id, task_id) return {"closed": count} diff --git a/sforge/harness/run_agent.py b/sforge/harness/run_agent.py index 0e040ef..d4bd2d7 100644 --- a/sforge/harness/run_agent.py +++ b/sforge/harness/run_agent.py @@ -32,7 +32,6 @@ import json import shlex import signal -import sys import tarfile as _tarfile import threading import time @@ -57,11 +56,6 @@ generate_game_prompt, generate_submit_script, ) -from sforge.harness.judge_client import ( - SFORGE_ADMIN_SECRET_HEADER, - JudgeConnection, - merge_no_proxy, -) from sforge.harness.task_spec import TaskSpec @@ -261,15 +255,13 @@ def _auto_eval_loop( backend: ContainerBackend, handle: ContainerHandle, task_spec: TaskSpec, - host_judge: JudgeConnection, + host_judge_url: str, session_token: str, eval_interval: int, stop_event: threading.Event, logger, log_dir: Path, admin_secret: str, - latest_archive: list[bytes], - transfer_cancel: threading.Event, ) -> None: """Host-side auto-eval: periodically extract code and submit to judge. @@ -283,22 +275,19 @@ def _auto_eval_loop( break try: archive = _extract_archive_from_container( - backend, handle, task_spec, shutdown_event=transfer_cancel, + backend, handle, task_spec, shutdown_event=stop_event, ) - latest_archive[:] = [archive] - if stop_event.is_set() or transfer_cancel.is_set(): + if stop_event.is_set(): break - resp = host_judge.request( - "POST", "/api/v1/submit", + resp = requests.post( + f"{host_judge_url}/api/v1/submit", data={ + "token": session_token, "kind": "auto", + "admin_secret": admin_secret, }, files={"archive": ("archive.tar.gz", archive, "application/gzip")}, timeout=120, - headers={ - "Authorization": f"Bearer {session_token}", - SFORGE_ADMIN_SECRET_HEADER: admin_secret, - }, ) resp.raise_for_status() data = resp.json() @@ -315,86 +304,6 @@ def _auto_eval_loop( logger.debug("Auto-eval tick failed: %s", e) -def _stop_auto_eval_thread( - stop_event: threading.Event | None, - thread: threading.Thread | None, - transfer_cancel: threading.Event | None, - logger, - *, - timeout: int = 30, - shutdown_event: threading.Event | None = None, -) -> bool: - if stop_event is None or thread is None: - return True - stop_event.set() - deadline = time.monotonic() + timeout - while thread.is_alive() and time.monotonic() < deadline: - if shutdown_event is not None and shutdown_event.is_set(): - break - thread.join(min(1, max(0, deadline - time.monotonic()))) - if not thread.is_alive(): - return True - logger.warning( - "Auto-eval did not finish within %ds; cancelling its transfer", - timeout, - ) - if transfer_cancel is not None: - transfer_cancel.set() - thread.join(20) - if thread.is_alive(): - logger.warning( - "Auto-eval thread is still stopping; continuing container cleanup" - ) - return False - return True - - -def _drain_pending_evaluations( - host_judge: JudgeConnection, - session_token: str, - admin_secret: str, - logger, - *, - shutdown_event: threading.Event | None = None, - timeout: int = 900, - poll_interval: int = 15, -) -> None: - """Wait for submitted evaluations, while remaining interruptible.""" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - if shutdown_event is not None and shutdown_event.is_set(): - logger.info("Judge drain interrupted; continuing cleanup") - return - try: - history = host_judge.request( - "GET", "/api/v1/history", - timeout=10, - headers={ - "Authorization": f"Bearer {session_token}", - SFORGE_ADMIN_SECRET_HEADER: admin_secret, - }, - ).json() - pending = [ - entry for entry in history.get("entries", []) - if entry.get("status") in ("running", "queued") - ] - if not pending: - return - logger.info(f"Draining {len(pending)} pending judge evals...") - except Exception: - return - wait_seconds = min(poll_interval, max(0, deadline - time.monotonic())) - if shutdown_event is None: - time.sleep(wait_seconds) - elif shutdown_event.wait(wait_seconds): - logger.info("Judge drain interrupted; continuing cleanup") - return - logger.warning( - f"Drain timed out after {timeout / 60:g} min; " - "some reports may be missing." - ) - - # --------------------------------------------------------------------------- # Main entry point # --------------------------------------------------------------------------- @@ -418,7 +327,6 @@ def run_agent( shutdown_event: threading.Event | None = None, max_submissions: int | None = None, submission_cooldown: int | None = None, - host_judge_connection: JudgeConnection | None = None, ) -> RunResult: """Run an agent on a task with iterative evaluation. @@ -440,32 +348,18 @@ def run_agent( verbose=verbose, ) - # judge_url is the Sandbox-facing URL. Docker's host alias is the only - # address that needs a host-side rewrite; E2B gateway URLs are routable - # from both the host and the Sandbox. + # judge_url is for container use; derive a host-local URL for registration/polling parsed_judge = urlparse(judge_url) if parsed_judge.hostname == "host.docker.internal": host_judge_url = judge_url.replace("host.docker.internal", "127.0.0.1") elif backend.backend_name == "k8s": + # judge_url points to VPC IP (for pods); host talks to judge server locally host_judge_url = f"http://127.0.0.1:{parsed_judge.port or 8080}" else: host_judge_url = judge_url - remote_judge = JudgeConnection.from_e2b_traffic_token( - judge_url, config.e2b_judge_access_token, - ) - host_judge = host_judge_connection or ( - JudgeConnection.from_e2b_traffic_token( - host_judge_url, - config.e2b_judge_access_token, - ) - ) handle = None net_isolation = None - auto_eval_stop = None - auto_eval_thread = None - auto_eval_transfer_cancel = None - latest_auto_archive: list[bytes] = [] install_parts: list[str] = [] admin_secret = "" @@ -492,10 +386,7 @@ def run_agent( logger.info("Images ready") # 1b. Register session with judge server - reg_body: dict = { - "task_id": task_spec.task_id, - "run_id": run_id, - } + reg_body: dict = {"task_id": task_spec.task_id, "run_id": run_id, "admin_secret": admin_secret} if config.judge_cpu_limit is not None: reg_body["judge_cpu_limit"] = config.judge_cpu_limit if config.judge_mem_limit is not None: @@ -513,17 +404,14 @@ def run_agent( if config.k8s_kubeconfig: reg_body["k8s_kubeconfig"] = config.k8s_kubeconfig elif backend.backend_name == "e2b": - reg_body.update({ - "backend": "e2b", - "e2b_template_map": config.e2b_template_map, - "e2b_sandbox_ttl": config.e2b_sandbox_ttl, - }) + reg_body["backend"] = "e2b" + reg_body["e2b_template_map"] = config.e2b_template_map + reg_body["e2b_sandbox_ttl"] = config.e2b_sandbox_ttl for _reg_attempt in range(1, 6): try: - reg_resp = host_judge.request( - "POST", "/api/v1/register", + reg_resp = requests.post( + f"{host_judge_url}/api/v1/register", json=reg_body, - headers={SFORGE_ADMIN_SECRET_HEADER: admin_secret}, timeout=10, ) reg_resp.raise_for_status() @@ -545,7 +433,7 @@ def run_agent( env = _build_agent_env(agent, model) if backend.backend_name == "e2b": sanitize_e2b_proxy_env(env, internet=internet) - env.update(remote_judge.agent_env()) + env["SFORGE_JUDGE_URL"] = judge_url env["SFORGE_TOKEN"] = session_token env["SFORGE_PATCH_DIR"] = task_spec.cwd env["SFORGE_SUBMIT_PATHS"] = " ".join(task_spec.submit_paths) @@ -563,7 +451,9 @@ def run_agent( # Ensure judge server URL bypasses proxy judge_host = urlparse(judge_url).hostname or "" for key in ("NO_PROXY", "no_proxy"): - env[key] = merge_no_proxy(env.get(key, ""), judge_host) + existing = env.get(key, "") + if judge_host and judge_host not in existing: + env[key] = f"{existing},{judge_host}" if existing else judge_host # Network isolation: preflight + extra_hosts pre-resolve container_extra_hosts = {"host.docker.internal": "host-gateway"} @@ -661,6 +551,7 @@ def run_agent( # the stop hook — otherwise the agent exits naturally as soon as # the model decides it's "done", losing the full timeout budget. effective_eval_interval = 0 if disable_auto_eval else eval_interval + auto_eval_stop = None if not task_spec.game_mode: _install_tools( backend, @@ -675,15 +566,13 @@ def run_agent( # Start host-side auto-eval thread (if enabled) if effective_eval_interval > 0: auto_eval_stop = threading.Event() - auto_eval_transfer_cancel = threading.Event() auto_eval_thread = threading.Thread( target=_auto_eval_loop, args=( - backend, handle, task_spec, host_judge, + backend, handle, task_spec, host_judge_url, session_token, effective_eval_interval, auto_eval_stop, logger, log_dir, admin_secret, - latest_auto_archive, auto_eval_transfer_cancel, ), daemon=True, ) @@ -806,66 +695,38 @@ def run_agent( # 7. Stop auto-eval thread before extracting final archive if auto_eval_stop is not None: - _stop_auto_eval_thread( - auto_eval_stop, auto_eval_thread, auto_eval_transfer_cancel, - logger, timeout=180, shutdown_event=shutdown_event, - ) + auto_eval_stop.set() logger.info("Auto-eval thread stopped") - # 8. Extract final archive for code tasks. Game tasks have no submit - # paths; their authoritative output is the Judge Server game history. - final_archive = b"" - if not task_spec.game_mode: - try: - interrupted = ( - shutdown_event is not None and shutdown_event.is_set() - ) - if interrupted: - final_archive = ( - latest_auto_archive[-1] if latest_auto_archive else b"" - ) - else: - final_archive = _extract_archive_from_container( - backend, handle, task_spec, - ) - (log_dir / "final_archive.tar.gz").write_bytes(final_archive) - logger.info(f"Final archive: {len(final_archive)} bytes") - except Exception as exc: - if backend.backend_name == "e2b": - raise - # Preserve the pre-E2B Docker/Kubernetes behavior. - logger.warning( - "Failed to extract final archive " - f"(container may have stopped): {exc}" - ) - final_archive = b"" + # 8. Extract final archive (tar of submit_paths) + try: + final_archive = _extract_archive_from_container( + backend, handle, task_spec, shutdown_event=shutdown_event, + ) + (log_dir / "final_archive.tar.gz").write_bytes(final_archive) + logger.info(f"Final archive: {len(final_archive)} bytes") + except Exception as e: + logger.warning(f"Failed to extract final archive (container may have stopped): {e}") + final_archive = b"" # 8. Collect results if task_spec.game_mode: try: - close_response = host_judge.request( - "POST", - f"/api/v1/game/{run_id}/{task_spec.task_id}/close-all", + requests.post( + f"{host_judge_url}/api/v1/game/{run_id}/{task_spec.task_id}/close-all", timeout=30, - headers={"Authorization": f"Bearer {session_token}"}, ) - close_response.raise_for_status() except Exception: - # Keep the pre-E2B best-effort semantics. The managed - # controller still exports any game artifacts during cleanup. logger.warning("Failed to close active game sessions") try: - history_resp = host_judge.request( - "GET", "/api/v1/history", + history_resp = requests.get( + f"{host_judge_url}/api/v1/history?token={session_token}", timeout=10, - headers={"Authorization": f"Bearer {session_token}"}, ) history_resp.raise_for_status() history = history_resp.json() except Exception: - # Docker and Kubernetes historically completed the run with - # an empty result when the Judge history was unavailable. logger.warning("Failed to fetch run history from judge server") history = {"run_id": run_id, "best_score": None, "entries": []} @@ -929,27 +790,34 @@ def run_agent( pass # Drain pending judge evaluations before querying final results - _drain_pending_evaluations( - host_judge, session_token, admin_secret, logger, - shutdown_event=shutdown_event, - ) + drain_deadline = time.time() + 900 + while time.time() < drain_deadline: + try: + h = requests.get( + f"{host_judge_url}/api/v1/history", + params={"token": session_token, "admin_secret": admin_secret}, + timeout=10, + ).json() + pending = [e for e in h.get("entries", []) if e.get("status") in ("running", "queued")] + if not pending: + break + logger.info(f"Draining {len(pending)} pending judge evals...") + except Exception: + break + time.sleep(15) + else: + logger.warning("Drain timed out after 15 min; some reports may be missing.") # Query Judge Server for authoritative results (with admin_secret to get full history) try: - history_resp = host_judge.request( - "GET", "/api/v1/history", + history_resp = requests.get( + f"{host_judge_url}/api/v1/history", + params={"token": session_token, "admin_secret": admin_secret}, timeout=10, - headers={ - "Authorization": f"Bearer {session_token}", - SFORGE_ADMIN_SECRET_HEADER: admin_secret, - }, ) history_resp.raise_for_status() history = history_resp.json() except Exception: - # Preserve the original Docker/Kubernetes behavior: a - # transient history failure must not turn an otherwise - # completed agent run into a task-level infrastructure error. logger.warning("Failed to fetch run history from judge server") history = { "run_id": run_id, "best_score": None, @@ -999,22 +867,14 @@ def run_agent( # Stop auto-eval thread if auto_eval_stop is not None: - _stop_auto_eval_thread( - auto_eval_stop, auto_eval_thread, auto_eval_transfer_cancel, - logger, shutdown_event=shutdown_event, - ) + auto_eval_stop.set() # Try to extract archive from the container before it's destroyed interrupted_archive = b"" try: - if handle is not None and not task_spec.game_mode: - interrupted_archive = ( - latest_auto_archive[-1] - if latest_auto_archive - else _extract_archive_from_container( - backend, handle, task_spec, - shutdown_event=shutdown_event, - ) + if handle is not None: + interrupted_archive = _extract_archive_from_container( + backend, handle, task_spec, shutdown_event=shutdown_event, ) (log_dir / "final_archive.tar.gz").write_bytes(interrupted_archive) logger.info(f"Final archive (interrupted): {len(interrupted_archive)} bytes") @@ -1023,13 +883,10 @@ def run_agent( # Query judge server for results accumulated before interruption (full history) try: - history_resp = host_judge.request( - "GET", "/api/v1/history", + history_resp = requests.get( + f"{host_judge_url}/api/v1/history", + params={"token": session_token, "admin_secret": admin_secret}, timeout=10, - headers={ - "Authorization": f"Bearer {session_token}", - SFORGE_ADMIN_SECRET_HEADER: admin_secret, - }, ) history_resp.raise_for_status() history = history_resp.json() @@ -1080,18 +937,15 @@ def run_agent( # Preserve the pre-E2B Docker/Kubernetes behavior. try: - history_resp = host_judge.request( - "GET", "/api/v1/history", + history_resp = requests.get( + f"{host_judge_url}/api/v1/history?token={session_token}", timeout=10, - headers={"Authorization": f"Bearer {session_token}"}, ) history_resp.raise_for_status() history = history_resp.json() best_pass_rate = history.get("best_pass_rate", 0.0) best_score_raw = history.get("best_score") - best_score = ( - float(best_score_raw) if best_score_raw is not None else None - ) + best_score = float(best_score_raw) if best_score_raw is not None else None best_round = history.get("best_round", "") agent_subs = history.get("agent_submissions", 0) auto_subs = history.get("auto_submissions", 0) @@ -1114,35 +968,19 @@ def run_agent( resume_count=locals().get("resume_count", 0), ) finally: - primary_error = sys.exc_info()[1] if auto_eval_stop is not None: auto_eval_stop.set() - if auto_eval_transfer_cancel is not None: - auto_eval_transfer_cancel.set() if net_isolation is not None: try: net_isolation.cleanup() except Exception as exc: if logger: logger.warning(f"Failed to cleanup network isolation: {exc}") - main_thread = threading.current_thread() is threading.main_thread() - previous_handler = None - if main_thread: - previous_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) + if threading.current_thread() is threading.main_thread(): + prev_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) print("\nStopping container, please wait... (Ctrl+C disabled during cleanup)") - cleanup_error = None - try: backend.cleanup_container(handle, logger) - except Exception as exc: - cleanup_error = exc - if primary_error is not None: - logger.exception( - "Work container cleanup failed while preserving the " - "original run error" - ) - finally: - if previous_handler is not None: - signal.signal(signal.SIGINT, previous_handler) - close_logger(logger) - if cleanup_error is not None and primary_error is None: - raise cleanup_error + signal.signal(signal.SIGINT, prev_handler) + else: + backend.cleanup_container(handle, logger) + close_logger(logger) From 941f89416dd6e12cabbd9d392a8111a98f0e2da1 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 25 Aug 2026 14:17:16 -0400 Subject: [PATCH 07/17] Restore main's dockerfile apt-mirror handling and image hash recipe --- sforge/harness/dockerfiles.py | 51 +++++++++++++++-------------------- sforge/harness/task_spec.py | 14 +++------- 2 files changed, 24 insertions(+), 41 deletions(-) diff --git a/sforge/harness/dockerfiles.py b/sforge/harness/dockerfiles.py index c19add4..4d8fe41 100644 --- a/sforge/harness/dockerfiles.py +++ b/sforge/harness/dockerfiles.py @@ -16,6 +16,9 @@ from __future__ import annotations +from __future__ import annotations + + # --- Base Dockerfile: official image + common tools --- _DOCKERFILE_BASE = """\ FROM --platform={platform} {official_image} @@ -23,10 +26,9 @@ ARG DEBIAN_FRONTEND=noninteractive ENV TZ=Etc/UTC {env_directives} -RUN {apt_mirror_prepare}apt-get update && apt-get install -y --no-install-recommends \\ +{apt_mirror_directive}RUN apt-get update && apt-get install -y --no-install-recommends \\ {packages} sudo \\ && rm -rf /var/lib/apt/lists/* \\ -{apt_mirror_restore}\ && useradd -m -s /bin/bash agent \\ && echo 'agent ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers \\ && git config --global safe.directory '*' @@ -66,34 +68,24 @@ def get_dockerfile_base( user_directive = spec.get("user_directive", "") post_install_directive = spec.get("post_install_directive", "") - apt_mirror_prepare = "" - apt_mirror_restore = "" + apt_mirror_directive = "" if apt_mirror_url: url = apt_mirror_url.rstrip("/") - apt_mirror_prepare = ( - "rm -rf /tmp/sforge-apt-original && " - "mkdir -p /tmp/sforge-apt-original && " - "if [ -e /etc/apt/sources.list ]; then " - "cp -a /etc/apt/sources.list /tmp/sforge-apt-original/; fi && " - "if [ -d /etc/apt/sources.list.d ]; then " - "cp -a /etc/apt/sources.list.d /tmp/sforge-apt-original/; fi && " - "find /etc/apt -maxdepth 2 -type f " - "\\( -name '*.list' -o -name '*.sources' \\) " - "-exec sed -Ei " - f"-e 's#https?://deb.debian.org#{url}#g' " - f"-e 's#https?://archive.ubuntu.com#{url}#g' " - f"-e 's#https?://security.ubuntu.com#{url}#g' " - "{} + && " - ) - apt_mirror_restore = ( - " && rm -f /etc/apt/sources.list \\\n" - " && rm -rf /etc/apt/sources.list.d \\\n" - " && if [ -e /tmp/sforge-apt-original/sources.list ]; then " - "mv /tmp/sforge-apt-original/sources.list /etc/apt/; fi \\\n" - " && if [ -d /tmp/sforge-apt-original/sources.list.d ]; then " - "mv /tmp/sforge-apt-original/sources.list.d /etc/apt/; " - "else mkdir -p /etc/apt/sources.list.d; fi \\\n" - " && rm -rf /tmp/sforge-apt-original \\\n" + apt_mirror_directive = ( + f"RUN sed -i 's|http://deb.debian.org|{url}|g; " + f"s|https://deb.debian.org|{url}|g; " + f"s|http://archive.ubuntu.com|{url}|g; " + f"s|http://security.ubuntu.com|{url}|g' " + f"/etc/apt/sources.list 2>/dev/null; " + f"sed -i 's|http://deb.debian.org|{url}|g; " + f"s|https://deb.debian.org|{url}|g; " + f"s|http://archive.ubuntu.com|{url}|g; " + f"s|http://security.ubuntu.com|{url}|g' " + f"/etc/apt/sources.list.d/*.list 2>/dev/null; " + f"sed -i 's|http://deb.debian.org|{url}|g; " + f"s|https://deb.debian.org|{url}|g' " + f"/etc/apt/sources.list.d/*.sources 2>/dev/null; " + f"true\n" ) pip_packages = spec.get("pip_packages", []) @@ -107,8 +99,7 @@ def get_dockerfile_base( official_image=spec["official_image"], user_directive=user_directive, env_directives=env_directives, - apt_mirror_prepare=apt_mirror_prepare, - apt_mirror_restore=apt_mirror_restore, + apt_mirror_directive=apt_mirror_directive, packages=packages, pip_directive=pip_directive, post_install_directive=post_install_directive, diff --git a/sforge/harness/task_spec.py b/sforge/harness/task_spec.py index ff3ceaf..a69234d 100644 --- a/sforge/harness/task_spec.py +++ b/sforge/harness/task_spec.py @@ -21,14 +21,12 @@ from sforge.harness.benchmark import BenchmarkMeta from sforge.harness.constants import ( - END_TEST_OUTPUT, - START_TEST_OUTPUT, SForgeTask, + START_TEST_OUTPUT, + END_TEST_OUTPUT, ) from sforge.harness.score_rescale import RescaleSpec, parse_rescale_spec -IMAGE_RECIPE_VERSION = 2 - @dataclass class WorkSpec: @@ -56,11 +54,7 @@ class JudgeSpec: def _compute_base_image_hash(base_image_key: str, base_image_spec: dict) -> str: - data = json.dumps({ - "recipe_version": IMAGE_RECIPE_VERSION, - "key": base_image_key, - "spec": base_image_spec, - }, sort_keys=True) + data = json.dumps({"key": base_image_key, "spec": base_image_spec}, sort_keys=True) return hashlib.sha256(data.encode()).hexdigest() @@ -106,7 +100,6 @@ def work_image_hash(self) -> str: f"Task '{self.task_id}': cannot compute work image hash without setup_cmds" ) data = json.dumps({ - "recipe_version": IMAGE_RECIPE_VERSION, "base_hash": self.base_image_hash, "platform": self.platform, "cwd": self.cwd, @@ -121,7 +114,6 @@ def judge_image_hash(self) -> str: f"Task '{self.task_id}': cannot compute judge image hash without setup_cmds" ) data = json.dumps({ - "recipe_version": IMAGE_RECIPE_VERSION, "base_hash": self.base_image_hash, "platform": self.platform, "cwd": self.cwd, From 9c124e135769fa23001224c90ec148796a490f36 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 25 Aug 2026 14:19:52 -0400 Subject: [PATCH 08/17] Trim non-essential deviations from main in cli, constants, and .gitignore --- .gitignore | 3 --- sforge/cli.py | 44 +++++++++++++++---------------------- sforge/harness/constants.py | 3 ++- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index d50df51..7336555 100644 --- a/.gitignore +++ b/.gitignore @@ -20,9 +20,6 @@ experiments/ # Benchmark task definitions downloaded by sforge fetch-tasks /tasks/ -# Local review tests (the upstream release tree does not ship a tests directory) -/tests/ - # IDE .idea/ .vscode/ diff --git a/sforge/cli.py b/sforge/cli.py index 1f713f7..3cd48fc 100644 --- a/sforge/cli.py +++ b/sforge/cli.py @@ -42,7 +42,6 @@ from sforge.harness.task_spec import TaskSpec, load_all_tasks, make_task_spec - def _resolve_task(args, config: SForgeConfig) -> TaskSpec: """Resolve a single task. Fails if the user passed more than one ID.""" specs = _resolve_tasks(args, config) @@ -327,15 +326,13 @@ def _run_single_task( print(f" Game sessions: {result.total_rounds}") if result.best_score is not None: print(f" Best score: {result.best_score:.0f}") - print(f" Game history: {run_log_dir / 'game_history.json'}") else: print(f" Total rounds: {result.total_rounds}") print(f" Best pass rate: {result.best_pass_rate:.2%}") if result.best_score is not None: print(f" Best score: {result.best_score:.0f}") print(f" Best round: {result.best_round}") - if not task_spec.game_mode: - print(f" Final archive: {run_log_dir / 'final_archive.tar.gz'}") + print(f" Final archive: {run_log_dir / 'final_archive.tar.gz'}") combined = { "agent": agent.name, @@ -565,6 +562,23 @@ def cmd_run(args): ) task_runs.append((ts, task_config, task_args)) + cfg_dict = _effective_config_dict(ts, task_args, task_config) + unified_tasks[ts.task_id] = cfg_dict + + task_log_dir = run_root / ts.task_id + task_log_dir.mkdir(parents=True, exist_ok=True) + (task_log_dir / "run_config.json").write_text( + json.dumps(cfg_dict, indent=2, ensure_ascii=False) + ) + + unified = { + "run_id": run_id, + "experiment": args.experiment or None, + "stagger": args.stagger or (experiment.stagger if experiment else None), + "tasks": unified_tasks, + } + (run_root / "run_config.json").write_text(json.dumps(unified, indent=2, ensure_ascii=False)) + backend_names = {task_config.backend for _, task_config, _ in task_runs} if len(backend_names) != 1: raise ValueError( @@ -629,26 +643,6 @@ def _invoke(ts: TaskSpec, task_config: SForgeConfig, task_args): summaries: list[dict] = [] try: - for ts, task_config, task_args in task_runs: - cfg_dict = _effective_config_dict(ts, task_args, task_config) - unified_tasks[ts.task_id] = cfg_dict - task_log_dir = run_root / ts.task_id - task_log_dir.mkdir(parents=True, exist_ok=True) - (task_log_dir / "run_config.json").write_text( - json.dumps(cfg_dict, indent=2, ensure_ascii=False) - ) - unified = { - "run_id": run_id, - "experiment": args.experiment or None, - "stagger": args.stagger or ( - experiment.stagger if experiment else None - ), - "tasks": unified_tasks, - } - (run_root / "run_config.json").write_text( - json.dumps(unified, indent=2, ensure_ascii=False) - ) - if multi: with concurrent.futures.ThreadPoolExecutor(max_workers=len(task_runs)) as ex: futures = [] @@ -703,8 +697,6 @@ def _invoke(ts: TaskSpec, task_config: SForgeConfig, task_args): mark = "-" print(f" {tid:<30} {mark}") print(f"\nSummary saved: {summary_path}") - if any("error" in summary for summary in summaries): - raise SystemExit(1) def cmd_eval(args): diff --git a/sforge/harness/constants.py b/sforge/harness/constants.py index a151735..77a55bc 100644 --- a/sforge/harness/constants.py +++ b/sforge/harness/constants.py @@ -14,7 +14,6 @@ from __future__ import annotations -import fcntl import os import secrets import stat @@ -54,6 +53,8 @@ def get_admin_secret(log_dir: Path | None = None) -> str: flags = os.O_RDWR | os.O_CREAT if hasattr(os, "O_NOFOLLOW"): flags |= os.O_NOFOLLOW + import fcntl # POSIX-only; imported lazily so Windows can still import the package + fd = os.open(path, flags, 0o600) with os.fdopen(fd, "r+") as secret_file: fcntl.flock(secret_file.fileno(), fcntl.LOCK_EX) From d135301a8497f8041699f0aa48f689e47e73ff76 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 25 Aug 2026 14:20:15 -0400 Subject: [PATCH 09/17] List submission archives with tar --quoting-style=literal so CJK filenames survive the submit_paths match --- sforge/harness/run_evaluation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sforge/harness/run_evaluation.py b/sforge/harness/run_evaluation.py index be7ad9d..2625c86 100644 --- a/sforge/harness/run_evaluation.py +++ b/sforge/harness/run_evaluation.py @@ -187,7 +187,10 @@ def judge_submission( # List archive contents list_result = backend.exec_run( handle, - f"tar tzf {tar_path}", + # --quoting-style=literal: GNU tar octal-escapes non-ASCII + # names in listings under some locales, which breaks the + # submit_paths match for CJK filenames + f"tar --quoting-style=literal -tzf {tar_path}", workdir=task_spec.cwd, user=DOCKER_USER, ) From 1762391661c78498e479f4023c8e209909d0e6a4 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Sat, 18 Jul 2026 12:45:35 -0400 Subject: [PATCH 10/17] Add claude-code-2.1.214 agent variant for Claude 5-family models --- sforge/harness/agent/claude_code.py | 12 ++++++++++++ sforge/harness/agent/factory.py | 5 ++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/sforge/harness/agent/claude_code.py b/sforge/harness/agent/claude_code.py index 166c16c..1b0cd93 100644 --- a/sforge/harness/agent/claude_code.py +++ b/sforge/harness/agent/claude_code.py @@ -120,6 +120,18 @@ def install_stop_hook( logger.info("Configured Claude Code settings with hooks") +class ClaudeCode2_1_214Agent(ClaudeCodeAgent): + """Claude Code pinned to 2.1.214 — required by Claude 5-family models + (Fable/Mythos); the 2.1.159 pin predates them. Kept as a separate agent + so existing claude-code results stay comparable.""" + + name = "claude-code-2.1.214" + install_cmds = [ + ClaudeCodeAgent.install_cmds[0], + "sudo -E npm install -g @anthropic-ai/claude-code@2.1.214", + ] + + # --------------------------------------------------------------------------- # Script generators (private to this module) # --------------------------------------------------------------------------- diff --git a/sforge/harness/agent/factory.py b/sforge/harness/agent/factory.py index 712178c..e0f06ca 100644 --- a/sforge/harness/agent/factory.py +++ b/sforge/harness/agent/factory.py @@ -19,8 +19,9 @@ from sforge.harness.agent.base import Agent from sforge.harness.config import SForgeConfig -from sforge.harness.agent.claude_code import ClaudeCodeAgent +from sforge.harness.agent.claude_code import ClaudeCode2_1_214Agent, ClaudeCodeAgent from sforge.harness.agent.codex import CodexAgent +from sforge.harness.agent.opencode import OpenCodeAgent # --------------------------------------------------------------------------- # Registry: name → agent class @@ -28,7 +29,9 @@ _REGISTRY: dict[str, type[Agent]] = { "claude-code": ClaudeCodeAgent, + "claude-code-2.1.214": ClaudeCode2_1_214Agent, "codex": CodexAgent, + "opencode": OpenCodeAgent, } From a918b542e159cf6da93b1ee29eb2c57c96a83d4a Mon Sep 17 00:00:00 2001 From: qinshengling Date: Mon, 20 Jul 2026 06:33:44 -0400 Subject: [PATCH 11/17] Add opencode agent with visualizer support --- docs/en/guide/agents.md | 11 + docs/zh/guide/agents.md | 6 + sforge/cli.py | 2 +- sforge/harness/agent/opencode.py | 136 +++++++++++++ sforge/visualizer/parsers/agent_output.py | 20 +- sforge/visualizer/parsers/opencode_output.py | 201 +++++++++++++++++++ sforge/visualizer/scanner.py | 4 + 7 files changed, 374 insertions(+), 6 deletions(-) create mode 100644 sforge/harness/agent/opencode.py create mode 100644 sforge/visualizer/parsers/opencode_output.py diff --git a/docs/en/guide/agents.md b/docs/en/guide/agents.md index 2fc0acd..ebb085e 100644 --- a/docs/en/guide/agents.md +++ b/docs/en/guide/agents.md @@ -8,14 +8,23 @@ SForge manages different Code Agents through a plugin-style agent registry. To r |-------|----------|-------------|-----------|---------------|-----------|-------------| | Claude Code | `claude-code` | `ANTHROPIC_AUTH_TOKEN` | `ANTHROPIC_MODEL` | -- | Yes | Yes | | Codex | `codex` | `CODEX_API_KEY` | `CODEX_MODEL` | -- | Yes | Yes | +| opencode | `opencode` | `ANTHROPIC_API_KEY` | -- | -- | No (auto-resume only) | Yes | Use the `--agent` flag to select an Agent: ```bash sforge run --task ad_placement_optimization --agent claude-code sforge run --task ad_placement_optimization --agent codex +sforge run --task ad_placement_optimization --agent opencode ``` +> **opencode notes**: models are addressed as `provider/model` (a bare model +> name is treated as `anthropic/`). The default model, custom base URL +> (`SFORGE_AGENT_API_BASE_URL`), and permissions are injected via the +> `OPENCODE_CONFIG_CONTENT` environment variable — no config file is written +> into the container. opencode has no Stop-hook mechanism; long runs rely on +> the harness auto-resume loop (`opencode run --continue`). + ## Agent Configuration ### API Key @@ -83,6 +92,7 @@ When an Agent such as Claude Code decides the task is complete and tries to exit |-------|-----------|-------------| | `claude-code` | Claude Code Stop Hook | Registered via `.claude/settings.json` | | `codex` | Codex Stop Hook | Registered via `/etc/codex/hooks.json` | +| `opencode` | -- | No hook mechanism; early exits are covered by auto-resume | ### Disabling the Stop Hook @@ -110,6 +120,7 @@ Auto-resume handles **abnormal agent exits** (API disconnects, transient errors, |-------|------------------| | `claude-code` | `claude --continue -p "Continue working."` | | `codex` | `codex exec resume --last "Continue working."` | +| `opencode` | `opencode run --continue "Continue working."` | ### Safety Guards diff --git a/docs/zh/guide/agents.md b/docs/zh/guide/agents.md index a42f69c..cc74587 100644 --- a/docs/zh/guide/agents.md +++ b/docs/zh/guide/agents.md @@ -10,14 +10,18 @@ SForge 内置了 2 种 Agent： |-----------|----------|-----------------|-------------|---------|-----------|-------------| | Claude Code | `claude-code` | `ANTHROPIC_AUTH_TOKEN` | `ANTHROPIC_MODEL` | — | 支持 | 支持 | | Codex | `codex` | `CODEX_API_KEY` | `CODEX_MODEL` | — | 支持 | 支持 | +| opencode | `opencode` | `ANTHROPIC_API_KEY` | — | — | 不支持（靠自动恢复） | 支持 | 使用 `--agent` 参数指定要运行的 Agent： ```bash sforge run --task ad_placement_optimization --agent claude-code sforge run --task ad_placement_optimization --agent codex +sforge run --task ad_placement_optimization --agent opencode ``` +> **opencode 说明**：模型名格式为 `provider/model`（裸模型名会视为 `anthropic/`）。默认模型、自定义 base URL（`SFORGE_AGENT_API_BASE_URL`）和权限均通过 `OPENCODE_CONFIG_CONTENT` 环境变量注入，容器内不写配置文件。opencode 无 Stop Hook 机制，长时运行依赖自动恢复循环（`opencode run --continue`）。 + ## Agent 配置 ### API Key @@ -85,6 +89,7 @@ Stop Hook 是 SForge 的一个重要机制，用于阻止 Agent 提前退出。 |-------|----------|------| | `claude-code` | Claude Code Stop Hook | 通过 `.claude/settings.json` 注册 | | `codex` | Codex Stop Hook | 通过 `/etc/codex/hooks.json` 注册 | +| `opencode` | — | 无 hook 机制，提前退出由自动恢复兜底 | ### 禁用 Stop Hook @@ -112,6 +117,7 @@ Auto-Resume 处理的是 **Agent 异常退出** 的情况（如 API 断连、瞬 |-------|---------| | `claude-code` | `claude --continue -p "Continue working."` | | `codex` | `codex exec resume --last "Continue working."` | +| `opencode` | `opencode run --continue "Continue working."` | ### 安全保护 diff --git a/sforge/cli.py b/sforge/cli.py index 3cd48fc..0fccf46 100644 --- a/sforge/cli.py +++ b/sforge/cli.py @@ -967,7 +967,7 @@ def main(): "Multiple tasks are run fully in parallel.") p_run.add_argument("--experiment", default=None, help="Path to experiment YAML config file (model config + per-task overrides)") - p_run.add_argument("--agent", default=None, help="Agent name (claude-code, aider, codex)") + p_run.add_argument("--agent", default=None, help="Agent name (claude-code, codex, opencode)") p_run.add_argument("--model", default=None, help="Model override") p_run.add_argument("--timeout", type=int, default=None, help="Agent timeout in seconds") p_run.add_argument("--eval-interval", type=int, default=None, help=f"Auto-eval interval in seconds (default {DEFAULT_EVAL_INTERVAL})") diff --git a/sforge/harness/agent/opencode.py b/sforge/harness/agent/opencode.py new file mode 100644 index 0000000..3879315 --- /dev/null +++ b/sforge/harness/agent/opencode.py @@ -0,0 +1,136 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""opencode agent — env-based config, model flag, permission handling. + +opencode (npm ``opencode-ai``) differs from Claude Code in two ways that +matter to the harness: + +* All configuration (default model, provider base URL, permissions) is + passed via the ``OPENCODE_CONFIG_CONTENT`` env var instead of CLI flags + or config files, so no container-side config install step is needed. +* There is no Stop-hook mechanism, so early exits are handled entirely by + the harness auto-resume loop (``opencode run --continue``). +""" + +from __future__ import annotations + +import json +import shlex + +from sforge.harness.agent.base import Agent + +# Permission set used when the task runs without internet: web tools are +# denied outright so the model doesn't waste turns on blocked requests. +# ``deny`` rules are enforced even under ``--auto``. +_NO_INTERNET_PERMISSION = json.dumps( + {"edit": "allow", "bash": "allow", "webfetch": "deny", "websearch": "deny"}, + separators=(",", ":"), +) + + +class OpenCodeAgent(Agent): + + name = "opencode" + install_cmds = [ + "sudo -E bash -c 'NODE_MIRROR=${SFORGE_NODEJS_MIRROR_URL:-https://nodejs.org/dist} && curl -fsSL $NODE_MIRROR/v20.18.0/node-v20.18.0-linux-x64.tar.xz | tar -xJ -C /usr/local --strip-components=1'", + "sudo -E npm install -g opencode-ai@1.18.2", + ] + run_cmd = 'opencode run --format json --auto "$(cat {prompt_file})"' + resume_cmd = 'opencode run --continue --format json --auto "Continue working."' + api_key_env = "ANTHROPIC_API_KEY" + # opencode reads no base-URL env var; the base URL is embedded into + # OPENCODE_CONFIG_CONTENT in augment_env(). default_api_base_url is + # still needed for the network-isolation allowlist. + default_api_base_url = "https://api.anthropic.com" + + def augment_env(self, env: dict[str, str], model: str | None) -> None: + config = self._config + qualified = _qualify_model( + model or config.agent_model or self.default_model + ) + + cfg: dict = { + "permission": { + "edit": "allow", + "bash": "allow", + "webfetch": "allow", + }, + } + if qualified: + # small_model too, or title generation falls back to an + # Anthropic haiku model that third-party endpoints reject. + cfg["model"] = qualified + cfg["small_model"] = qualified + + provider_id, _, model_id = (qualified or "anthropic/").partition("/") + provider_cfg: dict = {} + if config.agent_api_base_url: + provider_cfg["options"] = { + "baseURL": _sdk_base_url(config.agent_api_base_url), + } + if model_id: + # Declare the model so opencode accepts ids unknown to models.dev. + provider_cfg["models"] = {model_id: {}} + if provider_cfg: + cfg["provider"] = {provider_id: provider_cfg} + + env.setdefault("OPENCODE_CONFIG_CONTENT", json.dumps(cfg)) + env.setdefault("OPENCODE_DISABLE_AUTOUPDATE", "1") + env.setdefault("OPENCODE_DISABLE_LSP_DOWNLOAD", "1") + + def format_run_cmd( + self, + prompt_path: str, + *, + model: str | None = None, + cwd: str = "", + internet: bool = True, + resume: bool = False, + ) -> str: + cmd = super().format_run_cmd( + prompt_path, model=model, cwd=cwd, internet=internet, resume=resume, + ) + + qualified = _qualify_model( + model or self._config.agent_model or self.default_model + ) + if qualified: + cmd += f" --model {shlex.quote(qualified)}" + + if not internet: + # OPENCODE_PERMISSION overrides the config permissions; + # OPENCODE_DISABLE_MODELS_FETCH stops the models.dev refresh + # from stalling behind the network isolation. + cmd = ( + f"OPENCODE_PERMISSION={shlex.quote(_NO_INTERNET_PERMISSION)}" + " OPENCODE_DISABLE_MODELS_FETCH=1 " + cmd + ) + + return cmd + + +def _qualify_model(model: str | None) -> str | None: + """opencode expects ``provider/model``; default bare names to anthropic.""" + if not model: + return None + return model if "/" in model else f"anthropic/{model}" + + +def _sdk_base_url(base_url: str) -> str: + """SForge base URLs follow the Claude Code convention (no ``/v1``), + while the AI SDK expects the versioned root it appends ``/messages`` to. + """ + base = base_url.rstrip("/") + return base if base.endswith("/v1") else f"{base}/v1" diff --git a/sforge/visualizer/parsers/agent_output.py b/sforge/visualizer/parsers/agent_output.py index 29bd7ca..568401f 100644 --- a/sforge/visualizer/parsers/agent_output.py +++ b/sforge/visualizer/parsers/agent_output.py @@ -63,14 +63,19 @@ def summary(self) -> str: cmd = str(inp.get("command", "")) return cmd.splitlines()[0][:160] if cmd else "" if self.name in ("Read", "NotebookEdit"): - return str(inp.get("file_path") or inp.get("notebook_path", "")) + return str( + inp.get("file_path") + or inp.get("filePath") # opencode uses camelCase + or inp.get("notebook_path", "") + ) if self.name == "Edit": - fp = inp.get("file_path", "") - old = len(str(inp.get("old_string", ""))) - new = len(str(inp.get("new_string", ""))) + fp = inp.get("file_path") or inp.get("filePath", "") + old = len(str(inp.get("old_string") or inp.get("oldString", ""))) + new = len(str(inp.get("new_string") or inp.get("newString", ""))) return f"{fp} ({old}→{new} chars)" if self.name == "Write": - return f"{inp.get('file_path', '')} ({len(str(inp.get('content', '')))} chars)" + fp = inp.get("file_path") or inp.get("filePath", "") + return f"{fp} ({len(str(inp.get('content', '')))} chars)" if self.name == "Glob": return str(inp.get("pattern", "")) if self.name == "Grep": @@ -348,6 +353,7 @@ def get_trajectory(path: Path) -> Optional[Trajectory]: Dispatches to the appropriate parser based on the file's first bytes: - Claude Code: stream-json (first line is JSON starting with `{`) + - opencode: JSONL whose events carry a top-level camelCase "sessionID" - OpenAI Codex: plain text starting with "OpenAI Codex" or "Reading additional input from stdin..." """ @@ -372,6 +378,10 @@ def get_trajectory(path: Path) -> Optional[Trajectory]: if "OpenAI Codex" in head or head.lstrip().startswith("Reading additional input"): from sforge.visualizer.parsers.codex_output import parse as _codex_parse traj = _codex_parse(path) + elif head.lstrip().startswith("{") and '"sessionID"' in head: + # opencode events use camelCase sessionID; Claude Code uses session_id. + from sforge.visualizer.parsers.opencode_output import parse as _opencode_parse + traj = _opencode_parse(path) else: traj = parse(path) diff --git a/sforge/visualizer/parsers/opencode_output.py b/sforge/visualizer/parsers/opencode_output.py new file mode 100644 index 0000000..698dce6 --- /dev/null +++ b/sforge/visualizer/parsers/opencode_output.py @@ -0,0 +1,201 @@ +# Copyright (c) 2026 ByteDance Ltd. and/or its affiliates +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Parse opencode ``run --format json`` output into the shared Trajectory model. + +opencode format (one JSON event per line; every event carries a top-level +``sessionID`` and a ``part`` object): + + {"type":"step_start","sessionID":..,"part":{"messageID":..,"snapshot":..}} + {"type":"text", "part":{"id":..,"messageID":..,"text":..,"time":..}} + {"type":"tool_use", "part":{"tool":"bash","callID":.., + "state":{"status","input","output",..}}} + {"type":"step_finish","part":{"reason":"stop"|"tool-calls", + "tokens":{..},"cost":..}} + {"type":"error", "error":{"name":..,"data":{"message":..}}} + +Each assistant step (grouped by ``part.messageID``) becomes one Exchange. +Tool results arrive inline in the ``tool_use`` event's ``state``, so no +cross-event folding is needed. The trailing ``step_finish`` may be missing +when the process exits early — the parser tolerates truncated files. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from sforge.visualizer.parsers.agent_output import ( + Exchange, + ToolCall, + Trajectory, +) + +# opencode tool names are lowercase; map them onto the canonical names the +# shared ToolCall.summary property understands. +_TOOL_NAMES = { + "bash": "Bash", + "read": "Read", + "write": "Write", + "edit": "Edit", + "patch": "Patch", + "glob": "Glob", + "grep": "Grep", + "list": "List", + "task": "Task", + "todowrite": "TodoWrite", + "todoread": "TodoRead", + "webfetch": "WebFetch", + "websearch": "WebSearch", +} + + +def parse(path: Path) -> Trajectory: + traj = Trajectory() + if not path.is_file(): + return traj + + exchanges: list[Exchange] = [] + current: Exchange | None = None + current_message_id: str | None = None + # part.id → position, so re-emitted parts update in place. + text_parts: dict[str, int] = {} + tool_parts: dict[str, ToolCall] = {} + tokens_in = tokens_out = cache_read = cache_write = 0 + cost = 0.0 + + def start_exchange(message_id: str | None) -> Exchange: + nonlocal current, current_message_id + ex = Exchange(idx=len(exchanges), role="assistant") + exchanges.append(ex) + current = ex + current_message_id = message_id + text_parts.clear() + return ex + + def ensure_exchange(message_id: str | None) -> Exchange: + if current is None or ( + message_id and message_id != current_message_id + ): + return start_exchange(message_id) + return current + + with path.open("r", errors="replace") as fh: + for line in fh: + line = line.strip() + if not line or not line.startswith("{"): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + traj.total_events += 1 + + etype = event.get("type", "") + part = event.get("part") or {} + message_id = part.get("messageID") + + if not traj.meta.get("session_id") and event.get("sessionID"): + traj.meta["session_id"] = event["sessionID"] + + if etype == "step_start": + start_exchange(message_id) + + elif etype == "text": + ex = ensure_exchange(message_id) + text = str(part.get("text", "")) + part_id = part.get("id") + blocks = ex.text.split("\n\n") if ex.text else [] + if part_id and part_id in text_parts: + blocks[text_parts[part_id]] = text + else: + if part_id: + text_parts[part_id] = len(blocks) + blocks.append(text) + ex.text = "\n\n".join(blocks) + + elif etype == "tool_use": + ex = ensure_exchange(message_id) + state = part.get("state") or {} + call_id = str(part.get("callID") or part.get("id") or "") + status = state.get("status", "") + raw_name = str(part.get("tool", "")) + tool = tool_parts.get(call_id) + if tool is None: + tool = ToolCall( + id=call_id or f"opencode-tool-{traj.total_events}", + name=_TOOL_NAMES.get(raw_name, raw_name or "tool"), + ) + tool_parts[call_id] = tool + ex.tool_calls.append(tool) + inp = state.get("input") + if isinstance(inp, dict): + tool.input = inp + if status == "error": + tool.result_content = str( + state.get("error") or state.get("output") or "" + ) + tool.result_is_error = True + tool.has_result = True + elif status == "completed": + tool.result_content = str(state.get("output", "")) + tool.has_result = True + + elif etype == "step_finish": + ex = ensure_exchange(message_id) + ex.stop_reason = str(part.get("reason", "")) + tokens = part.get("tokens") or {} + tokens_in += int(tokens.get("input", 0) or 0) + tokens_out += int(tokens.get("output", 0) or 0) + cache = tokens.get("cache") or {} + cache_read += int(cache.get("read", 0) or 0) + cache_write += int(cache.get("write", 0) or 0) + cost += float(part.get("cost", 0) or 0) + + elif etype == "error": + err = event.get("error") or {} + data = err.get("data") or {} + msg = str(data.get("message") or err.get("name") or "error") + ex = Exchange( + idx=len(exchanges), role="system", text=f"Error: {msg}", + ) + exchanges.append(ex) + current = None + current_message_id = None + + traj.meta.update( + { + "agent_cli": "opencode", + "tokens_input": tokens_in, + "tokens_output": tokens_out, + "cache_read": cache_read, + "cache_write": cache_write, + } + ) + if cost: + traj.meta["cost"] = round(cost, 4) + + # Drop empty exchanges left by step_start events with no content. + exchanges = [ + ex for ex in exchanges + if ex.text or ex.thinking or ex.tool_calls or ex.stop_reason + ] + for i, ex in enumerate(exchanges): + ex.idx = i + + traj.exchanges = exchanges + traj.tool_use_count = sum(len(ex.tool_calls) for ex in exchanges) + return traj diff --git a/sforge/visualizer/scanner.py b/sforge/visualizer/scanner.py index 00f5592..4099c9d 100644 --- a/sforge/visualizer/scanner.py +++ b/sforge/visualizer/scanner.py @@ -382,6 +382,8 @@ def _peek_agent(task_dir: Path) -> str: return "claude-code" if "Running agent: codex " in head or "Running agent: codex\n" in head: return "codex" + if "Running agent: opencode " in head or "Running agent: opencode\n" in head: + return "opencode" if "Running agent: aider " in head or "Running agent: aider\n" in head: return "aider" # New harness format: "Agent supervisor attempt N: claude -p ..." @@ -393,6 +395,8 @@ def _peek_agent(task_dir: Path) -> str: return "claude-code" if after_colon.startswith("codex ") or after_colon.startswith("codex\n"): return "codex" + if after_colon.startswith("opencode ") or after_colon.startswith("opencode\n"): + return "opencode" if after_colon.startswith("aider ") or after_colon.startswith("aider\n"): return "aider" return "" From 9b85edace7077825801bd99b22d83c6dd086718b Mon Sep 17 00:00:00 2001 From: qinshengling Date: Mon, 20 Jul 2026 06:38:09 -0400 Subject: [PATCH 12/17] Add unified --effort flag translated per-agent --- sforge/cli.py | 16 ++++++++++++++++ sforge/harness/agent/claude_code.py | 3 +++ sforge/harness/agent/codex.py | 8 ++++++++ sforge/harness/agent/opencode.py | 12 +++++++++++- sforge/harness/config.py | 7 +++++++ sforge/harness/experiment.py | 6 +++++- 6 files changed, 50 insertions(+), 2 deletions(-) diff --git a/sforge/cli.py b/sforge/cli.py index 0fccf46..32ed547 100644 --- a/sforge/cli.py +++ b/sforge/cli.py @@ -251,6 +251,10 @@ def _run_single_task( from sforge.harness.agent import create_agent from sforge.harness.run_agent import run_agent + # CLI --effort wins over experiment/env values already on the config. + if getattr(args, "effort", None): + config.agent_effort = args.effort + agent = create_agent(args.agent, config) effective_timeout = args.timeout or config.agent_timeout or agent.timeout disable_stop_hook = getattr(args, "disable_stop_hook", False) @@ -272,6 +276,8 @@ def _run_single_task( print(f" Timeout: {effective_timeout}s") if args.model or config.agent_model or agent.default_model: print(f" Model: {args.model or config.agent_model or agent.default_model}") + if config.agent_effort: + print(f" Effort: {config.agent_effort}") if not task_spec.game_mode: eval_status = f"{effective_eval_interval}s" if not disable_auto_eval and effective_eval_interval > 0 else "disabled" print(f" Auto-eval: {eval_status}") @@ -339,6 +345,7 @@ def _run_single_task( "task": task_spec.task_id, "run_id": run_id, "model": args.model or config.agent_model or agent.default_model, + "effort": config.agent_effort, **result.to_dict(), } (run_log_dir / "final_result.json").write_text( @@ -407,6 +414,9 @@ def _apply_experiment_overrides( if task_args.model is None and merged.model is not None: task_args.model = merged.model + if getattr(task_args, "effort", None) is None and merged.effort is not None: + task_config.agent_effort = merged.effort + if task_args.timeout is None and merged.timeout is not None: task_args.timeout = merged.timeout @@ -480,6 +490,7 @@ def _effective_config_dict( "task_id": task_spec.task_id, "agent": agent_name, "model": model, + "effort": getattr(args, "effort", None) or config.agent_effort, "timeout": timeout, "eval_interval": eval_interval, "disable_stop_hook": getattr(args, "disable_stop_hook", False), @@ -969,6 +980,11 @@ def main(): help="Path to experiment YAML config file (model config + per-task overrides)") p_run.add_argument("--agent", default=None, help="Agent name (claude-code, codex, opencode)") p_run.add_argument("--model", default=None, help="Model override") + p_run.add_argument("--effort", default=None, + choices=["low", "medium", "high", "max"], + help="Reasoning effort; each agent translates to its native " + "mechanism (claude-code env, codex/opencode config). " + "Default: agent's own default") p_run.add_argument("--timeout", type=int, default=None, help="Agent timeout in seconds") p_run.add_argument("--eval-interval", type=int, default=None, help=f"Auto-eval interval in seconds (default {DEFAULT_EVAL_INTERVAL})") p_run.add_argument("--disable-auto-eval", action="store_true", default=False, diff --git a/sforge/harness/agent/claude_code.py b/sforge/harness/agent/claude_code.py index 1b0cd93..4bb50a8 100644 --- a/sforge/harness/agent/claude_code.py +++ b/sforge/harness/agent/claude_code.py @@ -55,6 +55,9 @@ def augment_env(self, env: dict[str, str], model: str | None) -> None: env["CLAUDE_CODE_ATTRIBUTION_HEADER"] = "0" if not self._config.agent_api_base_url and self._config.agent_api_key: env["ANTHROPIC_API_KEY"] = self._config.agent_api_key + # setdefault so an explicit extra_env value stays authoritative. + if self._config.agent_effort: + env.setdefault("CLAUDE_CODE_EFFORT_LEVEL", self._config.agent_effort) def format_run_cmd( self, diff --git a/sforge/harness/agent/codex.py b/sforge/harness/agent/codex.py index 2ae7d67..0ea9d28 100644 --- a/sforge/harness/agent/codex.py +++ b/sforge/harness/agent/codex.py @@ -70,6 +70,14 @@ def format_run_cmd( prompt_path, model=model, cwd=cwd, internet=internet, resume=resume, ) + effort = self._config.agent_effort + if effort: + # codex vocabulary tops out at "xhigh" where ours says "max". + effort = {"max": "xhigh"}.get(effort, effort) + cmd = cmd.replace( + "codex exec", f'codex exec -c model_reasoning_effort="{effort}"', 1, + ) + if not internet: cmd = cmd.replace( "codex exec", 'codex exec -c web_search="disabled"', 1, diff --git a/sforge/harness/agent/opencode.py b/sforge/harness/agent/opencode.py index 3879315..a9b64ac 100644 --- a/sforge/harness/agent/opencode.py +++ b/sforge/harness/agent/opencode.py @@ -82,7 +82,17 @@ def augment_env(self, env: dict[str, str], model: str | None) -> None: } if model_id: # Declare the model so opencode accepts ids unknown to models.dev. - provider_cfg["models"] = {model_id: {}} + model_cfg: dict = {} + # Anthropic-style providers have no reasoningEffort, only a + # thinking budgetTokens knob; the effort->tokens mapping is a + # policy decision we haven't made, so effort applies to + # OpenAI-style providers only for now. + effort = config.agent_effort + if effort and provider_id != "anthropic": + model_cfg["options"] = { + "reasoningEffort": {"max": "xhigh"}.get(effort, effort), + } + provider_cfg["models"] = {model_id: model_cfg} if provider_cfg: cfg["provider"] = {provider_id: provider_cfg} diff --git a/sforge/harness/config.py b/sforge/harness/config.py index 42648fc..c3d6173 100644 --- a/sforge/harness/config.py +++ b/sforge/harness/config.py @@ -42,6 +42,12 @@ class SForgeConfig: agent_api_key: str | None = None agent_api_base_url: str | None = None agent_model: str | None = None + # Reasoning effort, canonical vocabulary "low"/"medium"/"high"/"max"; + # each agent translates to its native mechanism (env var, CLI flag or + # config option) and maps aliases (e.g. max -> xhigh). None means the + # agent's own default — existing runs were recorded under that behavior, + # so never default this to an explicit level. + agent_effort: str | None = None agent_timeout: int | None = None agent_extra_env: dict[str, str] = field(default_factory=dict) @@ -105,6 +111,7 @@ def load_config(cli_overrides: dict | None = None) -> SForgeConfig: "agent_api_key": ["SFORGE_AGENT_API_KEY"], "agent_api_base_url": ["SFORGE_AGENT_API_BASE_URL"], "agent_model": ["SFORGE_AGENT_MODEL"], + "agent_effort": ["SFORGE_AGENT_EFFORT"], "agent_timeout": ["SFORGE_AGENT_TIMEOUT"], "git_user": ["SFORGE_GIT_USER"], "git_token": ["SFORGE_GIT_TOKEN"], diff --git a/sforge/harness/experiment.py b/sforge/harness/experiment.py index 8a5de22..de56cde 100644 --- a/sforge/harness/experiment.py +++ b/sforge/harness/experiment.py @@ -63,6 +63,7 @@ class ModelConfig: class TaskOverrides: agent: str | None = None model: str | None = None + effort: str | None = None timeout: int | None = None eval_interval: int | None = None disable_stop_hook: bool | None = None @@ -126,7 +127,7 @@ def _parse_task_overrides(data: dict | None) -> TaskOverrides: if not data: return TaskOverrides() ovr = TaskOverrides() - for key in ("agent", "model"): + for key in ("agent", "model", "effort"): val = data.get(key) if val is not None: setattr(ovr, key, str(val)) @@ -217,6 +218,9 @@ def resolve_task_overrides( merged = TaskOverrides( agent=task_ovr.agent if task_ovr.agent is not None else defaults.agent, model=task_ovr.model if task_ovr.model is not None else defaults.model, + effort=( + task_ovr.effort if task_ovr.effort is not None else defaults.effort + ), timeout=( task_ovr.timeout if task_ovr.timeout is not None else defaults.timeout ), From 37d3af03c5c2568dd9fb662724cbe10c4ef1a055 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Wed, 19 Aug 2026 23:01:32 +0800 Subject: [PATCH 13/17] Add codex-0.145.0 pinned agent --- sforge/harness/agent/codex.py | 12 ++++++++++++ sforge/harness/agent/factory.py | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/sforge/harness/agent/codex.py b/sforge/harness/agent/codex.py index 0ea9d28..dc21008 100644 --- a/sforge/harness/agent/codex.py +++ b/sforge/harness/agent/codex.py @@ -152,6 +152,18 @@ def install_stop_hook( logger.info("Configured Codex hooks") +class Codex0_145_0Agent(CodexAgent): + """Codex pinned to 0.145.0. Kept as a separate agent so existing + codex (0.130.0) results stay comparable.""" + + name = "codex-0.145.0" + install_cmds = [ + CodexAgent.install_cmds[0], + "sudo -E npm install -g @openai/codex@0.145.0", + CodexAgent.install_cmds[2], + ] + + # --------------------------------------------------------------------------- # Script generators (private to this module) # --------------------------------------------------------------------------- diff --git a/sforge/harness/agent/factory.py b/sforge/harness/agent/factory.py index e0f06ca..7c63fe4 100644 --- a/sforge/harness/agent/factory.py +++ b/sforge/harness/agent/factory.py @@ -20,7 +20,7 @@ from sforge.harness.config import SForgeConfig from sforge.harness.agent.claude_code import ClaudeCode2_1_214Agent, ClaudeCodeAgent -from sforge.harness.agent.codex import CodexAgent +from sforge.harness.agent.codex import Codex0_145_0Agent, CodexAgent from sforge.harness.agent.opencode import OpenCodeAgent # --------------------------------------------------------------------------- @@ -31,6 +31,7 @@ "claude-code": ClaudeCodeAgent, "claude-code-2.1.214": ClaudeCode2_1_214Agent, "codex": CodexAgent, + "codex-0.145.0": Codex0_145_0Agent, "opencode": OpenCodeAgent, } From c43a72c5bfaf7b8e4da5a5714d0e38ca54a45e39 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 25 Aug 2026 14:42:57 -0400 Subject: [PATCH 14/17] opencode: chown agent HOME after root npm install so runtime logs are writable --- sforge/harness/agent/opencode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sforge/harness/agent/opencode.py b/sforge/harness/agent/opencode.py index a9b64ac..52fa38c 100644 --- a/sforge/harness/agent/opencode.py +++ b/sforge/harness/agent/opencode.py @@ -46,6 +46,9 @@ class OpenCodeAgent(Agent): install_cmds = [ "sudo -E bash -c 'NODE_MIRROR=${SFORGE_NODEJS_MIRROR_URL:-https://nodejs.org/dist} && curl -fsSL $NODE_MIRROR/v20.18.0/node-v20.18.0-linux-x64.tar.xz | tar -xJ -C /usr/local --strip-components=1'", "sudo -E npm install -g opencode-ai@1.18.2", + # npm's postinstall runs as root with HOME=/home/agent and leaves + # root-owned ~/.local etc. that opencode then cannot write to + "sudo chown -R agent:agent /home/agent", ] run_cmd = 'opencode run --format json --auto "$(cat {prompt_file})"' resume_cmd = 'opencode run --continue --format json --auto "Continue working."' From 5d8ade2df30f0471b35362e3e9fa057f59329553 Mon Sep 17 00:00:00 2001 From: Tao Wang Date: Thu, 27 Aug 2026 15:02:38 +0800 Subject: [PATCH 15/17] feat(e2b): add SFORGE_E2B_TEMPLATE_NAMESPACE for public template resolution --- docs/en/configuration/container-backends.md | 13 +++-- .../en/configuration/environment-variables.md | 1 + docs/zh/configuration/container-backends.md | 13 +++-- .../zh/configuration/environment-variables.md | 1 + pyproject.toml | 2 +- sforge/cli.py | 2 + sforge/harness/backend/e2b_backend.py | 58 +++++++++++++++++-- sforge/harness/backend/factory.py | 2 + sforge/harness/config.py | 3 + sforge/harness/judge_server.py | 33 ++++++++++- sforge/harness/run_agent.py | 5 +- 11 files changed, 112 insertions(+), 21 deletions(-) diff --git a/docs/en/configuration/container-backends.md b/docs/en/configuration/container-backends.md index f2cabf2..7ee4c46 100644 --- a/docs/en/configuration/container-backends.md +++ b/docs/en/configuration/container-backends.md @@ -236,10 +236,11 @@ A Docker image and an E2B Template are different runtime objects: E2B Sandboxes start from Templates, not from Docker image references. Official E2B Templates for the published EdgeBench tasks are pre-built and published by the SForge maintainers, so no template preparation step is required. -`sforge run --backend e2b` derives each task image's Template reference -automatically (`edgebench.work.foo_bar:abc123` -> -`edgebench-work-foo-bar:abc123`). Set `SFORGE_E2B_TEMPLATE_MAP` only to -override this mapping, for example to run self-built Templates for a +Set `SFORGE_E2B_TEMPLATE_NAMESPACE` to the namespace that publishes the +Templates. `sforge run --backend e2b` then derives each task image's public +Template reference automatically (`edgebench.work.foo_bar:abc123` -> +`edgebench/edgebench-work-foo-bar:abc123`). Set `SFORGE_E2B_TEMPLATE_MAP` +only to override this mapping, for example to run self-built Templates for a modified task. ### Run a Task @@ -250,6 +251,7 @@ inside E2B Sandboxes submit to it over the internet): ```bash uv sync --extra e2b export E2B_API_KEY=... +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench export SFORGE_ADMIN_SECRET=... # same value on the serve and run hosts sforge serve --host 0.0.0.0 --port 8080 @@ -259,6 +261,7 @@ Then run the agent, pointing `--judge-url` at that host: ```bash export E2B_API_KEY=... +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench export SFORGE_ADMIN_SECRET=... # same value as on the serve host export SFORGE_AGENT_API_KEY=... export SFORGE_AGENT_API_BASE_URL=... @@ -328,7 +331,7 @@ tasks remove proxy variables entirely. See [Network Isolation](/en/features/netw | Symptom | Likely cause | Fix | | --- | --- | --- | -| Template not found | The task image version has no published official Template, or the task uses a modified image | Verify the task uses a published image version, or point `SFORGE_E2B_TEMPLATE_MAP` at a self-built Template | +| Template not found | The Template namespace is missing or incorrect, the task image version has no published official Template, or the task uses a modified image | Check `SFORGE_E2B_TEMPLATE_NAMESPACE` and the task image version, or point `SFORGE_E2B_TEMPLATE_MAP` at a self-built Template | | Agent-side Judge requests intermittently time out | The Work Sandbox could not reach the Judge Server over the internet | Check that the Judge host and port are publicly reachable, then retry the agent submission | | A run ends at the configured task timeout with `Sandbox not found` | The team-level maximum Sandbox lifetime was reached before final extraction and cleanup | Shorten the agent timeout, reduce multi-wave batch duration, or use a plan with a longer Sandbox lifetime | | A run leaves a Sandbox | Cleanup failed or the runner was terminated abruptly | Inspect the run log and wait for the configured E2B TTL as the final safeguard | diff --git a/docs/en/configuration/environment-variables.md b/docs/en/configuration/environment-variables.md index 3083648..21d829a 100644 --- a/docs/en/configuration/environment-variables.md +++ b/docs/en/configuration/environment-variables.md @@ -92,6 +92,7 @@ Judge CPU/memory limits are currently set through CLI flags (`--judge-cpu-limit` | `SFORGE_K8S_KUBECONFIG` | --- | Path to kubeconfig file | | `SFORGE_K8S_NODE_SELECTOR` | --- | Node selector for K8s pods, format: `"key1=val1,key2=val2"` | | `SFORGE_E2B_TEMPLATE_MAP` | --- | Explicit E2B image-key to template mapping | +| `SFORGE_E2B_TEMPLATE_NAMESPACE` | --- | Namespace for published E2B Templates (for example, `edgebench`) | | `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL in seconds | ## Path Variables diff --git a/docs/zh/configuration/container-backends.md b/docs/zh/configuration/container-backends.md index 6d95b48..80a1b39 100644 --- a/docs/zh/configuration/container-backends.md +++ b/docs/zh/configuration/container-backends.md @@ -232,10 +232,11 @@ E2B 后端使用 E2B Sandbox 运行 Work、Judge 和 Game 环境。Judge Server Docker 镜像和 E2B Template 是两种不同的运行对象：E2B Sandbox 从 Template 启动， 不能直接传入 Docker image 引用。已发布 EdgeBench 任务的官方 E2B Template 由 SForge 维护者预先构建并发布，用户不需要执行任何 Template 准备步骤。 -`sforge run --backend e2b` 会自动推导每个任务镜像对应的 Template 引用 -（`edgebench.work.foo_bar:abc123` -> `edgebench-work-foo-bar:abc123`）。只有需要 -覆盖该映射时（例如为修改过的任务运行自建 Template）才设置 -`SFORGE_E2B_TEMPLATE_MAP`。 +设置 `SFORGE_E2B_TEMPLATE_NAMESPACE` 指定发布 Template 的 namespace。 +`sforge run --backend e2b` 随后会自动推导每个任务镜像对应的公开 Template 引用 +（`edgebench.work.foo_bar:abc123` -> +`edgebench/edgebench-work-foo-bar:abc123`）。只有需要覆盖该映射时 +（例如为修改过的任务运行自建 Template）才设置 `SFORGE_E2B_TEMPLATE_MAP`。 ### 运行任务 @@ -244,6 +245,7 @@ SForge 维护者预先构建并发布，用户不需要执行任何 Template 准 ```bash uv sync --extra e2b export E2B_API_KEY=... +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench export SFORGE_ADMIN_SECRET=... # serve 和 run 两台主机需使用同一个值 sforge serve --host 0.0.0.0 --port 8080 @@ -253,6 +255,7 @@ sforge serve --host 0.0.0.0 --port 8080 ```bash export E2B_API_KEY=... +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench export SFORGE_ADMIN_SECRET=... # 与 serve 主机一致 export SFORGE_AGENT_API_KEY=... export SFORGE_AGENT_API_BASE_URL=... @@ -312,7 +315,7 @@ auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快 | 现象 | 可能原因 | 处理方式 | | --- | --- | --- | -| 找不到 Template | 该任务镜像版本没有已发布的官方 Template，或任务使用了修改过的镜像 | 确认任务使用已发布的镜像版本，或用 `SFORGE_E2B_TEMPLATE_MAP` 指向自建 Template | +| 找不到 Template | Template namespace 缺失或错误、该任务镜像版本没有已发布的官方 Template，或任务使用了修改过的镜像 | 检查 `SFORGE_E2B_TEMPLATE_NAMESPACE` 和任务镜像版本，或用 `SFORGE_E2B_TEMPLATE_MAP` 指向自建 Template | | Agent 侧 Judge 请求偶发超时 | Work Sandbox 无法通过公网访问 Judge Server | 确认 Judge 主机和端口公网可达后重试提交 | | task timeout 到达时出现 `Sandbox not found` | 在最终归档和清理前达到了 team 级 Sandbox 最大生命周期 | 缩短 Agent timeout、减少多波次批次总时长，或使用支持更长 Sandbox 生命周期的套餐 | | 运行结束后仍有 Sandbox | cleanup 失败或 Runner 被强制终止 | 检查 run 日志；平台最终按配置的 E2B TTL 回收 | diff --git a/docs/zh/configuration/environment-variables.md b/docs/zh/configuration/environment-variables.md index 38edd27..370d02d 100644 --- a/docs/zh/configuration/environment-variables.md +++ b/docs/zh/configuration/environment-variables.md @@ -91,6 +91,7 @@ Judge CPU/内存限制目前通过 CLI 参数（`--judge-cpu-limit`、`--judge-m | `SFORGE_K8S_KUBECONFIG` | --- | kubeconfig 文件路径 | | `SFORGE_K8S_NODE_SELECTOR` | --- | K8s Pod 的节点选择器，格式：`"key1=val1,key2=val2"` | | `SFORGE_E2B_TEMPLATE_MAP` | --- | E2B image key 到 Template 的显式映射 | +| `SFORGE_E2B_TEMPLATE_NAMESPACE` | --- | 已发布 E2B Template 的 namespace（例如 `edgebench`） | | `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL（秒） | ## 路径变量 diff --git a/pyproject.toml b/pyproject.toml index cbfbd82..43dd387 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ ] [project.optional-dependencies] -e2b = ["e2b>=2.42.0"] +e2b = ["e2b>=2.46.0,<3"] [project.scripts] sforge = "sforge.cli:main" diff --git a/sforge/cli.py b/sforge/cli.py index 32ed547..1713933 100644 --- a/sforge/cli.py +++ b/sforge/cli.py @@ -519,6 +519,8 @@ def _effective_config_dict( ) for key, value in config.agent_extra_env.items() } + if config.e2b_template_namespace: + d["e2b_template_namespace"] = config.e2b_template_namespace return d diff --git a/sforge/harness/backend/e2b_backend.py b/sforge/harness/backend/e2b_backend.py index 3bfbb7c..cdd6eb2 100644 --- a/sforge/harness/backend/e2b_backend.py +++ b/sforge/harness/backend/e2b_backend.py @@ -18,7 +18,8 @@ - endpoint/credentials via the standard e2b env vars (``E2B_API_KEY``, ``E2B_API_URL`` / ``E2B_DOMAIN``); -- image-key -> template mapping via ``SFORGE_E2B_TEMPLATE_MAP`` +- public Template namespace via ``SFORGE_E2B_TEMPLATE_NAMESPACE`` and optional + image-key overrides via ``SFORGE_E2B_TEMPLATE_MAP`` (see :meth:`E2BBackend.resolve_template`). Semantics notes vs the docker backend: @@ -41,6 +42,7 @@ import json import logging import os +import re import shlex import tarfile import threading @@ -82,6 +84,7 @@ SANDBOX_CREATE_ATTEMPTS = 3 _METADATA_NAME_KEY = "sforge_name" +_E2B_NAME_PATTERN = re.compile(r"[a-z0-9_-]{1,128}") class E2BSandboxHandle(ContainerHandle): @@ -167,6 +170,7 @@ def __init__( template_map: dict[str, str] | None = None, sandbox_ttl: int | None = None, *, + template_namespace: str | None = None, sandbox_cls=None, template_cls=None, ) -> None: @@ -188,6 +192,19 @@ def __init__( self._template_map = ( dict(template_map) if template_map else template_map_from_env() ) + self._template_namespace = ( + template_namespace + if template_namespace is not None + else os.environ.get("SFORGE_E2B_TEMPLATE_NAMESPACE", "") + ).strip() or None + if ( + self._template_namespace is not None + and not _E2B_NAME_PATTERN.fullmatch(self._template_namespace) + ): + raise ValueError( + "E2B Template namespace must contain only lowercase letters, " + "numbers, dashes, and underscores, and be at most 128 characters" + ) self._sandbox_ttl = ( int(os.environ.get("SFORGE_E2B_SANDBOX_TTL", DEFAULT_SANDBOX_TTL)) if sandbox_ttl is None @@ -212,16 +229,17 @@ def resolve_template(self, image_key: str) -> str: Explicit entries in ``SFORGE_E2B_TEMPLATE_MAP`` win (matched with and without the image tag); otherwise the repo part is normalized to the - template-alias charset and the image tag carries over as the template - tag (``edgebench.work.foo_bar:abc123`` -> ``edgebench-work-foo-bar:abc123``). - Templates built from SForge images must be tagged with the image tag. + template-name charset, the image tag carries over as the Template tag, + and ``SFORGE_E2B_TEMPLATE_NAMESPACE`` is prepended when configured + (``edgebench.work.foo_bar:abc123`` -> + ``edgebench/edgebench-work-foo-bar:abc123``). """ if image_key in self._template_map: return self._template_map[image_key] repo, sep, tag = image_key.partition(":") if repo in self._template_map: return self._template_map[repo] + sep + tag - return repo.lower().replace(".", "-").replace("_", "-") + sep + tag + return e2b_template_reference(image_key, self._template_namespace) # --- Lifecycle --- @@ -423,6 +441,13 @@ def image_exists(self, image_key: str) -> bool: if self._template_cls is None: return False template = self.resolve_template(image_key) + if _is_namespaced_template_reference(template): + logger.debug( + "E2B cannot preflight public Template %r across namespaces; " + "deferring validation to Sandbox.create()", + template, + ) + return True try: # Check the complete name:tag alias. Sandbox.create remains the # authoritative launchability check performed later in the flow. @@ -940,6 +965,29 @@ def _as_shell(cmd: str | list[str]) -> str: return shlex.join(cmd) if isinstance(cmd, list) else cmd +def e2b_template_reference(image_key: str, namespace: str | None = None) -> str: + repo, separator, tag = image_key.partition(":") + name = repo.lower().replace(".", "-").replace("_", "-") + if not _E2B_NAME_PATTERN.fullmatch(name): + raise ValueError( + f"E2B Template name derived from {image_key!r} is invalid: {name!r}" + ) + if namespace: + if not _E2B_NAME_PATTERN.fullmatch(namespace): + raise ValueError( + "E2B Template namespace must contain only lowercase letters, " + "numbers, dashes, and underscores, and be at most 128 characters" + ) + name = f"{namespace}/{name}" + return name + separator + tag + + +def _is_namespaced_template_reference(template: str) -> bool: + name, _, _tag = template.partition(":") + parts = name.split("/") + return len(parts) == 2 and all(_E2B_NAME_PATTERN.fullmatch(part) for part in parts) + + def _exit_code_of(exc: Exception) -> int | None: """Exit code carried by a CommandExitException, duck-typed so fakes and provider variants work.""" diff --git a/sforge/harness/backend/factory.py b/sforge/harness/backend/factory.py index 9db876d..7bac954 100644 --- a/sforge/harness/backend/factory.py +++ b/sforge/harness/backend/factory.py @@ -28,6 +28,7 @@ def create_backend( k8s_image_registry: str = "", k8s_kubeconfig: str | None = None, e2b_template_map: dict[str, str] | None = None, + e2b_template_namespace: str | None = None, e2b_sandbox_ttl: int | None = None, ) -> ContainerBackend: if name == "docker": @@ -48,6 +49,7 @@ def create_backend( return E2BBackend( template_map=e2b_template_map, + template_namespace=e2b_template_namespace, sandbox_ttl=e2b_sandbox_ttl, ) supported = "docker, k8s, e2b" diff --git a/sforge/harness/config.py b/sforge/harness/config.py index c3d6173..7056f87 100644 --- a/sforge/harness/config.py +++ b/sforge/harness/config.py @@ -87,6 +87,7 @@ class SForgeConfig: # e2b backend settings (endpoint/credentials come from the standard # E2B_API_KEY / E2B_API_URL / E2B_DOMAIN env vars read by the e2b SDK) e2b_template_map: dict[str, str] = field(default_factory=dict) + e2b_template_namespace: str | None = None e2b_sandbox_ttl: int | None = None # Paths log_dir: Path = field(default_factory=lambda: LOG_DIR) @@ -120,6 +121,7 @@ def load_config(cli_overrides: dict | None = None) -> SForgeConfig: "k8s_namespace": ["SFORGE_K8S_NAMESPACE"], "k8s_image_registry": ["SFORGE_K8S_IMAGE_REGISTRY"], "k8s_kubeconfig": ["SFORGE_K8S_KUBECONFIG"], + "e2b_template_namespace": ["SFORGE_E2B_TEMPLATE_NAMESPACE"], "work_cpu_limit": ["SFORGE_WORK_CPU_LIMIT"], "work_mem_limit": ["SFORGE_WORK_MEM_LIMIT"], "log_dir": ["SFORGE_LOG_DIR"], @@ -340,5 +342,6 @@ def create_backend_from_config(config: SForgeConfig, docker_client=None): k8s_image_registry=config.k8s_image_registry, k8s_kubeconfig=config.k8s_kubeconfig, e2b_template_map=config.e2b_template_map or None, + e2b_template_namespace=config.e2b_template_namespace, e2b_sandbox_ttl=config.e2b_sandbox_ttl, ) diff --git a/sforge/harness/judge_server.py b/sforge/harness/judge_server.py index e1516fb..ad09e91 100644 --- a/sforge/harness/judge_server.py +++ b/sforge/harness/judge_server.py @@ -135,6 +135,7 @@ class RegisterRequest(BaseModel): max_agent_submissions: int | None = None submission_cooldown: int | None = None e2b_template_map: dict[str, str] | None = None + e2b_template_namespace: str | None = None e2b_sandbox_ttl: int | None = None @@ -287,6 +288,7 @@ def register_session(self, task_id: str, run_id: str, max_agent_submissions: int | None = None, submission_cooldown: int | None = None, e2b_template_map: dict[str, str] | None = None, + e2b_template_namespace: str | None = None, e2b_sandbox_ttl: int | None = None) -> str: if task_id not in self.tasks: raise ValueError(f"Unknown task: {task_id}") @@ -308,7 +310,30 @@ def register_session(self, task_id: str, run_id: str, "judge_cpu_limit": judge_cpu_limit, "judge_mem_limit": judge_mem_limit, } - if backend and backend != self.backend.backend_name: + effective_e2b_template_map = ( + self.config.e2b_template_map + if not e2b_template_map + else e2b_template_map + ) + effective_e2b_template_namespace = ( + self.config.e2b_template_namespace + if not e2b_template_namespace + else e2b_template_namespace + ) + effective_e2b_sandbox_ttl = ( + self.config.e2b_sandbox_ttl + if e2b_sandbox_ttl is None + else e2b_sandbox_ttl + ) + e2b_config_changed = backend == "e2b" and ( + effective_e2b_template_map != self.config.e2b_template_map + or effective_e2b_template_namespace + != self.config.e2b_template_namespace + or effective_e2b_sandbox_ttl != self.config.e2b_sandbox_ttl + ) + if backend and ( + backend != self.backend.backend_name or e2b_config_changed + ): from sforge.harness.backend.factory import create_backend self.run_backends[run_key] = create_backend( backend, @@ -316,8 +341,9 @@ def register_session(self, task_id: str, run_id: str, k8s_node_selector=k8s_node_selector or self.config.k8s_node_selector, k8s_image_registry=k8s_image_registry or self.config.k8s_image_registry, k8s_kubeconfig=k8s_kubeconfig or self.config.k8s_kubeconfig, - e2b_template_map=e2b_template_map or self.config.e2b_template_map, - e2b_sandbox_ttl=e2b_sandbox_ttl or self.config.e2b_sandbox_ttl, + e2b_template_map=effective_e2b_template_map, + e2b_template_namespace=effective_e2b_template_namespace, + e2b_sandbox_ttl=effective_e2b_sandbox_ttl, ) return token @@ -888,6 +914,7 @@ def register(req: RegisterRequest) -> RegisterResponse: max_agent_submissions=req.max_agent_submissions, submission_cooldown=req.submission_cooldown, e2b_template_map=req.e2b_template_map, + e2b_template_namespace=req.e2b_template_namespace, e2b_sandbox_ttl=req.e2b_sandbox_ttl, ) return RegisterResponse(token=token) diff --git a/sforge/harness/run_agent.py b/sforge/harness/run_agent.py index d4bd2d7..d38b949 100644 --- a/sforge/harness/run_agent.py +++ b/sforge/harness/run_agent.py @@ -377,8 +377,8 @@ def run_agent( raise RuntimeError( f"E2B Template for {image_key!r} was not found. Make sure " "the task uses a published image version with an official " - "Template, or point SFORGE_E2B_TEMPLATE_MAP at a " - "self-built Template." + "Template and SFORGE_E2B_TEMPLATE_NAMESPACE is set, or " + "point SFORGE_E2B_TEMPLATE_MAP at a self-built Template." ) raise RuntimeError( f"Image '{image_key}' not found. Run `sforge pull --task {task_spec.task_id}` to fetch from registry, or `sforge build --task {task_spec.task_id}` to build locally." @@ -406,6 +406,7 @@ def run_agent( elif backend.backend_name == "e2b": reg_body["backend"] = "e2b" reg_body["e2b_template_map"] = config.e2b_template_map + reg_body["e2b_template_namespace"] = config.e2b_template_namespace reg_body["e2b_sandbox_ttl"] = config.e2b_sandbox_ttl for _reg_attempt in range(1, 6): try: From c4283de25a2b19eb41fdb8134164758ba0482849 Mon Sep 17 00:00:00 2001 From: Tao Wang Date: Thu, 3 Sep 2026 22:35:02 +0800 Subject: [PATCH 16/17] fix(e2b): harden sandbox cleanup and document runtime behavior --- docs/en/configuration/container-backends.md | 5 +- .../en/configuration/environment-variables.md | 1 + docs/en/configuration/experiments.md | 1 + docs/en/guide/agents.md | 11 +++ docs/en/reference/cli.md | 1 + docs/zh/configuration/container-backends.md | 4 +- .../zh/configuration/environment-variables.md | 1 + docs/zh/configuration/experiments.md | 1 + docs/zh/guide/agents.md | 13 ++- docs/zh/reference/cli.md | 1 + sforge/harness/backend/e2b_backend.py | 93 ++++++++++++++----- sforge/harness/run_agent.py | 29 +++--- 12 files changed, 123 insertions(+), 38 deletions(-) diff --git a/docs/en/configuration/container-backends.md b/docs/en/configuration/container-backends.md index 7ee4c46..b79d703 100644 --- a/docs/en/configuration/container-backends.md +++ b/docs/en/configuration/container-backends.md @@ -295,6 +295,9 @@ existing E2B Template. Keep the agent timeout comfortably below the team's maximum Sandbox lifetime. The lifetime must also cover agent installation, archive extraction, pending Judge evaluations, result collection, and cleanup. +At cleanup, SForge stops lease renewal, reduces the provider-side lifetime to +at most 60 seconds, and then requests immediate deletion. This terminal lease +transition is independent of the runtime TTL configured by the user. Check each task's `judge.eval_timeout` as well. A Judge Sandbox is subject to the same team lifetime limit, so a task whose declared evaluator timeout is @@ -334,7 +337,7 @@ tasks remove proxy variables entirely. See [Network Isolation](/en/features/netw | Template not found | The Template namespace is missing or incorrect, the task image version has no published official Template, or the task uses a modified image | Check `SFORGE_E2B_TEMPLATE_NAMESPACE` and the task image version, or point `SFORGE_E2B_TEMPLATE_MAP` at a self-built Template | | Agent-side Judge requests intermittently time out | The Work Sandbox could not reach the Judge Server over the internet | Check that the Judge host and port are publicly reachable, then retry the agent submission | | A run ends at the configured task timeout with `Sandbox not found` | The team-level maximum Sandbox lifetime was reached before final extraction and cleanup | Shorten the agent timeout, reduce multi-wave batch duration, or use a plan with a longer Sandbox lifetime | -| A run leaves a Sandbox | Cleanup failed or the runner was terminated abruptly | Inspect the run log and wait for the configured E2B TTL as the final safeguard | +| A completed run retains a Sandbox for more than 60 seconds | Both the provider-side termination deadline and the immediate deletion request failed | Inspect the lifecycle cleanup error in the run or Judge Server log | ## Troubleshooting diff --git a/docs/en/configuration/environment-variables.md b/docs/en/configuration/environment-variables.md index 21d829a..0f90264 100644 --- a/docs/en/configuration/environment-variables.md +++ b/docs/en/configuration/environment-variables.md @@ -46,6 +46,7 @@ Used when running agents via `sforge run`. These are injected as container envir | `SFORGE_AGENT_API_KEY` | API key passed to the selected agent using that agent's expected env var (`ANTHROPIC_AUTH_TOKEN` or `CODEX_API_KEY`) | | `SFORGE_AGENT_API_BASE_URL` | API base URL override passed through the selected agent's base-url env var when it has one | | `SFORGE_AGENT_MODEL` | Model override for the agent | +| `SFORGE_AGENT_EFFORT` | Reasoning effort (`low`, `medium`, `high`, or `max`) | | `SFORGE_AGENT_TIMEOUT` | Agent timeout in seconds | | `SFORGE_AGENT_EXTRA_ENV` | Extra env vars for agent container, format: `"KEY1=VAL1,KEY2=VAL2"` | | `SFORGE_HTTP_PROXY` | HTTP proxy injected into the agent container at run time. Not recommended for normal use; incompatible with `--disable-internet`. | diff --git a/docs/en/configuration/experiments.md b/docs/en/configuration/experiments.md index 2fe0bb8..8408d12 100644 --- a/docs/en/configuration/experiments.md +++ b/docs/en/configuration/experiments.md @@ -56,6 +56,7 @@ Default settings applied to all tasks unless overridden per-task. |-------|------|---------|-------------| | `agent` | `string` | --- | Agent name (`claude-code`, `codex`) | | `model` | `string` | --- | Model override (overrides `model.model` for specific tasks if set per-task) | +| `effort` | `string` | --- | Reasoning effort (`low`, `medium`, `high`, or `max`); mapped to the selected Agent when supported | | `timeout` | `int` | --- | Agent timeout in seconds | | `eval_interval` | `int` | --- | Auto-eval interval in seconds | | `disable_stop_hook` | `bool` | `false` | Disable the agent stop hook | diff --git a/docs/en/guide/agents.md b/docs/en/guide/agents.md index ebb085e..584f62f 100644 --- a/docs/en/guide/agents.md +++ b/docs/en/guide/agents.md @@ -53,6 +53,17 @@ Override the model using the `--model` CLI flag or the `SFORGE_AGENT_MODEL` envi sforge run --task ad_placement_optimization --agent claude-code --model claude-opus-4-8 ``` +### Reasoning Effort + +Use `--effort` or `SFORGE_AGENT_EFFORT` to select `low`, `medium`, `high`, or +`max`. If neither is set, the Agent keeps its own default. + +| Agent | Native setting | Notes | +|-------|----------------|-------| +| Claude Code | `CLAUDE_CODE_EFFORT_LEVEL` | Uses the selected value directly | +| Codex | `model_reasoning_effort` | Maps `max` to Codex's `xhigh` value | +| opencode | `reasoningEffort` | Applies to non-Anthropic providers; Anthropic providers retain their own thinking configuration | + ### Extra Environment Variables Pass additional environment variables to the agent container using `SFORGE_AGENT_EXTRA_ENV`: diff --git a/docs/en/reference/cli.md b/docs/en/reference/cli.md index cabaf66..457baee 100644 --- a/docs/en/reference/cli.md +++ b/docs/en/reference/cli.md @@ -139,6 +139,7 @@ sforge run --experiment experiment.yaml | `--agent` | required* | Agent name (e.g., `claude-code`, `codex`). Required unless `--experiment` is specified. | | `--experiment` | --- | Path to experiment YAML config file. If `--task` is omitted, all YAML tasks run; if `--task` is provided, only that subset runs with the experiment settings. | | `--model` | --- | Model override (e.g., `claude-opus-4-8`) | +| `--effort` | Agent default | Reasoning effort (`low`, `medium`, `high`, or `max`). SForge maps the value to the selected Agent when supported. | | `--timeout` | `3600` | Agent timeout in seconds | | `--eval-interval` | `300` | Auto-eval daemon interval in seconds | | `--run-id` | random | Run identifier for tracking and log organization | diff --git a/docs/zh/configuration/container-backends.md b/docs/zh/configuration/container-backends.md index 80a1b39..dc9b4aa 100644 --- a/docs/zh/configuration/container-backends.md +++ b/docs/zh/configuration/container-backends.md @@ -285,6 +285,8 @@ E2B 套餐限制是运行契约的一部分。Template 的 CPU 和内存会在 Agent timeout 应明显短于当前套餐允许的最大 Sandbox 生命周期。Sandbox 生命周期还要覆盖 Agent 安装、归档提取、等待 Judge 评测、读取结果和资源清理。 +进入清理阶段后，SForge 会停止续租，将服务端剩余生命周期收紧到最多 60 秒，再请求即时 +删除。该终止租约转换不受用户配置的运行期 TTL 影响。 还需要检查每个任务的 `judge.eval_timeout`。Judge Sandbox 同样受 team 最大生命周期限制； 如果任务声明的评测超时长于该限制，那么当 evaluator 实际使用完整预算时，就无法保证评测 @@ -318,7 +320,7 @@ auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快 | 找不到 Template | Template namespace 缺失或错误、该任务镜像版本没有已发布的官方 Template，或任务使用了修改过的镜像 | 检查 `SFORGE_E2B_TEMPLATE_NAMESPACE` 和任务镜像版本，或用 `SFORGE_E2B_TEMPLATE_MAP` 指向自建 Template | | Agent 侧 Judge 请求偶发超时 | Work Sandbox 无法通过公网访问 Judge Server | 确认 Judge 主机和端口公网可达后重试提交 | | task timeout 到达时出现 `Sandbox not found` | 在最终归档和清理前达到了 team 级 Sandbox 最大生命周期 | 缩短 Agent timeout、减少多波次批次总时长，或使用支持更长 Sandbox 生命周期的套餐 | -| 运行结束后仍有 Sandbox | cleanup 失败或 Runner 被强制终止 | 检查 run 日志；平台最终按配置的 E2B TTL 回收 | +| 运行结束超过 60 秒后仍有 Sandbox | 服务端终止期限和即时删除请求均未生效 | 检查运行日志或 Judge Server 日志中的生命周期清理错误 | ## 常见问题 diff --git a/docs/zh/configuration/environment-variables.md b/docs/zh/configuration/environment-variables.md index 370d02d..211b918 100644 --- a/docs/zh/configuration/environment-variables.md +++ b/docs/zh/configuration/environment-variables.md @@ -46,6 +46,7 @@ SForge 大部分配置可通过 `SFORGE_*` 环境变量注入；仅 CLI/YAML 支 | `SFORGE_AGENT_API_KEY` | 传递给所选 Agent 的 API 密钥，会映射为该 Agent 需要的环境变量（`ANTHROPIC_AUTH_TOKEN` 或 `CODEX_API_KEY`） | | `SFORGE_AGENT_API_BASE_URL` | API Base URL 覆盖，会传递给所选 Agent 支持的 base-url 环境变量 | | `SFORGE_AGENT_MODEL` | Agent 模型覆盖 | +| `SFORGE_AGENT_EFFORT` | Agent 推理强度（`low`、`medium`、`high` 或 `max`） | | `SFORGE_AGENT_TIMEOUT` | Agent 超时时间（秒） | | `SFORGE_AGENT_EXTRA_ENV` | Agent 容器额外环境变量，格式：`"KEY1=VAL1,KEY2=VAL2"` | | `SFORGE_HTTP_PROXY` | 运行阶段注入 Agent 容器的 HTTP 代理。不推荐常规使用；不兼容 `--disable-internet`。 | diff --git a/docs/zh/configuration/experiments.md b/docs/zh/configuration/experiments.md index 8a9358c..52cff25 100644 --- a/docs/zh/configuration/experiments.md +++ b/docs/zh/configuration/experiments.md @@ -56,6 +56,7 @@ tasks: |------|------|--------|------| | `agent` | `string` | --- | Agent 名称（`claude-code`、`codex`） | | `model` | `string` | --- | 模型覆盖（在逐任务设置中可覆盖 `model.model`） | +| `effort` | `string` | --- | 推理强度（`low`、`medium`、`high` 或 `max`），在所选 Agent 支持时映射到其原生配置 | | `timeout` | `int` | --- | Agent 超时时间（秒） | | `eval_interval` | `int` | --- | 自动评测间隔（秒） | | `disable_stop_hook` | `bool` | `false` | 禁用 Agent Stop Hook | diff --git a/docs/zh/guide/agents.md b/docs/zh/guide/agents.md index cc74587..48c3012 100644 --- a/docs/zh/guide/agents.md +++ b/docs/zh/guide/agents.md @@ -4,7 +4,7 @@ SForge 通过插件式的 Agent 注册表管理不同的 Code Agent。运行 Edg ## Agent 注册表 -SForge 内置了 2 种 Agent： +SForge 内置了以下 Agent： | Agent 名称 | CLI 名称 | API Key 环境变量 | 模型环境变量 | 默认模型 | Stop Hook | Auto-Resume | |-----------|----------|-----------------|-------------|---------|-----------|-------------| @@ -50,6 +50,17 @@ sforge run --task ad_placement_optimization --agent claude-code sforge run --task ad_placement_optimization --agent claude-code --model claude-opus-4-8 ``` +### 推理强度 + +通过 `--effort` 或 `SFORGE_AGENT_EFFORT` 选择 `low`、`medium`、`high` 或 +`max`。两者均未设置时，Agent 保持自身默认配置。 + +| Agent | 原生配置 | 说明 | +|-------|----------|------| +| Claude Code | `CLAUDE_CODE_EFFORT_LEVEL` | 直接使用所选值 | +| Codex | `model_reasoning_effort` | 将 `max` 映射为 Codex 的 `xhigh` | +| opencode | `reasoningEffort` | 对非 Anthropic provider 生效；Anthropic provider 保持自身 thinking 配置 | + ### 额外环境变量 通过 `SFORGE_AGENT_EXTRA_ENV` 向 Agent 容器注入额外的环境变量（逗号分隔的 `KEY=VALUE` 对）： diff --git a/docs/zh/reference/cli.md b/docs/zh/reference/cli.md index 17391ee..f41de47 100644 --- a/docs/zh/reference/cli.md +++ b/docs/zh/reference/cli.md @@ -138,6 +138,7 @@ sforge run --experiment experiment.yaml | `--agent` | 必填* | Agent 名称（如 `claude-code`、`codex`）。除非指定 `--experiment`，否则必填。 | | `--experiment` | --- | 实验 YAML 配置文件路径。如果未指定 `--task`，运行 YAML 中的全部任务；如果指定了 `--task`，则只运行该子集并套用实验配置。 | | `--model` | --- | 模型覆盖（如 `claude-opus-4-8`） | +| `--effort` | Agent 默认值 | 推理强度（`low`、`medium`、`high` 或 `max`）。SForge 会在所选 Agent 支持时映射到其原生配置。 | | `--timeout` | `3600` | Agent 超时时间（秒） | | `--eval-interval` | `300` | 自动评测守护进程的间隔时间（秒） | | `--run-id` | 随机生成 | 运行标识符，用于跟踪和日志组织 | diff --git a/sforge/harness/backend/e2b_backend.py b/sforge/harness/backend/e2b_backend.py index cdd6eb2..69f5bbf 100644 --- a/sforge/harness/backend/e2b_backend.py +++ b/sforge/harness/backend/e2b_backend.py @@ -82,6 +82,9 @@ FILE_TRANSFER_ATTEMPTS = 3 FILE_TRANSFER_REQUEST_TIMEOUT = 300 SANDBOX_CREATE_ATTEMPTS = 3 +SANDBOX_CLEANUP_ATTEMPTS = 3 +SANDBOX_LIFECYCLE_REQUEST_TIMEOUT = 10 +SANDBOX_CLEANUP_GRACE_PERIOD = 60 _METADATA_NAME_KEY = "sforge_name" _E2B_NAME_PATTERN = re.compile(r"[a-z0-9_-]{1,128}") @@ -99,6 +102,8 @@ def __init__( self._sandbox = sandbox self._name = name self.default_user = default_user + self._lease_lock = threading.Lock() + self._closing = False @property def id(self) -> str: @@ -375,28 +380,65 @@ def cleanup_container( return log = logger_ or logger sandbox = self._raw(handle) - # Stop extending the lease before deletion. If every kill attempt - # fails, E2B's on-timeout=kill lifecycle remains the final safeguard - # instead of this process keeping the leaked Sandbox alive forever. + # Cleanup is a terminal lease transition, independent of the runtime + # TTL selected by the user. Remove the handle from lease renewal first, + # then shorten its provider-side deadline before attempting deletion. + # If E2B temporarily rejects kill(), on_timeout=kill still reclaims the + # Sandbox within the cleanup grace period. with self._handles_lock: self._handles.pop(handle.id, None) self._renewal_wakeup.set() + cleanup_deadline_armed = False + cleanup_timeout = min( + self._sandbox_ttl, SANDBOX_CLEANUP_GRACE_PERIOD, + ) + with handle._lease_lock: + handle._closing = True + for attempt in range(1, SANDBOX_CLEANUP_ATTEMPTS + 1): + try: + sandbox.set_timeout( + cleanup_timeout, + request_timeout=SANDBOX_LIFECYCLE_REQUEST_TIMEOUT, + ) + cleanup_deadline_armed = True + break + except Exception as exc: + if attempt == SANDBOX_CLEANUP_ATTEMPTS: + log.warning( + "Could not shorten sandbox %s cleanup deadline " + "after %d attempts: %s", + handle.id, attempt, exc, + ) + else: + time.sleep(attempt) log.info(f"Killing sandbox {handle.name} ({handle.id})...") - for attempt in range(1, 4): + for attempt in range(1, SANDBOX_CLEANUP_ATTEMPTS + 1): try: - sandbox.kill() - log.info(f"Sandbox {handle.name} killed.") + killed = sandbox.kill( + request_timeout=SANDBOX_LIFECYCLE_REQUEST_TIMEOUT, + ) + if killed: + log.info(f"Sandbox {handle.name} killed.") + else: + log.info(f"Sandbox {handle.name} was already gone.") return except Exception as exc: - if attempt == 3: - # Match the docker/k8s cleanup contract: log and rely on - # the on_timeout=kill lifecycle instead of raising. - log.error( - "Failed to kill sandbox %s after %d attempts: %s", - handle.id, - attempt, - exc, - ) + if attempt == SANDBOX_CLEANUP_ATTEMPTS: + if cleanup_deadline_armed: + log.warning( + "Failed to kill sandbox %s after %d attempts; " + "E2B will reclaim it within %ds: %s", + handle.id, attempt, + cleanup_timeout, exc, + ) + else: + message = ( + "Failed to arm cleanup deadline and kill sandbox " + "%s after %d attempts; original TTL remains the " + "only safeguard: %s" + ) % (handle.id, attempt, exc) + log.error(message) + raise RuntimeError(message) from exc return time.sleep(attempt) @@ -922,12 +964,21 @@ def _renewal_loop(self) -> None: if woke_early: continue for handle in handles: - try: - handle.raw.set_timeout(self._sandbox_ttl) - except Exception as exc: - logger.warning( - f"Lease renewal failed for sandbox {handle.id}: {exc}" - ) + self._renew_handle(handle) + + def _renew_handle(self, handle: E2BSandboxHandle) -> None: + with handle._lease_lock: + if handle._closing: + return + try: + handle.raw.set_timeout( + self._sandbox_ttl, + request_timeout=SANDBOX_LIFECYCLE_REQUEST_TIMEOUT, + ) + except Exception as exc: + logger.warning( + f"Lease renewal failed for sandbox {handle.id}: {exc}" + ) # --- Helpers --- diff --git a/sforge/harness/run_agent.py b/sforge/harness/run_agent.py index d38b949..270e9c4 100644 --- a/sforge/harness/run_agent.py +++ b/sforge/harness/run_agent.py @@ -214,17 +214,17 @@ def _extract_archive_from_container( tar czf {shlex.quote(str(remote_archive))} --files-from /dev/null fi """ - result = backend.exec_run_with_exit_code( - handle, ["/bin/bash", "-c", tar_cmd], timeout=300, - user="agent", - ) - if result.timed_out: - raise RuntimeError("Timed out while creating the submission archive") - if result.exit_code != 0: - raise RuntimeError( - "Could not create the submission archive: " + result.output[-4000:] - ) try: + result = backend.exec_run_with_exit_code( + handle, ["/bin/bash", "-c", tar_cmd], timeout=300, + ) + if result.timed_out: + raise RuntimeError("Timed out while creating the submission archive") + if result.exit_code != 0: + raise RuntimeError( + "Could not create the submission archive: " + + result.output[-4000:] + ) if backend.backend_name == "e2b": raw = backend.copy_from_container( handle, remote_archive, shutdown_event=shutdown_event, @@ -520,8 +520,9 @@ def run_agent( cap_drop=container_cap_drop or None, cpu_limit=cpu, mem_limit=mem, - # e2b execs default to the SDK's "user" account, not the image USER - user="agent", + # E2B execs default to the SDK's "user" account rather than the + # image USER. Docker and Kubernetes retain the image default. + user="agent" if backend.backend_name == "e2b" else None, ) backend.start_container(handle) logger.info(f"Container started: {container_name} (judge_url={judge_url})") @@ -702,7 +703,7 @@ def run_agent( # 8. Extract final archive (tar of submit_paths) try: final_archive = _extract_archive_from_container( - backend, handle, task_spec, shutdown_event=shutdown_event, + backend, handle, task_spec, ) (log_dir / "final_archive.tar.gz").write_bytes(final_archive) logger.info(f"Final archive: {len(final_archive)} bytes") @@ -875,7 +876,7 @@ def run_agent( try: if handle is not None: interrupted_archive = _extract_archive_from_container( - backend, handle, task_spec, shutdown_event=shutdown_event, + backend, handle, task_spec, ) (log_dir / "final_archive.tar.gz").write_bytes(interrupted_archive) logger.info(f"Final archive (interrupted): {len(interrupted_archive)} bytes") From 55badf73321f81509878f6501ae2697ac2d58f58 Mon Sep 17 00:00:00 2001 From: qinshengling Date: Tue, 8 Sep 2026 14:04:03 +0800 Subject: [PATCH 17/17] docs: add E2B single-task example, condense plan limits, sync agent/CLI docs with code --- docs/.vitepress/config.mts | 2 + docs/en/configuration/container-backends.md | 62 +++---- .../en/configuration/environment-variables.md | 4 +- docs/en/configuration/experiments.md | 2 +- docs/en/examples/single-task-e2b.md | 173 ++++++++++++++++++ docs/en/guide/agents.md | 41 ++++- docs/en/guide/getting-started.md | 2 + docs/en/reference/cli.md | 4 +- docs/zh/configuration/container-backends.md | 48 ++--- .../zh/configuration/environment-variables.md | 4 +- docs/zh/configuration/experiments.md | 2 +- docs/zh/examples/single-task-e2b.md | 158 ++++++++++++++++ docs/zh/guide/agents.md | 31 +++- docs/zh/guide/getting-started.md | 2 + docs/zh/reference/cli.md | 4 +- 15 files changed, 447 insertions(+), 92 deletions(-) create mode 100644 docs/en/examples/single-task-e2b.md create mode 100644 docs/zh/examples/single-task-e2b.md diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 3272ff8..8c2b0e6 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -36,6 +36,7 @@ export default defineConfig({ text: 'Examples', items: [ { text: 'Single Task (Docker)', link: '/en/examples/single-task-docker' }, + { text: 'Single Task (E2B)', link: '/en/examples/single-task-e2b' }, { text: 'All Tasks (Kubernetes)', link: '/en/examples/all-tasks-k8s' }, ], }, @@ -105,6 +106,7 @@ export default defineConfig({ text: '示例', items: [ { text: '单任务运行 (Docker)', link: '/zh/examples/single-task-docker' }, + { text: '单任务运行 (E2B)', link: '/zh/examples/single-task-e2b' }, { text: '全部任务 (Kubernetes)', link: '/zh/examples/all-tasks-k8s' }, ], }, diff --git a/docs/en/configuration/container-backends.md b/docs/en/configuration/container-backends.md index b79d703..50933a6 100644 --- a/docs/en/configuration/container-backends.md +++ b/docs/en/configuration/container-backends.md @@ -286,36 +286,25 @@ per `sforge run` command. ### Plan Limits and Run Sizing -E2B plan limits are part of the runtime contract. Template CPU and memory are -fixed when the Template is built, and requests above the team's limits are -rejected during Template creation. Rebuild the Template with values supported -by the target team; runtime `--work-*` and `--judge-*` flags cannot resize an -existing E2B Template. - -Keep the agent timeout comfortably below the team's maximum Sandbox lifetime. -The lifetime must also cover agent installation, archive extraction, pending -Judge evaluations, result collection, and cleanup. -At cleanup, SForge stops lease renewal, reduces the provider-side lifetime to -at most 60 seconds, and then requests immediate deletion. This terminal lease -transition is independent of the runtime TTL configured by the user. - -Check each task's `judge.eval_timeout` as well. A Judge Sandbox is subject to -the same team lifetime limit, so a task whose declared evaluator timeout is -longer than that limit cannot be guaranteed to finish when the evaluator uses -its full budget. Do not silently lower the evaluator timeout if leaderboard -comparability matters; use a plan with a sufficient lifetime instead. - -Tasks run fully in parallel, so total E2B usage scales with the number of -selected tasks. Budget for the Work Sandboxes, temporary Judge Sandboxes, and -active Game Sandboxes, and leave capacity for overlap while evaluations finish -and resources are cleaned up. - -The final score retains SForge's backend-independent semantics: it is the best -result among completed agent and auto-eval submissions. `final_archive.tar.gz` -is a recovery snapshot and is not submitted implicitly at timeout. For short -runs, choose an auto-eval interval that leaves enough time for at least one -evaluation to finish, or have the agent call `sforge-submit` before the run -ends. +E2B plan limits are part of the runtime contract: + +- **Resources**: Template CPU and memory are fixed at build time; over-limit + requests fail at Template creation. Rebuild with supported values, since + `--work-*` and `--judge-*` flags cannot resize an existing Template. +- **Sandbox lifetime**: Keep the agent timeout well below the team's maximum + Sandbox lifetime, which must also cover agent install, archive extraction, + Judge evaluation, result collection, and cleanup. At cleanup SForge caps the + remaining lifetime at 60 seconds and deletes the Sandbox, regardless of the + configured TTL. +- **Evaluator timeout**: Judge Sandboxes share the same limit, so a task whose + `judge.eval_timeout` exceeds it may not finish. Prefer a plan with a longer + lifetime over lowering the timeout. +- **Concurrency**: Tasks run fully in parallel. Budget Work, Judge, and Game + Sandboxes for all selected tasks, plus overlap during evaluation and cleanup. +- **Final score**: As on other backends, the score is the best completed agent + or auto-eval submission. `final_archive.tar.gz` is a recovery snapshot, not + an implicit submission at timeout. For short runs, allow at least one + auto-eval to finish, or have the agent call `sforge-submit` before the end. ### Required User Input @@ -330,15 +319,6 @@ Judge, or Game Sandboxes. A remote-reachable proxy is forwarded only when the user explicitly configures `SFORGE_HTTP_PROXY` or `SFORGE_HTTPS_PROXY`; isolated tasks remove proxy variables entirely. See [Network Isolation](/en/features/network-isolation). -### E2B Troubleshooting - -| Symptom | Likely cause | Fix | -| --- | --- | --- | -| Template not found | The Template namespace is missing or incorrect, the task image version has no published official Template, or the task uses a modified image | Check `SFORGE_E2B_TEMPLATE_NAMESPACE` and the task image version, or point `SFORGE_E2B_TEMPLATE_MAP` at a self-built Template | -| Agent-side Judge requests intermittently time out | The Work Sandbox could not reach the Judge Server over the internet | Check that the Judge host and port are publicly reachable, then retry the agent submission | -| A run ends at the configured task timeout with `Sandbox not found` | The team-level maximum Sandbox lifetime was reached before final extraction and cleanup | Shorten the agent timeout, reduce multi-wave batch duration, or use a plan with a longer Sandbox lifetime | -| A completed run retains a Sandbox for more than 60 seconds | Both the provider-side termination deadline and the immediate deletion request failed | Inspect the lifecycle cleanup error in the run or Judge Server log | - ## Troubleshooting | Backend | Symptom | Likely cause | Fix | @@ -349,3 +329,7 @@ tasks remove proxy variables entirely. See [Network Isolation](/en/features/netw | Kubernetes | Work cannot reach Judge | The Judge URL is not pod-reachable, or the server is not listening externally | Start `sforge serve` with `--host 0.0.0.0` and pass a pod-reachable IP or Service URL | | Kubernetes | `--disable-internet` has no effect | CNI does not enforce NetworkPolicy, or permissions are insufficient | Confirm NetworkPolicy support and permissions | | Kubernetes | Pod remains Pending with a node selector | No nodes match the selector | Check `SFORGE_K8S_NODE_SELECTOR` and node labels | +| E2B | Template not found | The Template namespace is missing or incorrect, the task image version has no published official Template, or the task uses a modified image | Check `SFORGE_E2B_TEMPLATE_NAMESPACE` and the task image version, or point `SFORGE_E2B_TEMPLATE_MAP` at a self-built Template | +| E2B | Agent-side Judge requests intermittently time out | The Work Sandbox could not reach the Judge Server over the internet | Check that the Judge host and port are publicly reachable, then retry the agent submission | +| E2B | A run ends at the configured task timeout with `Sandbox not found` | The team-level maximum Sandbox lifetime was reached before final extraction and cleanup | Shorten the agent timeout, reduce multi-wave batch duration, or use a plan with a longer Sandbox lifetime | +| E2B | A completed run retains a Sandbox for more than 60 seconds | Both the provider-side termination deadline and the immediate deletion request failed | Inspect the lifecycle cleanup error in the run or Judge Server log | diff --git a/docs/en/configuration/environment-variables.md b/docs/en/configuration/environment-variables.md index 0f90264..135d009 100644 --- a/docs/en/configuration/environment-variables.md +++ b/docs/en/configuration/environment-variables.md @@ -43,7 +43,7 @@ Used when running agents via `sforge run`. These are injected as container envir | Variable | Purpose | |----------|---------| -| `SFORGE_AGENT_API_KEY` | API key passed to the selected agent using that agent's expected env var (`ANTHROPIC_AUTH_TOKEN` or `CODEX_API_KEY`) | +| `SFORGE_AGENT_API_KEY` | API key passed to the selected agent using that agent's expected env var (`ANTHROPIC_AUTH_TOKEN` for Claude Code, `CODEX_API_KEY` for Codex, `ANTHROPIC_API_KEY` for OpenCode) | | `SFORGE_AGENT_API_BASE_URL` | API base URL override passed through the selected agent's base-url env var when it has one | | `SFORGE_AGENT_MODEL` | Model override for the agent | | `SFORGE_AGENT_EFFORT` | Reasoning effort (`low`, `medium`, `high`, or `max`) | @@ -92,7 +92,7 @@ Judge CPU/memory limits are currently set through CLI flags (`--judge-cpu-limit` | `SFORGE_K8S_IMAGE_REGISTRY` | --- | Container registry for K8s image pulls | | `SFORGE_K8S_KUBECONFIG` | --- | Path to kubeconfig file | | `SFORGE_K8S_NODE_SELECTOR` | --- | Node selector for K8s pods, format: `"key1=val1,key2=val2"` | -| `SFORGE_E2B_TEMPLATE_MAP` | --- | Explicit E2B image-key to template mapping | +| `SFORGE_E2B_TEMPLATE_MAP` | --- | Explicit E2B image-key to Template mapping, overriding the derived name. Inline `"image_key=template,..."` pairs, or a path ending in `.json` holding a JSON object | | `SFORGE_E2B_TEMPLATE_NAMESPACE` | --- | Namespace for published E2B Templates (for example, `edgebench`) | | `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL in seconds | diff --git a/docs/en/configuration/experiments.md b/docs/en/configuration/experiments.md index 8408d12..97a45b2 100644 --- a/docs/en/configuration/experiments.md +++ b/docs/en/configuration/experiments.md @@ -54,7 +54,7 @@ Default settings applied to all tasks unless overridden per-task. | Field | Type | Default | Description | |-------|------|---------|-------------| -| `agent` | `string` | --- | Agent name (`claude-code`, `codex`) | +| `agent` | `string` | --- | Agent name (`claude-code`, `codex`, `opencode`, or a pinned variant such as `claude-code-2.1.214`) | | `model` | `string` | --- | Model override (overrides `model.model` for specific tasks if set per-task) | | `effort` | `string` | --- | Reasoning effort (`low`, `medium`, `high`, or `max`); mapped to the selected Agent when supported | | `timeout` | `int` | --- | Agent timeout in seconds | diff --git a/docs/en/examples/single-task-e2b.md b/docs/en/examples/single-task-e2b.md new file mode 100644 index 0000000..24546e0 --- /dev/null +++ b/docs/en/examples/single-task-e2b.md @@ -0,0 +1,173 @@ +--- +title: "Single Task (E2B)" +--- + +# Single Task (E2B) + +Run one EdgeBench task (`ad_placement_optimization`) in [E2B](https://e2b.dev) cloud Sandboxes. No local Docker daemon or Kubernetes cluster is needed; the only thing you host yourself is the Judge Server. + +::: tip When to use E2B +E2B is the easiest way to run tasks when you cannot run Docker locally. Work, Judge, and Game environments are all created as E2B Sandboxes from pre-built official Templates. Concurrency and Sandbox lifetime are bounded by your E2B plan; see [Plan Limits and Run Sizing](/en/configuration/container-backends#plan-limits-and-run-sizing) before scaling up. +::: + +::: warning Cost +Frontier-model runs are expensive — one task over the official 12-hour budget can cost hundreds to over a thousand USD in API usage, plus E2B Sandbox time. Start with a short `--timeout` (this example uses 2 hours) to gauge your burn rate before scaling up. +::: + +## Prerequisites + +| Requirement | Check | +|-------------|-------| +| E2B account and API key | `E2B_API_KEY` from the E2B dashboard | +| A host with a **publicly routable** address for the Judge Server | Sandboxes submit to it over the internet | +| Python >= 3.10 | `python --version` | + +> **Note:** The agent timeout must fit within your E2B plan's maximum Sandbox +> lifetime, including time for evaluation and cleanup. Check your plan before +> running long tasks. + +## Using Claude Code with Anthropic API + +### 1. Install SForge with the E2B extra + +```bash +pip install "sforge[e2b]" +``` + +### 2. Fetch task definitions + +```bash +sforge fetch-tasks edgebench +``` + +Downloads the EdgeBench task JSONs and `BENCHMARK.yaml` into `./tasks/`. +Verify with: + +```bash +sforge list +``` + +### 3. Configure E2B + +E2B Sandboxes start from Templates, not Docker images, so there is no +`sforge pull` step. Official Templates for every published EdgeBench task are +already built; SForge derives the Template name from each task's image +reference. Set the following on **both** the Judge Server host and the run +host: + +```bash +export E2B_API_KEY="e2b_xxxx" +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench +``` + +If the Judge Server and `sforge run` are on **different machines**, they also +need a shared admin secret. The Judge Server only accepts registrations whose +secret matches its own, and each side otherwise generates its own random one +under `SFORGE_LOG_DIR/.judge-admin-secret`. Export the same value on both hosts: + +```bash +export SFORGE_ADMIN_SECRET="$(openssl rand -base64 32)" # same value on both hosts +``` + +When both run on the same machine with the same log directory, they share the +generated file automatically and nothing needs to be set. Never pass the secret +to the agent. + +### 4. Start the judge server + +On the publicly reachable host: + +```bash +sforge serve --host 0.0.0.0 --port 8080 +``` + +The Judge Server needs `E2B_API_KEY` too: when a run registers, it creates +the Judge and Game Sandboxes on E2B, evaluates each submitted archive in an +ephemeral Judge Sandbox, and returns scores. Make sure port 8080 is open to +the internet. + +### 5. Run the agent + +On the run host (can be the same machine), point `--judge-url` at the Judge +Server's public address: + +```bash +SFORGE_AGENT_API_KEY="sk-ant-xxxx" \ +sforge run --backend e2b \ + --task ad_placement_optimization --agent claude-code \ + --model claude-opus-4-8[1m] \ + --timeout 7200 \ + --judge-url http://YOUR_PUBLIC_HOST:8080 \ + --run-id ad-placement-optimization-e2b-001 +``` + +This launches Claude Opus 4.8 in an E2B Work Sandbox for 2 hours. +You will see the agent's work output streamed to stdout in real time. + +Each `sforge run` with `--backend e2b` evaluates tasks independently; passing +several task IDs schedules several separate Sandbox sets. + +### 6. View results + +You can view the progress in real time via the built-in web UI: + +```bash +sforge visualizer +# Open http://127.0.0.1:8000 +``` + +Or inspect files directly: + +```bash +ls logs/runs/*/ad_placement_optimization/ +cat logs/runs/*/ad_placement_optimization/final_result.json +``` + +Sandboxes are deleted automatically when the run finishes. If one lingers for +more than a minute, see [Troubleshooting](/en/configuration/container-backends#troubleshooting). + + +## Using a Third-Party Model + +The model-routing and context-window settings are identical to the Docker +example; only the backend flags change. Follow +[Using a Third-Party Model](/en/examples/single-task-docker#using-a-third-party-model) +and add `--backend e2b --judge-url http://YOUR_PUBLIC_HOST:8080` to the +`sforge run` command, for example: + +```bash +export SFORGE_AGENT_API_KEY="your-deepseek-key" +export SFORGE_AGENT_API_BASE_URL="https://api.deepseek.com/anthropic" +export SFORGE_CLAUDE_CACHE_OPT=1 +export SFORGE_AGENT_EXTRA_ENV="ANTHROPIC_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_OPUS_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_SONNET_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_HAIKU_MODEL=deepseek-v4-pro[1m],CLAUDE_CODE_SUBAGENT_MODEL=deepseek-v4-pro[1m]" + +sforge run --backend e2b \ + --task ad_placement_optimization --agent claude-code \ + --model deepseek-v4-pro[1m] \ + --timeout 7200 \ + --judge-url http://YOUR_PUBLIC_HOST:8080 \ + --run-id ad-placement-deepseek-e2b-001 +``` + + +## Network Isolation + +The `internet` field in each task JSON and the `--disable-internet` / +`--enable-internet` flags work the same as on Docker. On E2B, SForge applies +the allowlist through E2B's native `update_network` API instead of host +iptables, so nothing is left behind on your machine. E2B rules are host-level +rather than port-level: allowing a hostname allows all of its ports. + +Host `HTTP_PROXY` / `HTTPS_PROXY` variables are **not** forwarded into +Sandboxes. If the model API must go through a proxy, it has to be reachable +from the internet and set explicitly via `SFORGE_HTTP_PROXY` / +`SFORGE_HTTPS_PROXY`. See [Network Isolation](/en/features/network-isolation#e2b-backend). + + +## LLM-Graded Tasks + +LLM-graded tasks such as `college_english_exam_bank` need grading credentials +in the Judge environment, exactly as on Docker. Set `SFORGE_JUDGE_EXTRA_ENV` +**before starting the judge server**; see +[LLM-Graded Tasks](/en/examples/single-task-docker#llm-graded-tasks) for the +variables. diff --git a/docs/en/guide/agents.md b/docs/en/guide/agents.md index 584f62f..59e1a8b 100644 --- a/docs/en/guide/agents.md +++ b/docs/en/guide/agents.md @@ -8,22 +8,45 @@ SForge manages different Code Agents through a plugin-style agent registry. To r |-------|----------|-------------|-----------|---------------|-----------|-------------| | Claude Code | `claude-code` | `ANTHROPIC_AUTH_TOKEN` | `ANTHROPIC_MODEL` | -- | Yes | Yes | | Codex | `codex` | `CODEX_API_KEY` | `CODEX_MODEL` | -- | Yes | Yes | -| opencode | `opencode` | `ANTHROPIC_API_KEY` | -- | -- | No (auto-resume only) | Yes | +| OpenCode | `opencode` | `ANTHROPIC_API_KEY` | -- | -- | No | Yes | Use the `--agent` flag to select an Agent: ```bash sforge run --task ad_placement_optimization --agent claude-code sforge run --task ad_placement_optimization --agent codex -sforge run --task ad_placement_optimization --agent opencode +sforge run --task ad_placement_optimization --agent OpenCode ``` -> **opencode notes**: models are addressed as `provider/model` (a bare model -> name is treated as `anthropic/`). The default model, custom base URL -> (`SFORGE_AGENT_API_BASE_URL`), and permissions are injected via the -> `OPENCODE_CONFIG_CONTENT` environment variable — no config file is written -> into the container. opencode has no Stop-hook mechanism; long runs rely on -> the harness auto-resume loop (`opencode run --continue`). +::: details Pinned versions +Each agent installs a fixed release inside the Work container: + +| CLI name | Installed release | +|----------|-------------------| +| `claude-code` | Claude Code 2.1.159 | +| `claude-code-2.1.214` | Claude Code 2.1.214 | +| `codex` | Codex 0.130.0 | +| `codex-0.145.0` | Codex 0.145.0 | +| `opencode` | OpenCode 1.18.2 | + +`claude-code-2.1.214` and `codex-0.145.0` are newer pins kept as separate +agents so existing results stay comparable. Claude 5-family models +(Fable/Mythos) require `claude-code-2.1.214`. +::: + +::: details OpenCode notes +- Models are addressed as `provider/model`; a bare model name is treated as + `anthropic/`. +- The default model, custom base URL (`SFORGE_AGENT_API_BASE_URL`), and + permissions are injected via the `OPENCODE_CONFIG_CONTENT` environment + variable. No config file is written into the container. +- `SFORGE_AGENT_API_BASE_URL` follows the Claude Code convention (no `/v1`); + SForge appends `/v1` for OpenCode's AI SDK client. +- When the task runs without internet, `webfetch` and `websearch` are denied so + the model does not waste turns on blocked requests. +- OpenCode has no Stop-hook mechanism; long runs rely on the harness + auto-resume loop (`opencode run --continue`). +::: ## Agent Configuration @@ -62,7 +85,7 @@ Use `--effort` or `SFORGE_AGENT_EFFORT` to select `low`, `medium`, `high`, or |-------|----------------|-------| | Claude Code | `CLAUDE_CODE_EFFORT_LEVEL` | Uses the selected value directly | | Codex | `model_reasoning_effort` | Maps `max` to Codex's `xhigh` value | -| opencode | `reasoningEffort` | Applies to non-Anthropic providers; Anthropic providers retain their own thinking configuration | +| OpenCode | `reasoningEffort` | Applies to non-Anthropic providers; Anthropic providers retain their own thinking configuration | ### Extra Environment Variables diff --git a/docs/en/guide/getting-started.md b/docs/en/guide/getting-started.md index 26c5147..c81664f 100644 --- a/docs/en/guide/getting-started.md +++ b/docs/en/guide/getting-started.md @@ -34,6 +34,8 @@ Install the released package: pip install sforge ``` +To use the [E2B backend](/en/configuration/container-backends#e2b-backend), include the `e2b` extra: `pip install "sforge[e2b]"`. + Or install from source: ```bash diff --git a/docs/en/reference/cli.md b/docs/en/reference/cli.md index 457baee..b3d8172 100644 --- a/docs/en/reference/cli.md +++ b/docs/en/reference/cli.md @@ -136,7 +136,7 @@ sforge run --experiment experiment.yaml | Flag | Default | Description | |------|---------|-------------| | `--task` | required* | One or more task IDs (space-separated). Multiple IDs batch-schedule independent task runs; they are not combined into one evaluation. | -| `--agent` | required* | Agent name (e.g., `claude-code`, `codex`). Required unless `--experiment` is specified. | +| `--agent` | required* | Agent name: `claude-code`, `codex`, `opencode`, or a pinned variant (`claude-code-2.1.214`, `codex-0.145.0`). Required unless `--experiment` is specified. | | `--experiment` | --- | Path to experiment YAML config file. If `--task` is omitted, all YAML tasks run; if `--task` is provided, only that subset runs with the experiment settings. | | `--model` | --- | Model override (e.g., `claude-opus-4-8`) | | `--effort` | Agent default | Reasoning effort (`low`, `medium`, `high`, or `max`). SForge maps the value to the selected Agent when supported. | @@ -155,7 +155,7 @@ sforge run --experiment experiment.yaml | `--disable-stop-hook` | `false` | Disable the stop hook (allow agent to exit normally) | | `--disable-auto-eval` | `false` | Disable the background auto-evaluation daemon | | `--disable-auto-resume` | `false` | Disable auto-resume on abnormal agent exit | -| `--disable-internet` | `false` | Force network isolation (only judge server + API allowed). Requires `sudo` for iptables. Mutually exclusive with `--enable-internet`. | +| `--disable-internet` | `false` | Force network isolation (only judge server + API allowed). On the Docker backend this requires passwordless `sudo` for iptables; k8s and e2b isolate inside the cluster/provider. Mutually exclusive with `--enable-internet`. | | `--enable-internet` | `false` | Force full internet access (overrides per-task `internet: false` setting). Mutually exclusive with `--disable-internet`. | ::: warning Requirement diff --git a/docs/zh/configuration/container-backends.md b/docs/zh/configuration/container-backends.md index dc9b4aa..999db01 100644 --- a/docs/zh/configuration/container-backends.md +++ b/docs/zh/configuration/container-backends.md @@ -279,27 +279,20 @@ Sandbox。 ### 套餐限制与运行规模 -E2B 套餐限制是运行契约的一部分。Template 的 CPU 和内存会在构建时固化；超过当前 team -上限的请求会在创建 Template 时被拒绝。应使用目标 team 支持的规格重新构建 Template； -运行时的 `--work-*` 和 `--judge-*` 参数不能调整已有 E2B Template 的资源。 - -Agent timeout 应明显短于当前套餐允许的最大 Sandbox 生命周期。Sandbox 生命周期还要覆盖 -Agent 安装、归档提取、等待 Judge 评测、读取结果和资源清理。 -进入清理阶段后，SForge 会停止续租，将服务端剩余生命周期收紧到最多 60 秒，再请求即时 -删除。该终止租约转换不受用户配置的运行期 TTL 影响。 - -还需要检查每个任务的 `judge.eval_timeout`。Judge Sandbox 同样受 team 最大生命周期限制； -如果任务声明的评测超时长于该限制，那么当 evaluator 实际使用完整预算时，就无法保证评测 -完成。如果要求与榜单口径可比，不应静默缩短 evaluator timeout，而应使用生命周期足够长的 -套餐。 - -任务是全并行运行的，E2B 总用量随所选任务数增长。容量规划要计入 Work Sandbox、临时 -Judge Sandbox 和活跃 Game Sandbox，并为评测完成和资源清理的重叠阶段留出余量。 - -最终分数沿用 SForge 与后端无关的既有语义：只在已经完成的 agent submission 和 -auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快照，timeout 时不会被 -隐式提交。短任务应选择能为至少一次完整评分留出时间的 auto-eval 间隔，或者让 Agent 在 -结束前主动执行 `sforge-submit`。 +E2B 套餐限制是运行契约的一部分： + +- **资源规格**：Template 的 CPU 和内存在构建时固化，超限会在创建 Template 时被拒绝。 + 请按 team 支持的规格重建；`--work-*` 和 `--judge-*` 参数不能调整已有 Template。 +- **Sandbox 生命周期**：Agent timeout 应明显短于套餐的最大 Sandbox 生命周期，后者还要覆盖 + Agent 安装、归档提取、Judge 评测、读取结果和清理。清理时 SForge 会把剩余生命周期收紧到 + 60 秒内并删除 Sandbox，不受配置的 TTL 影响。 +- **评测超时**：Judge Sandbox 受同样限制，任务的 `judge.eval_timeout` 若超过该限制则可能 + 无法完成。应换更长生命周期的套餐，而不是缩短超时。 +- **并发用量**：任务全并行运行，需按所选任务数为 Work、Judge 和 Game Sandbox 预留容量， + 并为评测与清理的重叠阶段留出余量。 +- **最终分数**：与其他后端一致，取已完成的 agent 或 auto-eval submission 中的最佳结果。 + `final_archive.tar.gz` 是恢复快照，timeout 时不会隐式提交。短任务应保证至少一次 auto-eval + 能完成，或让 Agent 在结束前执行 `sforge-submit`。 ### 用户需要提供的内容 @@ -313,15 +306,6 @@ auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快 只有用户显式配置 `SFORGE_HTTP_PROXY` 或 `SFORGE_HTTPS_PROXY` 时，远端可访问的代理才会 被转发；网络隔离任务会移除全部代理变量。详见[网络隔离](/zh/features/network-isolation)。 -### E2B 常见问题 - -| 现象 | 可能原因 | 处理方式 | -| --- | --- | --- | -| 找不到 Template | Template namespace 缺失或错误、该任务镜像版本没有已发布的官方 Template，或任务使用了修改过的镜像 | 检查 `SFORGE_E2B_TEMPLATE_NAMESPACE` 和任务镜像版本，或用 `SFORGE_E2B_TEMPLATE_MAP` 指向自建 Template | -| Agent 侧 Judge 请求偶发超时 | Work Sandbox 无法通过公网访问 Judge Server | 确认 Judge 主机和端口公网可达后重试提交 | -| task timeout 到达时出现 `Sandbox not found` | 在最终归档和清理前达到了 team 级 Sandbox 最大生命周期 | 缩短 Agent timeout、减少多波次批次总时长，或使用支持更长 Sandbox 生命周期的套餐 | -| 运行结束超过 60 秒后仍有 Sandbox | 服务端终止期限和即时删除请求均未生效 | 检查运行日志或 Judge Server 日志中的生命周期清理错误 | - ## 常见问题 | 后端 | 现象 | 可能原因 | 处理方式 | @@ -332,3 +316,7 @@ auto-eval submission 中选取最佳结果。`final_archive.tar.gz` 是恢复快 | Kubernetes | Work Pod 无法提交评测 | Judge URL 对 Pod 不可达，或服务没有监听外部地址 | 用 `--host 0.0.0.0` 启动 `sforge serve`，并设置 Pod 可访问的 IP 或 Service URL | | Kubernetes | `--disable-internet` 没有效果 | CNI 不执行 NetworkPolicy，或权限不足 | 确认 CNI 支持 NetworkPolicy，并检查当前身份的权限 | | Kubernetes | 配置 node selector 后 Pod Pending | 没有节点匹配 selector | 检查 `SFORGE_K8S_NODE_SELECTOR` 和节点标签 | +| E2B | 找不到 Template | Template namespace 缺失或错误、该任务镜像版本没有已发布的官方 Template，或任务使用了修改过的镜像 | 检查 `SFORGE_E2B_TEMPLATE_NAMESPACE` 和任务镜像版本，或用 `SFORGE_E2B_TEMPLATE_MAP` 指向自建 Template | +| E2B | Agent 侧 Judge 请求偶发超时 | Work Sandbox 无法通过公网访问 Judge Server | 确认 Judge 主机和端口公网可达后重试提交 | +| E2B | task timeout 到达时出现 `Sandbox not found` | 在最终归档和清理前达到了 team 级 Sandbox 最大生命周期 | 缩短 Agent timeout、减少多波次批次总时长，或使用支持更长 Sandbox 生命周期的套餐 | +| E2B | 运行结束超过 60 秒后仍有 Sandbox | 服务端终止期限和即时删除请求均未生效 | 检查运行日志或 Judge Server 日志中的生命周期清理错误 | diff --git a/docs/zh/configuration/environment-variables.md b/docs/zh/configuration/environment-variables.md index 211b918..56872bb 100644 --- a/docs/zh/configuration/environment-variables.md +++ b/docs/zh/configuration/environment-variables.md @@ -43,7 +43,7 @@ SForge 大部分配置可通过 `SFORGE_*` 环境变量注入；仅 CLI/YAML 支 | 变量 | 用途 | |------|------| -| `SFORGE_AGENT_API_KEY` | 传递给所选 Agent 的 API 密钥，会映射为该 Agent 需要的环境变量（`ANTHROPIC_AUTH_TOKEN` 或 `CODEX_API_KEY`） | +| `SFORGE_AGENT_API_KEY` | 传递给所选 Agent 的 API 密钥，会映射为该 Agent 需要的环境变量（Claude Code 为 `ANTHROPIC_AUTH_TOKEN`，Codex 为 `CODEX_API_KEY`，OpenCode 为 `ANTHROPIC_API_KEY`） | | `SFORGE_AGENT_API_BASE_URL` | API Base URL 覆盖，会传递给所选 Agent 支持的 base-url 环境变量 | | `SFORGE_AGENT_MODEL` | Agent 模型覆盖 | | `SFORGE_AGENT_EFFORT` | Agent 推理强度（`low`、`medium`、`high` 或 `max`） | @@ -91,7 +91,7 @@ Judge CPU/内存限制目前通过 CLI 参数（`--judge-cpu-limit`、`--judge-m | `SFORGE_K8S_IMAGE_REGISTRY` | --- | K8s 镜像拉取使用的容器镜像仓库 | | `SFORGE_K8S_KUBECONFIG` | --- | kubeconfig 文件路径 | | `SFORGE_K8S_NODE_SELECTOR` | --- | K8s Pod 的节点选择器，格式：`"key1=val1,key2=val2"` | -| `SFORGE_E2B_TEMPLATE_MAP` | --- | E2B image key 到 Template 的显式映射 | +| `SFORGE_E2B_TEMPLATE_MAP` | --- | E2B image key 到 Template 的显式映射，覆盖自动推导的名称。支持内联 `"image_key=template,..."`，或以 `.json` 结尾的 JSON 对象文件路径 | | `SFORGE_E2B_TEMPLATE_NAMESPACE` | --- | 已发布 E2B Template 的 namespace（例如 `edgebench`） | | `SFORGE_E2B_SANDBOX_TTL` | `3600` | E2B Sandbox TTL（秒） | diff --git a/docs/zh/configuration/experiments.md b/docs/zh/configuration/experiments.md index 52cff25..dbb2870 100644 --- a/docs/zh/configuration/experiments.md +++ b/docs/zh/configuration/experiments.md @@ -54,7 +54,7 @@ tasks: | 字段 | 类型 | 默认值 | 说明 | |------|------|--------|------| -| `agent` | `string` | --- | Agent 名称（`claude-code`、`codex`） | +| `agent` | `string` | --- | Agent 名称（`claude-code`、`codex`、`opencode`，或 `claude-code-2.1.214` 等固定版本变体） | | `model` | `string` | --- | 模型覆盖（在逐任务设置中可覆盖 `model.model`） | | `effort` | `string` | --- | 推理强度（`low`、`medium`、`high` 或 `max`），在所选 Agent 支持时映射到其原生配置 | | `timeout` | `int` | --- | Agent 超时时间（秒） | diff --git a/docs/zh/examples/single-task-e2b.md b/docs/zh/examples/single-task-e2b.md new file mode 100644 index 0000000..b93bbc1 --- /dev/null +++ b/docs/zh/examples/single-task-e2b.md @@ -0,0 +1,158 @@ +--- +title: "单任务运行 (E2B)" +--- + +# 单任务运行 (E2B) + +在 [E2B](https://e2b.dev) 云端 Sandbox 中运行一个 EdgeBench 任务（`ad_placement_optimization`）。不需要本地 Docker daemon 或 Kubernetes 集群，唯一需要自己部署的是 Judge Server。 + +::: tip 适用场景 +本地没法跑 Docker 时，E2B 是最省事的方式。Work、Judge 和 Game 环境都会从官方预构建的 Template 创建为 E2B Sandbox。并发数和 Sandbox 生命周期受你的 E2B 套餐限制，扩大规模前请先看[套餐限制与运行规模](/zh/configuration/container-backends#套餐限制与运行规模)。 +::: + +::: warning 费用 +前沿模型的运行费用很高——单个任务跑满官方 12 小时预算，API 费用可达数百甚至上千美元，另外还有 E2B Sandbox 时长费用。建议先用较短的 `--timeout`（本示例为 2 小时）试跑，摸清费用水平后再扩大规模。 +::: + +## 前提条件 + +| 要求 | 验证方式 | +|------|----------| +| E2B 账号和 API key | 在 E2B 控制台获取 `E2B_API_KEY` | +| 一台有**公网可达**地址的主机运行 Judge Server | Sandbox 通过公网向它提交 | +| Python >= 3.10 | `python --version` | + +> **注意：** Agent timeout 必须落在 E2B 套餐允许的最大 Sandbox 生命周期之内， +> 且要给评测和清理留出时间。跑长任务前先确认套餐上限。 + +## 使用 Claude Code + Anthropic API + +### 1. 安装带 E2B 依赖的 SForge + +```bash +pip install "sforge[e2b]" +``` + +### 2. 获取任务定义 + +```bash +sforge fetch-tasks edgebench +``` + +将 EdgeBench 任务 JSON 和 `BENCHMARK.yaml` 下载到 `./tasks/`。验证： + +```bash +sforge list +``` + +### 3. 配置 E2B + +E2B Sandbox 从 Template 启动而不是 Docker 镜像，所以没有 `sforge pull` 这一步。 +所有已发布 EdgeBench 任务的官方 Template 已经构建好，SForge 会根据任务镜像引用 +自动推导 Template 名称。在 Judge Server 主机和运行主机上**都**设置： + +```bash +export E2B_API_KEY="e2b_xxxx" +export SFORGE_E2B_TEMPLATE_NAMESPACE=edgebench +``` + +如果 Judge Server 和 `sforge run` **不在同一台机器**，两边还需要一个共享的 admin +secret。Judge Server 只接受与自己一致的 secret，而未设置时两边会各自在 +`SFORGE_LOG_DIR/.judge-admin-secret` 下随机生成，值对不上。在两台主机上导出同一个值： + +```bash +export SFORGE_ADMIN_SECRET="$(openssl rand -base64 32)" # 两台主机使用同一个值 +``` + +同一台机器、同一个 log 目录下运行时，两边自动共用生成的文件，不需要设置。 +不要把该 secret 传给 Agent。 + +### 4. 启动 Judge 服务器 + +在公网可达的主机上： + +```bash +sforge serve --host 0.0.0.0 --port 8080 +``` + +Judge Server 同样需要 `E2B_API_KEY`：运行注册后，它会在 E2B 上创建 Judge 和 +Game Sandbox，在临时 Judge Sandbox 中评测每个提交的压缩包并返回分数。 +确保 8080 端口对公网开放。 + +### 5. 运行 Agent + +在运行主机上（可以和 Judge Server 同一台机器），`--judge-url` 指向 Judge Server +的公网地址： + +```bash +SFORGE_AGENT_API_KEY="sk-ant-xxxx" \ +sforge run --backend e2b \ + --task ad_placement_optimization --agent claude-code \ + --model claude-opus-4-8[1m] \ + --timeout 7200 \ + --judge-url http://YOUR_PUBLIC_HOST:8080 \ + --run-id ad-placement-optimization-e2b-001 +``` + +这会在 E2B Work Sandbox 中启动 Claude Opus 4.8 处理该任务，持续 2 小时。 +你会在 stdout 中实时看到 Agent 的工作输出。 + +`--backend e2b` 下每个任务独立评测；传入多个 task ID 只是调度多套互不相关的 Sandbox。 + +### 6. 查看结果 + +通过内置 Web UI 实时查看评分进展： + +```bash +sforge visualizer +# 打开 http://127.0.0.1:8000 +``` + +也可以直接查看文件： + +```bash +ls logs/runs/*/ad_placement_optimization/ +cat logs/runs/*/ad_placement_optimization/final_result.json +``` + +运行结束后 Sandbox 会自动删除。如果超过一分钟仍有残留，见[常见问题](/zh/configuration/container-backends#常见问题)。 + + +## 使用第三方模型 + +模型路由和上下文窗口的配置与 Docker 示例完全相同，只需改后端参数。按照 +[使用第三方模型](/zh/examples/single-task-docker#使用第三方模型)配置，并在 +`sforge run` 命令中加上 `--backend e2b --judge-url http://YOUR_PUBLIC_HOST:8080`，例如： + +```bash +export SFORGE_AGENT_API_KEY="your-deepseek-key" +export SFORGE_AGENT_API_BASE_URL="https://api.deepseek.com/anthropic" +export SFORGE_CLAUDE_CACHE_OPT=1 +export SFORGE_AGENT_EXTRA_ENV="ANTHROPIC_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_OPUS_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_SONNET_MODEL=deepseek-v4-pro[1m],ANTHROPIC_DEFAULT_HAIKU_MODEL=deepseek-v4-pro[1m],CLAUDE_CODE_SUBAGENT_MODEL=deepseek-v4-pro[1m]" + +sforge run --backend e2b \ + --task ad_placement_optimization --agent claude-code \ + --model deepseek-v4-pro[1m] \ + --timeout 7200 \ + --judge-url http://YOUR_PUBLIC_HOST:8080 \ + --run-id ad-placement-deepseek-e2b-001 +``` + + +## 网络隔离 + +任务 JSON 中的 `internet` 字段以及 `--disable-internet` / `--enable-internet` +参数与 Docker 下用法一致。E2B 上 SForge 通过 E2B 原生的 `update_network` API +下发白名单，而不是宿主机 iptables，因此本机不会留下任何残留。E2B 规则是主机级 +而非端口级：放行一个域名即放行它的所有端口。 + +宿主机的 `HTTP_PROXY` / `HTTPS_PROXY` **不会**转发进 Sandbox。如果模型 API +必须走代理，该代理需要公网可达，并通过 `SFORGE_HTTP_PROXY` / +`SFORGE_HTTPS_PROXY` 显式指定。详见[网络隔离](/zh/features/network-isolation#e2b-backend)。 + + +## LLM 评分任务 + +`college_english_exam_bank` 这类 LLM 评分任务需要在 Judge 环境中提供评分用凭证， +与 Docker 下相同。**在启动 Judge 服务器前**设置 `SFORGE_JUDGE_EXTRA_ENV`，变量 +说明见 [LLM 评分任务](/zh/examples/single-task-docker#llm-评分任务)。 diff --git a/docs/zh/guide/agents.md b/docs/zh/guide/agents.md index 48c3012..f5ba000 100644 --- a/docs/zh/guide/agents.md +++ b/docs/zh/guide/agents.md @@ -10,17 +10,40 @@ SForge 内置了以下 Agent： |-----------|----------|-----------------|-------------|---------|-----------|-------------| | Claude Code | `claude-code` | `ANTHROPIC_AUTH_TOKEN` | `ANTHROPIC_MODEL` | — | 支持 | 支持 | | Codex | `codex` | `CODEX_API_KEY` | `CODEX_MODEL` | — | 支持 | 支持 | -| opencode | `opencode` | `ANTHROPIC_API_KEY` | — | — | 不支持（靠自动恢复） | 支持 | +| OpenCode | `opencode` | `ANTHROPIC_API_KEY` | — | — | 不支持 | 支持 | 使用 `--agent` 参数指定要运行的 Agent： ```bash sforge run --task ad_placement_optimization --agent claude-code sforge run --task ad_placement_optimization --agent codex -sforge run --task ad_placement_optimization --agent opencode +sforge run --task ad_placement_optimization --agent OpenCode ``` -> **opencode 说明**：模型名格式为 `provider/model`（裸模型名会视为 `anthropic/`）。默认模型、自定义 base URL（`SFORGE_AGENT_API_BASE_URL`）和权限均通过 `OPENCODE_CONFIG_CONTENT` 环境变量注入，容器内不写配置文件。opencode 无 Stop Hook 机制，长时运行依赖自动恢复循环（`opencode run --continue`）。 +::: details 固定版本 +每个 Agent 在 Work 容器内安装固定版本： + +| CLI 名称 | 安装版本 | +|----------|----------| +| `claude-code` | Claude Code 2.1.159 | +| `claude-code-2.1.214` | Claude Code 2.1.214 | +| `codex` | Codex 0.130.0 | +| `codex-0.145.0` | Codex 0.145.0 | +| `opencode` | OpenCode 1.18.2 | + +`claude-code-2.1.214` 和 `codex-0.145.0` 是更新的固定版本，作为独立 Agent 保留以保证 +既有结果可比。Claude 5 系列模型（Fable/Mythos）需要使用 `claude-code-2.1.214`。 +::: + +::: details OpenCode 说明 +- 模型名格式为 `provider/model`，裸模型名会视为 `anthropic/`。 +- 默认模型、自定义 base URL（`SFORGE_AGENT_API_BASE_URL`）和权限均通过 + `OPENCODE_CONFIG_CONTENT` 环境变量注入，容器内不写配置文件。 +- `SFORGE_AGENT_API_BASE_URL` 沿用 Claude Code 约定（不带 `/v1`），SForge 会为 + OpenCode 的 AI SDK 客户端自动补上 `/v1`。 +- 任务断网时会禁用 `webfetch` 和 `websearch`，避免模型在被阻断的请求上浪费轮次。 +- OpenCode 无 Stop Hook 机制，长时运行依赖自动恢复循环（`opencode run --continue`）。 +::: ## Agent 配置 @@ -59,7 +82,7 @@ sforge run --task ad_placement_optimization --agent claude-code --model claude-o |-------|----------|------| | Claude Code | `CLAUDE_CODE_EFFORT_LEVEL` | 直接使用所选值 | | Codex | `model_reasoning_effort` | 将 `max` 映射为 Codex 的 `xhigh` | -| opencode | `reasoningEffort` | 对非 Anthropic provider 生效；Anthropic provider 保持自身 thinking 配置 | +| OpenCode | `reasoningEffort` | 对非 Anthropic provider 生效；Anthropic provider 保持自身 thinking 配置 | ### 额外环境变量 diff --git a/docs/zh/guide/getting-started.md b/docs/zh/guide/getting-started.md index 1e4739d..922d6be 100644 --- a/docs/zh/guide/getting-started.md +++ b/docs/zh/guide/getting-started.md @@ -34,6 +34,8 @@ Docker backend 只适合小批量任务。在单机上并发运行大量任务 pip install sforge ``` +如需使用 [E2B 后端](/zh/configuration/container-backends#e2b-后端)，请带上 `e2b` extra：`pip install "sforge[e2b]"`。 + 也可以从源码安装： ```bash diff --git a/docs/zh/reference/cli.md b/docs/zh/reference/cli.md index f41de47..cecf27b 100644 --- a/docs/zh/reference/cli.md +++ b/docs/zh/reference/cli.md @@ -135,7 +135,7 @@ sforge run --experiment experiment.yaml | 选项 | 默认值 | 说明 | |------|--------|------| | `--task` | 必填* | 一个或多个任务 ID（空格分隔）。传入多个 ID 时，CLI 批量调度多个独立 task，不会把它们合并为一次评测。 | -| `--agent` | 必填* | Agent 名称（如 `claude-code`、`codex`）。除非指定 `--experiment`，否则必填。 | +| `--agent` | 必填* | Agent 名称：`claude-code`、`codex`、`opencode`，或固定版本变体（`claude-code-2.1.214`、`codex-0.145.0`）。除非指定 `--experiment`，否则必填。 | | `--experiment` | --- | 实验 YAML 配置文件路径。如果未指定 `--task`，运行 YAML 中的全部任务；如果指定了 `--task`，则只运行该子集并套用实验配置。 | | `--model` | --- | 模型覆盖（如 `claude-opus-4-8`） | | `--effort` | Agent 默认值 | 推理强度（`low`、`medium`、`high` 或 `max`）。SForge 会在所选 Agent 支持时映射到其原生配置。 | @@ -154,7 +154,7 @@ sforge run --experiment experiment.yaml | `--disable-stop-hook` | `false` | 禁用 stop hook（允许 Agent 正常退出） | | `--disable-auto-eval` | `false` | 禁用后台自动评测守护进程 | | `--disable-auto-resume` | `false` | 禁用 Agent 异常退出时的自动恢复 | -| `--disable-internet` | `false` | 强制网络隔离（仅允许 Judge 服务器 + API 访问）。需要 `sudo` 权限来配置 iptables。与 `--enable-internet` 互斥。 | +| `--disable-internet` | `false` | 强制网络隔离（仅允许 Judge 服务器 + API 访问）。Docker 后端需要免密 `sudo` 配置 iptables；k8s 和 e2b 在集群/服务商侧隔离，不需要。与 `--enable-internet` 互斥。 | | `--enable-internet` | `false` | 强制开启完整网络访问（覆盖任务的 `internet: false` 设置）。与 `--disable-internet` 互斥。 | ::: warning 必填参数