From ab6b83191bc74b01ed2f94acc422e75775b76259 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 15:40:52 -0700 Subject: [PATCH 1/9] Import AgentEnv bundles with openenv import Add an agentenv importer beside ORS/OpenReward and Verifiers. It detects AgentEnv bundles (folders whose tasks/ holds task files) by reading JSON only, vendors the bundle under its own folder name, and generates a server/app.py that serves it with agentenv-framework's build_app. prepare_import_package takes an optional vendor_dir and append_dependency_files an optional requires_python, so the generated package keeps the bundle's name and locks against agentenv-framework. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/source/guides/first-environment.md | 2 +- docs/source/reference/cli.md | 23 ++- src/openenv/cli/importers/__init__.py | 4 +- src/openenv/cli/importers/agentenv.py | 103 ++++++++++++++ src/openenv/cli/importers/base.py | 15 +- .../importers/templates/agentenv_app.py.tpl | 26 ++++ tests/test_cli/test_import_env.py | 133 ++++++++++++++++++ 8 files changed, 301 insertions(+), 7 deletions(-) create mode 100644 src/openenv/cli/importers/agentenv.py create mode 100644 src/openenv/cli/importers/templates/agentenv_app.py.tpl diff --git a/README.md b/README.md index 04d5e12466..f942c94dfd 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ openenv validate my_env --level static --skip-build # quick check against the openenv push my_env # deploy it to Hugging Face Spaces ``` -See [Your First Environment](https://huggingface.co/docs/openenv/guides/first-environment) and [Deploying an Environment](https://huggingface.co/docs/openenv/getting_started/environment-builder). `openenv import` wraps an existing environment from ORS/OpenReward or Verifiers. +See [Your First Environment](https://huggingface.co/docs/openenv/guides/first-environment) and [Deploying an Environment](https://huggingface.co/docs/openenv/getting_started/environment-builder). `openenv import` wraps an existing environment from ORS/OpenReward or Verifiers, or an AgentEnv bundle. ## Environments diff --git a/docs/source/guides/first-environment.md b/docs/source/guides/first-environment.md index 2adb451da6..fddaedd993 100644 --- a/docs/source/guides/first-environment.md +++ b/docs/source/guides/first-environment.md @@ -33,7 +33,7 @@ my_env/ Class names come from the environment name, with a trailing `_env` dropped: `my_env` gives `MyAction`, `MyObservation`, `MyEnvironment` and `MyEnv`. The steps below go through each file. Edit them to replace the echo logic with yours. If you're working inside the OpenEnv repo, move the folder under `envs/`. > [!TIP] -> If you already have an ORS/OpenReward or Prime Intellect Verifiers environment, run `openenv import SOURCE --name my_env --output-dir DIR` instead. It detects the source type, vendors the source under the generated package and emits an OpenEnv wrapper with task/split and MCP-style tool actions. Non-secret data files and portable dependencies from the source tree are carried over. +> If you already have an ORS/OpenReward or Prime Intellect Verifiers environment, or an AgentEnv bundle, run `openenv import SOURCE --name my_env --output-dir DIR` instead. It detects the source type, vendors the source under the generated package and emits an OpenEnv wrapper with task/split and MCP-style tool actions. Non-secret data files and portable dependencies from the source tree are carried over. ## 2. Define the models diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index e9abcdca31..5462a1ad51 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -10,8 +10,8 @@ The `openenv` CLI provides a set of commands for building, validating, and pushi Import a supported third-party source environment into a generated OpenEnv wrapper package. The command detects the source format from the directory -contents, so ORS/OpenReward and Prime Intellect Verifiers sources do not -require `--type` in the common case. +contents, so ORS/OpenReward, Prime Intellect Verifiers and AgentEnv sources do +not require `--type` in the common case. The generated wrapper vendors the source tree into the package and includes vendored files as package data, so non-secret fixture/data files are available to @@ -27,6 +27,25 @@ openenv import path/to/source --name my_env --output-dir ./envs openenv import path/to/source --name my_env --output-dir ./envs --env-class MyEnv ``` +An [AgentEnv](https://github.com/scaleapi/agentenv-framework) bundle is a folder +whose `tasks/` holds task files, each a JSON list of steps. The importer finds +every bundle under the source; when there are several, pick one with +`--env-class` and the bundle's folder name. The generated `server/app.py` serves +the vendored bundle with `build_app` from `agentenv-framework`, which the package +depends on: each session plays one task, `reset(split=..., index=...)` deploys +the task's environment, the tools are the environment's MCP tools plus +`end_turn`, and `end_turn` returns the task's score as the reward. The split +`all` holds every task, and each of the bundle's evals is a split of its own. +Episodes deploy where agent-env's sandbox configuration says, local Docker by +default, so the server needs a Docker daemon or a remote sandbox provider. A +bundle that uses types from an agent-env plugin needs the plugin listed in the +source's `requirements.txt` or `pyproject.toml`. + +```bash +openenv import path/to/bundle --name my_env --output-dir ./envs +openenv import path/to/plugin --name my_env --output-dir ./envs --env-class my-bundle +``` + [[autodoc]] openenv.cli.commands.import_env.import_env ## `openenv build` diff --git a/src/openenv/cli/importers/__init__.py b/src/openenv/cli/importers/__init__.py index 9519278cfd..ba216e8971 100644 --- a/src/openenv/cli/importers/__init__.py +++ b/src/openenv/cli/importers/__init__.py @@ -1,12 +1,14 @@ """Deterministic source importers for OpenEnv environments.""" +from .agentenv import AgentEnvImporter from .base import DetectedEnvironment, EnvironmentImporter, ImporterRegistry from .ors import ORSImporter from .verifiers import VerifiersImporter -DEFAULT_IMPORTERS = [ORSImporter(), VerifiersImporter()] +DEFAULT_IMPORTERS = [ORSImporter(), VerifiersImporter(), AgentEnvImporter()] __all__ = [ + "AgentEnvImporter", "DEFAULT_IMPORTERS", "DetectedEnvironment", "EnvironmentImporter", diff --git a/src/openenv/cli/importers/agentenv.py b/src/openenv/cli/importers/agentenv.py new file mode 100644 index 0000000000..d53c0df2ae --- /dev/null +++ b/src/openenv/cli/importers/agentenv.py @@ -0,0 +1,103 @@ +"""AgentEnv bundle source importer.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from .base import ( + append_dependency_files, + collect_source_dependencies, + DetectedEnvironment, + iter_directories, + prepare_import_package, + render_importer_template, + write_text, +) + + +_AGENTENV_DEPENDENCY = "agentenv-framework>=0.9.1360" +_REQUIRES_PYTHON = ">=3.11.4" + + +def _is_task_file(path: Path) -> bool: + try: + steps = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, ValueError): + return False + return ( + isinstance(steps, list) + and bool(steps) + and all( + isinstance(step, dict) and isinstance(step.get("type"), str) + for step in steps + ) + ) + + +def detect_agentenv_bundles(source: Path) -> list[DetectedEnvironment]: + """Detect AgentEnv bundles, folders whose tasks/ holds task files, without importing source.""" + source = source.resolve() + matches: list[DetectedEnvironment] = [] + + for tasks_dir in iter_directories(source, "tasks"): + if not any(_is_task_file(path) for path in sorted(tasks_dir.glob("*.json"))): + continue + bundle = tasks_dir.parent + matches.append( + DetectedEnvironment( + source_type="agentenv", + class_name=bundle.name, + module_path="" + if bundle == source + else bundle.relative_to(source).as_posix(), + file_path=bundle, + ) + ) + + return matches + + +def _app_source(*, vendor_dir: str, bundle_path: str) -> str: + return render_importer_template( + "agentenv_app.py.tpl", + bundle=f"{vendor_dir}/{bundle_path}" if bundle_path else vendor_dir, + ) + + +class AgentEnvImporter: + """Importer for AgentEnv bundles, served by agentenv-framework's OpenEnv server.""" + + source_type = "agentenv" + + def detect(self, source: Path) -> list[DetectedEnvironment]: + return detect_agentenv_bundles(source) + + def generate( + self, + *, + source: Path, + destination: Path, + env_name: str, + detected: DetectedEnvironment, + ) -> None: + package = prepare_import_package( + source, destination, env_name, vendor_dir=source.name + ) + (destination / "server" / f"{env_name}_environment.py").unlink() + write_text( + destination / "server" / "__init__.py", + f'"""{env_name} environment server: an AgentEnv bundle."""\n', + ) + write_text( + destination / "server" / "app.py", + _app_source( + vendor_dir=package.vendor_dir, bundle_path=detected.module_path + ), + ) + dependencies = collect_source_dependencies(source) + if _AGENTENV_DEPENDENCY not in dependencies: + dependencies.append(_AGENTENV_DEPENDENCY) + append_dependency_files( + destination, env_name, dependencies, requires_python=_REQUIRES_PYTHON + ) diff --git a/src/openenv/cli/importers/base.py b/src/openenv/cli/importers/base.py index f106af091e..ab2209b054 100644 --- a/src/openenv/cli/importers/base.py +++ b/src/openenv/cli/importers/base.py @@ -133,6 +133,14 @@ def iter_python_files(source: Path) -> list[Path]: ] +def iter_directories(source: Path, name: str) -> list[Path]: + return [ + path + for path in sorted(source.rglob(name)) + if path.is_dir() and not _is_excluded(path.relative_to(source)) + ] + + def module_path(source: Path, file_path: Path) -> str: rel = file_path.relative_to(source).with_suffix("") parts = list(rel.parts) @@ -196,6 +204,7 @@ def append_dependency_files( env_dir: Path, env_name: str, dependencies: list[str], + requires_python: str | None = None, ) -> None: requirements = env_dir / "server" / "requirements.txt" if requirements.exists(): @@ -231,6 +240,8 @@ def append_dependency_files( project_dependencies.append(dependency) existing.add(_dependency_name(dependency)) project["dependencies"] = project_dependencies + if requires_python: + project["requires-python"] = requires_python tool = data.setdefault("tool", {}) setuptools = tool.setdefault("setuptools", {}) @@ -266,7 +277,7 @@ class ImportPackageContext: def prepare_import_package( - source: Path, destination: Path, env_name: str + source: Path, destination: Path, env_name: str, vendor_dir: str | None = None ) -> ImportPackageContext: from openenv.cli.commands.init import ( _copy_template_directory, @@ -282,7 +293,7 @@ def prepare_import_package( env_name, ) - vendor_dir = safe_vendor_dir_name(source) + vendor_dir = vendor_dir or safe_vendor_dir_name(source) vendor_path = destination / "vendor" / vendor_dir copy_source_tree(source, vendor_path) ensure_vendor_package(vendor_path) diff --git a/src/openenv/cli/importers/templates/agentenv_app.py.tpl b/src/openenv/cli/importers/templates/agentenv_app.py.tpl new file mode 100644 index 0000000000..d42160f8b3 --- /dev/null +++ b/src/openenv/cli/importers/templates/agentenv_app.py.tpl @@ -0,0 +1,26 @@ +from __future__ import annotations + +from pathlib import Path + +from agent_env.openenv import build_app + + +BUNDLE = Path(__file__).resolve().parents[1] / "vendor" / "${bundle}" + +app = build_app(BUNDLE) + + +def main(): + import argparse + + import uvicorn + + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=8000) + args = parser.parse_args() + uvicorn.run(app, host=args.host, port=args.port) + + +if __name__ == "__main__": + main() diff --git a/tests/test_cli/test_import_env.py b/tests/test_cli/test_import_env.py index 35c3b06239..81d63abbc8 100644 --- a/tests/test_cli/test_import_env.py +++ b/tests/test_cli/test_import_env.py @@ -9,6 +9,7 @@ from __future__ import annotations import importlib.util +import json import logging import sys from pathlib import Path @@ -17,6 +18,7 @@ import pytest from fastapi.testclient import TestClient from openenv.cli.__main__ import app +from openenv.cli.importers.agentenv import detect_agentenv_bundles from openenv.cli.importers.base import safe_vendor_dir_name from openenv.cli.importers.ors import detect_ors_dependencies, detect_ors_environments from openenv.cli.importers.verifiers import detect_verifiers_environments @@ -218,6 +220,39 @@ def load_environment() -> vf.Environment: ) +def _write_agentenv_bundle(root: Path) -> None: + (root / "envs" / "echo").mkdir(parents=True) + (root / "envs" / "echo" / "server.py").write_text( + 'raise RuntimeError("should not import")\n', encoding="utf-8" + ) + (root / "tasks").mkdir() + steps = [ + {"id": "env", "type": "deploy_env", "env_id": "echo"}, + {"id": "policy", "type": "deploy_agent", "agent_name": "policy"}, + {"id": "play", "type": "prompt_agent", "agent_name": "policy"}, + ] + (root / "tasks" / "ping.json").write_text(json.dumps(steps), encoding="utf-8") + + +def _write_fake_agentenv_sdk(root: Path) -> None: + package = root / "agent_env" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "openenv.py").write_text( + """ +from fastapi import FastAPI + +served = [] + + +def build_app(bundle): + served.append(bundle) + return FastAPI() +""".lstrip(), + encoding="utf-8", + ) + + def test_ors_detector_finds_environment_class_without_importing_source( tmp_path: Path, ) -> None: @@ -846,3 +881,101 @@ def test_imported_server_main_parses_host_and_port( importlib.import_module(f"{env_name}.server.app").main() assert calls == [{"host": "127.0.0.1", "port": 8001}] + + +def test_agentenv_detector_finds_bundles_without_importing_source( + tmp_path: Path, +) -> None: + source = tmp_path / "source" + _write_agentenv_bundle(source / "bundles" / "desk") + _write_agentenv_bundle(source / ".venv" / "bundles" / "installed") + (source / "notes" / "tasks").mkdir(parents=True) + (source / "notes" / "tasks" / "todo.json").write_text( + '{"title": "not a task"}', encoding="utf-8" + ) + + matches = detect_agentenv_bundles(source) + + assert [(m.source_type, m.class_name, m.module_path) for m in matches] == [ + ("agentenv", "desk", "bundles/desk") + ] + + +def test_import_command_detects_agentenv_bundle_and_generates_server( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + import uvicorn + + source = tmp_path / "echo-desk" + _write_agentenv_bundle(source) + (source / "requirements.txt").write_text("numpy>=1.26\n", encoding="utf-8") + output_dir = tmp_path / "out" + + with patch("openenv.cli.commands.import_env._generate_uv_lock", return_value=True): + result = runner.invoke( + app, + [ + "import", + str(source), + "--name", + "desk_env", + "--output-dir", + str(output_dir), + ], + ) + + assert result.exit_code == 0, result.output + env_dir = output_dir / "desk_env" + bundle = env_dir / "vendor" / "echo-desk" + assert (bundle / "tasks" / "ping.json").exists() + assert not (env_dir / "server" / "desk_env_environment.py").exists() + pyproject = (env_dir / "pyproject.toml").read_text(encoding="utf-8") + requirements = (env_dir / "server" / "requirements.txt").read_text(encoding="utf-8") + for text in (pyproject, requirements): + assert "agentenv-framework>=0.9.1360" in text + assert "numpy>=1.26" in text + assert 'requires-python = ">=3.11.4"' in pyproject + assert "vendor/**/*" in pyproject + + sdk = tmp_path / "sdk" + _write_fake_agentenv_sdk(sdk) + calls = [] + monkeypatch.setattr(uvicorn, "run", lambda app, **kwargs: calls.append(kwargs)) + monkeypatch.setattr(sys, "argv", ["server", "--port", "8001"]) + sys.path[:0] = [str(sdk), str(output_dir)] + try: + generated = importlib.import_module("desk_env.server.app") + served = importlib.import_module("agent_env.openenv").served + + assert served == [bundle.resolve()] + assert TestClient(generated.app).get("/docs").status_code == 200 + generated.main() + assert calls == [{"host": "0.0.0.0", "port": 8001}] + finally: + sys.path.remove(str(sdk)) + sys.path.remove(str(output_dir)) + for name in list(sys.modules): + if name.split(".", 1)[0] in {"agent_env", "desk_env"}: + sys.modules.pop(name) + + +def test_import_command_requires_env_class_when_multiple_agentenv_bundles( + tmp_path: Path, +) -> None: + source = tmp_path / "plugin" + _write_agentenv_bundle(source / "bundles" / "train") + _write_agentenv_bundle(source / "bundles" / "eval") + output_dir = tmp_path / "out" + args = ["import", str(source), "--name", "picked_env", "--output-dir"] + + with patch("openenv.cli.commands.import_env._generate_uv_lock", return_value=True): + ambiguous = runner.invoke(app, [*args, str(output_dir)]) + picked = runner.invoke(app, [*args, str(output_dir), "--env-class", "train"]) + + assert ambiguous.exit_code != 0 + assert "Multiple environment entrypoints" in ambiguous.output + assert picked.exit_code == 0, picked.output + app_source = (output_dir / "picked_env" / "server" / "app.py").read_text( + encoding="utf-8" + ) + assert '"vendor" / "plugin/bundles/train"' in app_source From 77369a4da3e08efb73e8a47adf18f92e88934413 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 17:17:28 -0700 Subject: [PATCH 2/9] Vendor only the picked AgentEnv bundle and keep the framework floor The agentenv importer copied the whole source tree even when --env-class picked one bundle out of a repository, carrying unrelated files and local config into the package; it now vendors the bundle folder alone and takes dependencies from the bundle's own requirements. An agentenv-framework entry the bundle already lists now gets the >=0.9.1360 floor merged into it rather than replacing it. The CLI reference says the server needs local Docker to build a bundle's environment images, and that the generated image can't serve a bundle yet. Co-Authored-By: Claude Opus 5.5 --- docs/source/reference/cli.md | 24 +++++++---- src/openenv/cli/importers/agentenv.py | 42 ++++++++++++------- .../importers/templates/agentenv_app.py.tpl | 2 +- tests/test_cli/test_import_env.py | 34 ++++++++++++--- 4 files changed, 73 insertions(+), 29 deletions(-) diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index 5462a1ad51..0651d573ae 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -30,16 +30,24 @@ openenv import path/to/source --name my_env --output-dir ./envs --env-class MyEn An [AgentEnv](https://github.com/scaleapi/agentenv-framework) bundle is a folder whose `tasks/` holds task files, each a JSON list of steps. The importer finds every bundle under the source; when there are several, pick one with -`--env-class` and the bundle's folder name. The generated `server/app.py` serves -the vendored bundle with `build_app` from `agentenv-framework`, which the package -depends on: each session plays one task, `reset(split=..., index=...)` deploys -the task's environment, the tools are the environment's MCP tools plus +`--env-class` and the bundle's folder name. Only that folder is vendored, and the +package's dependencies are the ones the bundle declares in its own +`requirements.txt` or `pyproject.toml`, plus `agentenv-framework`. Everything the +bundle's tasks use has to be in the bundle or built into agent-env, so a bundle +whose environments or verifiers a plugin registers by other means won't serve +from the package alone. + +The generated `server/app.py` serves the bundle with `build_app` from +`agentenv-framework`: each session plays one task, `reset(split=..., index=...)` +deploys the task's environment, the tools are the environment's MCP tools plus `end_turn`, and `end_turn` returns the task's score as the reward. The split `all` holds every task, and each of the bundle's evals is a split of its own. -Episodes deploy where agent-env's sandbox configuration says, local Docker by -default, so the server needs a Docker daemon or a remote sandbox provider. A -bundle that uses types from an agent-env plugin needs the plugin listed in the -source's `requirements.txt` or `pyproject.toml`. + +The server prepares the bundle when it starts, building with the local Docker +any environment image the bundle defines with a Dockerfile, and deploys each +episode where agent-env's sandbox configuration says, local Docker by default. +Run it on a host with Docker. The generated `server/Dockerfile` has no Docker +inside, so the image it builds can't serve a bundle yet. ```bash openenv import path/to/bundle --name my_env --output-dir ./envs diff --git a/src/openenv/cli/importers/agentenv.py b/src/openenv/cli/importers/agentenv.py index d53c0df2ae..4cf31b17f8 100644 --- a/src/openenv/cli/importers/agentenv.py +++ b/src/openenv/cli/importers/agentenv.py @@ -5,6 +5,10 @@ import json from pathlib import Path +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name + from .base import ( append_dependency_files, collect_source_dependencies, @@ -16,7 +20,8 @@ ) -_AGENTENV_DEPENDENCY = "agentenv-framework>=0.9.1360" +_FRAMEWORK = "agentenv-framework" +_FRAMEWORK_FLOOR = ">=0.9.1360" _REQUIRES_PYTHON = ">=3.11.4" @@ -58,11 +63,19 @@ def detect_agentenv_bundles(source: Path) -> list[DetectedEnvironment]: return matches -def _app_source(*, vendor_dir: str, bundle_path: str) -> str: - return render_importer_template( - "agentenv_app.py.tpl", - bundle=f"{vendor_dir}/{bundle_path}" if bundle_path else vendor_dir, - ) +def _with_framework_floor(dependencies: list[str]) -> list[str]: + """The bundle's dependencies, with agentenv-framework at least the version that serves bundles.""" + for index, dependency in enumerate(dependencies): + try: + requirement = Requirement(dependency) + except InvalidRequirement: + continue + if canonicalize_name(requirement.name) == _FRAMEWORK: + if requirement.url is None: + requirement.specifier &= SpecifierSet(_FRAMEWORK_FLOOR) + dependencies[index] = str(requirement) + return dependencies + return [*dependencies, f"{_FRAMEWORK}{_FRAMEWORK_FLOOR}"] class AgentEnvImporter: @@ -81,9 +94,8 @@ def generate( env_name: str, detected: DetectedEnvironment, ) -> None: - package = prepare_import_package( - source, destination, env_name, vendor_dir=source.name - ) + bundle = detected.file_path + prepare_import_package(bundle, destination, env_name, vendor_dir=bundle.name) (destination / "server" / f"{env_name}_environment.py").unlink() write_text( destination / "server" / "__init__.py", @@ -91,13 +103,13 @@ def generate( ) write_text( destination / "server" / "app.py", - _app_source( - vendor_dir=package.vendor_dir, bundle_path=detected.module_path + render_importer_template( + "agentenv_app.py.tpl", bundle=json.dumps(bundle.name) ), ) - dependencies = collect_source_dependencies(source) - if _AGENTENV_DEPENDENCY not in dependencies: - dependencies.append(_AGENTENV_DEPENDENCY) append_dependency_files( - destination, env_name, dependencies, requires_python=_REQUIRES_PYTHON + destination, + env_name, + _with_framework_floor(collect_source_dependencies(bundle)), + requires_python=_REQUIRES_PYTHON, ) diff --git a/src/openenv/cli/importers/templates/agentenv_app.py.tpl b/src/openenv/cli/importers/templates/agentenv_app.py.tpl index d42160f8b3..60b592c0b3 100644 --- a/src/openenv/cli/importers/templates/agentenv_app.py.tpl +++ b/src/openenv/cli/importers/templates/agentenv_app.py.tpl @@ -5,7 +5,7 @@ from pathlib import Path from agent_env.openenv import build_app -BUNDLE = Path(__file__).resolve().parents[1] / "vendor" / "${bundle}" +BUNDLE = Path(__file__).resolve().parents[1] / "vendor" / ${bundle} app = build_app(BUNDLE) diff --git a/tests/test_cli/test_import_env.py b/tests/test_cli/test_import_env.py index 81d63abbc8..7203f37e73 100644 --- a/tests/test_cli/test_import_env.py +++ b/tests/test_cli/test_import_env.py @@ -908,7 +908,9 @@ def test_import_command_detects_agentenv_bundle_and_generates_server( source = tmp_path / "echo-desk" _write_agentenv_bundle(source) - (source / "requirements.txt").write_text("numpy>=1.26\n", encoding="utf-8") + (source / "requirements.txt").write_text( + "numpy>=1.26\nagentenv-framework\n", encoding="utf-8" + ) output_dir = tmp_path / "out" with patch("openenv.cli.commands.import_env._generate_uv_lock", return_value=True): @@ -933,6 +935,7 @@ def test_import_command_detects_agentenv_bundle_and_generates_server( requirements = (env_dir / "server" / "requirements.txt").read_text(encoding="utf-8") for text in (pyproject, requirements): assert "agentenv-framework>=0.9.1360" in text + assert text.count("agentenv-framework") == 1 assert "numpy>=1.26" in text assert 'requires-python = ">=3.11.4"' in pyproject assert "vendor/**/*" in pyproject @@ -959,12 +962,18 @@ def test_import_command_detects_agentenv_bundle_and_generates_server( sys.modules.pop(name) -def test_import_command_requires_env_class_when_multiple_agentenv_bundles( +def test_import_command_vendors_only_the_agentenv_bundle_it_picks( tmp_path: Path, ) -> None: source = tmp_path / "plugin" _write_agentenv_bundle(source / "bundles" / "train") _write_agentenv_bundle(source / "bundles" / "eval") + (source / "bundles" / "train" / "requirements.txt").write_text( + "agentenv-framework>=0.9.1400\nnumpy>=1.26\n", encoding="utf-8" + ) + (source / "requirements.txt").write_text("torch>=2.0\n", encoding="utf-8") + (source / ".agentenv").mkdir() + (source / ".agentenv" / "config.toml").write_text("[stores]\n", encoding="utf-8") output_dir = tmp_path / "out" args = ["import", str(source), "--name", "picked_env", "--output-dir"] @@ -975,7 +984,22 @@ def test_import_command_requires_env_class_when_multiple_agentenv_bundles( assert ambiguous.exit_code != 0 assert "Multiple environment entrypoints" in ambiguous.output assert picked.exit_code == 0, picked.output - app_source = (output_dir / "picked_env" / "server" / "app.py").read_text( - encoding="utf-8" + env_dir = output_dir / "picked_env" + vendored = sorted( + path.relative_to(env_dir / "vendor").as_posix() + for path in (env_dir / "vendor").rglob("*") + if path.is_file() ) - assert '"vendor" / "plugin/bundles/train"' in app_source + assert vendored == [ + "__init__.py", + "train/__init__.py", + "train/envs/echo/server.py", + "train/requirements.txt", + "train/tasks/ping.json", + ] + app_source = (env_dir / "server" / "app.py").read_text(encoding="utf-8") + assert '"vendor" / "train"' in app_source + requirements = (env_dir / "server" / "requirements.txt").read_text(encoding="utf-8") + assert "agentenv-framework>=0.9.1360,>=0.9.1400" in requirements + assert "numpy>=1.26" in requirements + assert "torch" not in requirements From 4319a102617e3540c0694c3176a493e8e33cca80 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 19:49:02 -0700 Subject: [PATCH 3/9] Point the AgentEnv import docs at AgentEnv's own page Keep the CLI reference to what the importer detects and vendors, and link to AgentEnv's OpenEnv page for what the served env does and needs, so details that change with agentenv-framework are documented there. Co-Authored-By: Claude Opus 5.5 --- docs/source/reference/cli.md | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index 0651d573ae..ac2d281ec8 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -28,26 +28,12 @@ openenv import path/to/source --name my_env --output-dir ./envs --env-class MyEn ``` An [AgentEnv](https://github.com/scaleapi/agentenv-framework) bundle is a folder -whose `tasks/` holds task files, each a JSON list of steps. The importer finds -every bundle under the source; when there are several, pick one with -`--env-class` and the bundle's folder name. Only that folder is vendored, and the -package's dependencies are the ones the bundle declares in its own -`requirements.txt` or `pyproject.toml`, plus `agentenv-framework`. Everything the -bundle's tasks use has to be in the bundle or built into agent-env, so a bundle -whose environments or verifiers a plugin registers by other means won't serve -from the package alone. - -The generated `server/app.py` serves the bundle with `build_app` from -`agentenv-framework`: each session plays one task, `reset(split=..., index=...)` -deploys the task's environment, the tools are the environment's MCP tools plus -`end_turn`, and `end_turn` returns the task's score as the reward. The split -`all` holds every task, and each of the bundle's evals is a split of its own. - -The server prepares the bundle when it starts, building with the local Docker -any environment image the bundle defines with a Dockerfile, and deploys each -episode where agent-env's sandbox configuration says, local Docker by default. -Run it on a host with Docker. The generated `server/Dockerfile` has no Docker -inside, so the image it builds can't serve a bundle yet. +whose `tasks/` holds task files. When a source holds several bundles, pick one +with `--env-class` and its folder name; only that folder is vendored. The +generated server runs it with `build_app` from `agentenv-framework`, which by +default deploys each episode on the local Docker. See +[AgentEnv's OpenEnv page](https://www.agentenvframework.com/docs/openenv) for +what it serves and what it needs. ```bash openenv import path/to/bundle --name my_env --output-dir ./envs From 3294418cbc08a617176a573a80dec68ae960bb5b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 26 Oct 2026 22:45:30 -0700 Subject: [PATCH 4/9] Depend on agentenv-framework's openenv extra in imported bundles agentenv-framework 0.9.1364 makes OpenEnv an optional extra, so the generated package asks for agentenv-framework[openenv]>=0.9.1364, and an agentenv-framework entry the bundle already lists gets the extra added along with the floor. Co-Authored-By: Claude Opus 5.5 --- src/openenv/cli/importers/agentenv.py | 10 ++++++---- tests/test_cli/test_import_env.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/openenv/cli/importers/agentenv.py b/src/openenv/cli/importers/agentenv.py index 4cf31b17f8..a7f7199459 100644 --- a/src/openenv/cli/importers/agentenv.py +++ b/src/openenv/cli/importers/agentenv.py @@ -21,7 +21,8 @@ _FRAMEWORK = "agentenv-framework" -_FRAMEWORK_FLOOR = ">=0.9.1360" +_FRAMEWORK_EXTRA = "openenv" +_FRAMEWORK_FLOOR = ">=0.9.1364" _REQUIRES_PYTHON = ">=3.11.4" @@ -64,18 +65,19 @@ def detect_agentenv_bundles(source: Path) -> list[DetectedEnvironment]: def _with_framework_floor(dependencies: list[str]) -> list[str]: - """The bundle's dependencies, with agentenv-framework at least the version that serves bundles.""" + """The bundle's dependencies, with agentenv-framework's openenv extra at least the version that serves bundles.""" for index, dependency in enumerate(dependencies): try: requirement = Requirement(dependency) except InvalidRequirement: continue if canonicalize_name(requirement.name) == _FRAMEWORK: + requirement.extras.add(_FRAMEWORK_EXTRA) if requirement.url is None: requirement.specifier &= SpecifierSet(_FRAMEWORK_FLOOR) - dependencies[index] = str(requirement) + dependencies[index] = str(requirement) return dependencies - return [*dependencies, f"{_FRAMEWORK}{_FRAMEWORK_FLOOR}"] + return [*dependencies, f"{_FRAMEWORK}[{_FRAMEWORK_EXTRA}]{_FRAMEWORK_FLOOR}"] class AgentEnvImporter: diff --git a/tests/test_cli/test_import_env.py b/tests/test_cli/test_import_env.py index 7203f37e73..2c37efc60a 100644 --- a/tests/test_cli/test_import_env.py +++ b/tests/test_cli/test_import_env.py @@ -934,7 +934,7 @@ def test_import_command_detects_agentenv_bundle_and_generates_server( pyproject = (env_dir / "pyproject.toml").read_text(encoding="utf-8") requirements = (env_dir / "server" / "requirements.txt").read_text(encoding="utf-8") for text in (pyproject, requirements): - assert "agentenv-framework>=0.9.1360" in text + assert "agentenv-framework[openenv]>=0.9.1364" in text assert text.count("agentenv-framework") == 1 assert "numpy>=1.26" in text assert 'requires-python = ">=3.11.4"' in pyproject @@ -1000,6 +1000,6 @@ def test_import_command_vendors_only_the_agentenv_bundle_it_picks( app_source = (env_dir / "server" / "app.py").read_text(encoding="utf-8") assert '"vendor" / "train"' in app_source requirements = (env_dir / "server" / "requirements.txt").read_text(encoding="utf-8") - assert "agentenv-framework>=0.9.1360,>=0.9.1400" in requirements + assert "agentenv-framework[openenv]>=0.9.1364,>=0.9.1400" in requirements assert "numpy>=1.26" in requirements assert "torch" not in requirements From 35e772123892fcd95b38b7f466408a7752660a1f Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 27 Oct 2026 11:30:12 +0100 Subject: [PATCH 5/9] Document delegated AgentEnv API and Docker prerequisites (#1493) --- docs/source/guides/first-environment.md | 2 +- docs/source/reference/cli.md | 7 ++- src/openenv/cli/importers/agentenv.py | 14 +++++ .../templates/agentenv_README.md.tpl | 55 +++++++++++++++++ tests/test_cli/test_import_env.py | 61 +++++++++++++++++++ 5 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 src/openenv/cli/importers/templates/agentenv_README.md.tpl diff --git a/docs/source/guides/first-environment.md b/docs/source/guides/first-environment.md index fddaedd993..931af8efda 100644 --- a/docs/source/guides/first-environment.md +++ b/docs/source/guides/first-environment.md @@ -33,7 +33,7 @@ my_env/ Class names come from the environment name, with a trailing `_env` dropped: `my_env` gives `MyAction`, `MyObservation`, `MyEnvironment` and `MyEnv`. The steps below go through each file. Edit them to replace the echo logic with yours. If you're working inside the OpenEnv repo, move the folder under `envs/`. > [!TIP] -> If you already have an ORS/OpenReward or Prime Intellect Verifiers environment, or an AgentEnv bundle, run `openenv import SOURCE --name my_env --output-dir DIR` instead. It detects the source type, vendors the source under the generated package and emits an OpenEnv wrapper with task/split and MCP-style tool actions. Non-secret data files and portable dependencies from the source tree are carried over. +> If you already have an ORS/OpenReward or Prime Intellect Verifiers environment, or an AgentEnv bundle, run `openenv import SOURCE --name my_env --output-dir DIR` instead. It detects the source type and vendors non-secret data files and portable dependencies into the generated package. ORS/OpenReward and Verifiers use generated wrappers; AgentEnv delegates serving to `agent_env.openenv.build_app` and vendors only the selected bundle. AgentEnv requires local Docker, and the generated Dockerfile alone cannot provide a working deployment. Its generated echo client/models are template scaffolding and do not match the delegated API. See the generated README and [CLI reference](../reference/cli#openenv-import) for prerequisites. ## 2. Define the models diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index ac2d281ec8..8debff62dd 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -31,7 +31,12 @@ An [AgentEnv](https://github.com/scaleapi/agentenv-framework) bundle is a folder whose `tasks/` holds task files. When a source holds several bundles, pick one with `--env-class` and its folder name; only that folder is vendored. The generated server runs it with `build_app` from `agentenv-framework`, which by -default deploys each episode on the local Docker. See +default deploys each episode using local Docker. The server needs a Docker CLI +and a running daemon accessible for image builds and episode containers. The +generated `server/Dockerfile` does not supply that runtime, so `openenv build` +and `openenv push` alone do not produce a working deployment. The generated +README describes local startup and these prerequisites; the generated echo +client and models are template scaffolding and do not match the delegated API. See [AgentEnv's OpenEnv page](https://www.agentenvframework.com/docs/openenv) for what it serves and what it needs. diff --git a/src/openenv/cli/importers/agentenv.py b/src/openenv/cli/importers/agentenv.py index a7f7199459..19cbbdae32 100644 --- a/src/openenv/cli/importers/agentenv.py +++ b/src/openenv/cli/importers/agentenv.py @@ -9,6 +9,7 @@ from packaging.specifiers import SpecifierSet from packaging.utils import canonicalize_name +from .._cli_utils import console from .base import ( append_dependency_files, collect_source_dependencies, @@ -109,6 +110,19 @@ def generate( "agentenv_app.py.tpl", bundle=json.dumps(bundle.name) ), ) + write_text( + destination / "README.md", + render_importer_template( + "agentenv_README.md.tpl", env_name=env_name, bundle=bundle.name + ), + ) + console.print( + "[yellow]Warning:[/yellow] AgentEnv requires local Docker: a Docker CLI " + "and a running daemon accessible to the server for image builds and episode " + "containers. The generated server/Dockerfile does not provide this " + "runtime; openenv build / openenv push alone do not produce a working " + "deployment. See README.md for prerequisites and client limitations." + ) append_dependency_files( destination, env_name, diff --git a/src/openenv/cli/importers/templates/agentenv_README.md.tpl b/src/openenv/cli/importers/templates/agentenv_README.md.tpl new file mode 100644 index 0000000000..462c657940 --- /dev/null +++ b/src/openenv/cli/importers/templates/agentenv_README.md.tpl @@ -0,0 +1,55 @@ +# ${env_name}: AgentEnv bundle + +This package serves the vendored `${bundle}` bundle through +`agent_env.openenv.build_app`. The bundle keeps its original folder name at +`vendor/${bundle}`; the generated package name is `${env_name}`. + +## Prerequisites + +- Python >=3.11.4 and `agentenv-framework[openenv]>=0.9.1364` (declared in + `pyproject.toml` and `server/requirements.txt`). +- Local Docker: a Docker CLI and a running daemon accessible to the server. + The delegated server builds the bundle's environment images at startup and + deploys episode containers using local Docker by default. +- A self-contained bundle: tasks, environment definitions, verifiers, data and + dependencies must be included in the bundle or provided by agentenv-framework. + Only the selected bundle is vendored; external plugin registrations and files + elsewhere in the source repository are not included. + +## Run locally + +From this generated package directory, after setting up Docker: + +```bash +uv sync +uv run server --host 127.0.0.1 --port 8000 +``` + +The `server` entry point runs `server/app.py`, which passes +`vendor/${bundle}` to `build_app`. Task selection, splits, tools and rewards +are defined by the bundle and agentenv-framework. Follow +[AgentEnv's OpenEnv documentation](https://www.agentenvframework.com/docs/openenv) +for its API and client usage. + +`client.py`, `models.py` and the exports in `__init__.py` are echo template +scaffolding. They do not match the delegated AgentEnv API and must not be used +as a working client for this server. + +## Deployment limitations + +**Warning:** `server/Dockerfile` does not provide the Docker runtime required +by the delegated server. `openenv build` / `openenv push` alone do not produce +a working deployment, including on Hugging Face Spaces. Deployment requires +separately provisioning the server's image-build and episode-container runtime; +this importer does not configure it. Review AgentEnv's runtime requirements +before choosing a deployment host. + +## Generated files + +- `server/app.py`: delegated application and local server entry point. +- `vendor/${bundle}`: the selected bundle and its data. +- `pyproject.toml`: package dependencies and the `server` entry point. +- `server/requirements.txt`: server dependencies. +- `openenv.yaml`: generated OpenEnv manifest. +- `server/Dockerfile`: shared image template with the limitation described above. +- `client.py`, `models.py`, `__init__.py`: echo template scaffolding. diff --git a/tests/test_cli/test_import_env.py b/tests/test_cli/test_import_env.py index 2c37efc60a..5e7bd02d58 100644 --- a/tests/test_cli/test_import_env.py +++ b/tests/test_cli/test_import_env.py @@ -1003,3 +1003,64 @@ def test_import_command_vendors_only_the_agentenv_bundle_it_picks( assert "agentenv-framework[openenv]>=0.9.1364,>=0.9.1400" in requirements assert "numpy>=1.26" in requirements assert "torch" not in requirements + + +@pytest.mark.parametrize( + ("bundle_name", "env_name"), [("echo-desk", "desk_env"), ("train", "picked_env")] +) +def test_agentenv_generated_readme_matches_paths_and_prerequisites( + tmp_path: Path, bundle_name: str, env_name: str +) -> None: + import re + + import tomli + + source = tmp_path / bundle_name + _write_agentenv_bundle(source) + output_dir = tmp_path / "out" + with patch("openenv.cli.commands.import_env._generate_uv_lock", return_value=True): + result = runner.invoke( + app, + [ + "import", + str(source), + "--name", + env_name, + "--output-dir", + str(output_dir), + ], + ) + + assert result.exit_code == 0, result.output + env_dir = output_dir / env_name + readme = (env_dir / "README.md").read_text(encoding="utf-8") + # Check the published file inventory against the output, including the bundle + # whose name differs from the package and may contain a hyphen. + inventory = readme.split("## Generated files\n", 1)[1] + paths = re.findall(r"`([^`]+)`", inventory) + assert f"vendor/{bundle_name}" in paths + assert "server/app.py" in paths + for path in paths: + assert (env_dir / path).exists(), path + assert f"{env_name}_environment.py" not in readme + assert "from_docker_image" not in readme + assert "echoed_message" not in readme + + project = tomli.loads((env_dir / "pyproject.toml").read_text())["project"] + assert "uv sync\nuv run server --host 127.0.0.1 --port 8000" in readme + assert project["scripts"]["server"] == f"{env_name}.server.app:main" + assert project["requires-python"] == ">=3.11.4" + assert "Python >=3.11.4" in readme + requirement = "agentenv-framework[openenv]>=0.9.1364" + assert requirement in project["dependencies"] + assert requirement in (env_dir / "server/requirements.txt").read_text() + assert requirement in readme + assert "agent_env.openenv.build_app" in readme + assert "do not match the delegated AgentEnv API" in readme + for text in (readme, result.output): + normalized = " ".join(text.split()) + assert "Docker CLI and a running daemon accessible to the server" in normalized + assert "openenv build" in text and "openenv push" in text + assert "do not produce a working deployment" in normalized + assert "server/Dockerfile" in result.output + assert "Warning:" in result.output From c83434937f871285453a8b274f88297c8a74e2de Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 08:26:35 -0700 Subject: [PATCH 6/9] Link the delegated adapter's source and require step ids in detection The generated README now links agent_env.openenv's environment.py and app.py at the minimum supported agentenv-framework release, so the served environment can be read from the generated package. Detection also requires each step to carry a string id, as AgentEnv task steps do, which makes it less likely that another project's tasks/*.json is mistaken for a bundle. Co-Authored-By: Claude Opus 5.5 --- src/openenv/cli/importers/agentenv.py | 17 ++++++++++++++--- .../importers/templates/agentenv_README.md.tpl | 7 +++++-- tests/test_cli/test_import_env.py | 7 +++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/openenv/cli/importers/agentenv.py b/src/openenv/cli/importers/agentenv.py index 19cbbdae32..2250335a7b 100644 --- a/src/openenv/cli/importers/agentenv.py +++ b/src/openenv/cli/importers/agentenv.py @@ -23,7 +23,12 @@ _FRAMEWORK = "agentenv-framework" _FRAMEWORK_EXTRA = "openenv" -_FRAMEWORK_FLOOR = ">=0.9.1364" +_FRAMEWORK_VERSION = "0.9.1364" +_FRAMEWORK_FLOOR = f">={_FRAMEWORK_VERSION}" +_ADAPTER_SOURCE = ( + "https://github.com/scaleapi/agentenv-framework/blob/" + f"v{_FRAMEWORK_VERSION}/src/agent_env/openenv" +) _REQUIRES_PYTHON = ">=3.11.4" @@ -36,7 +41,9 @@ def _is_task_file(path: Path) -> bool: isinstance(steps, list) and bool(steps) and all( - isinstance(step, dict) and isinstance(step.get("type"), str) + isinstance(step, dict) + and isinstance(step.get("id"), str) + and isinstance(step.get("type"), str) for step in steps ) ) @@ -113,7 +120,11 @@ def generate( write_text( destination / "README.md", render_importer_template( - "agentenv_README.md.tpl", env_name=env_name, bundle=bundle.name + "agentenv_README.md.tpl", + env_name=env_name, + bundle=bundle.name, + framework=f"{_FRAMEWORK}[{_FRAMEWORK_EXTRA}]{_FRAMEWORK_FLOOR}", + adapter=_ADAPTER_SOURCE, ), ) console.print( diff --git a/src/openenv/cli/importers/templates/agentenv_README.md.tpl b/src/openenv/cli/importers/templates/agentenv_README.md.tpl index 462c657940..20aaaee33b 100644 --- a/src/openenv/cli/importers/templates/agentenv_README.md.tpl +++ b/src/openenv/cli/importers/templates/agentenv_README.md.tpl @@ -6,7 +6,7 @@ This package serves the vendored `${bundle}` bundle through ## Prerequisites -- Python >=3.11.4 and `agentenv-framework[openenv]>=0.9.1364` (declared in +- Python >=3.11.4 and `${framework}` (declared in `pyproject.toml` and `server/requirements.txt`). - Local Docker: a Docker CLI and a running daemon accessible to the server. The delegated server builds the bundle's environment images at startup and @@ -27,7 +27,10 @@ uv run server --host 127.0.0.1 --port 8000 The `server` entry point runs `server/app.py`, which passes `vendor/${bundle}` to `build_app`. Task selection, splits, tools and rewards -are defined by the bundle and agentenv-framework. Follow +are defined by the bundle and agentenv-framework. The served environment is +`agent_env.openenv`'s `AgentEnvEnvironment`; read it, and `build_app`, at the +minimum supported version: [environment.py](${adapter}/environment.py) and +[app.py](${adapter}/app.py). Follow [AgentEnv's OpenEnv documentation](https://www.agentenvframework.com/docs/openenv) for its API and client usage. diff --git a/tests/test_cli/test_import_env.py b/tests/test_cli/test_import_env.py index 5e7bd02d58..d3f6158408 100644 --- a/tests/test_cli/test_import_env.py +++ b/tests/test_cli/test_import_env.py @@ -893,6 +893,10 @@ def test_agentenv_detector_finds_bundles_without_importing_source( (source / "notes" / "tasks" / "todo.json").write_text( '{"title": "not a task"}', encoding="utf-8" ) + (source / "jobs" / "tasks").mkdir(parents=True) + (source / "jobs" / "tasks" / "queue.json").write_text( + '[{"type": "build"}, {"type": "deploy"}]', encoding="utf-8" + ) matches = detect_agentenv_bundles(source) @@ -1056,6 +1060,9 @@ def test_agentenv_generated_readme_matches_paths_and_prerequisites( assert requirement in (env_dir / "server/requirements.txt").read_text() assert requirement in readme assert "agent_env.openenv.build_app" in readme + adapter = "https://github.com/scaleapi/agentenv-framework/blob/v0.9.1364/src/agent_env/openenv" + assert f"({adapter}/environment.py)" in readme + assert f"({adapter}/app.py)" in readme assert "do not match the delegated AgentEnv API" in readme for text in (readme, result.output): normalized = " ".join(text.split()) From bc98140bf3052770c4c9c61ed16bcd5984847338 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 27 Oct 2026 16:39:11 +0100 Subject: [PATCH 7/9] fix(cli): reject unverifiable AgentEnv framework direct references (#1493) The framework floor was skipped for named direct references. Validate collected dependencies before scaffolding and reject sources whose adapter version cannot be established statically. Preserve normal range constraints, extras and markers. --- src/openenv/cli/importers/agentenv.py | 14 ++++- tests/test_cli/test_import_env.py | 91 +++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/src/openenv/cli/importers/agentenv.py b/src/openenv/cli/importers/agentenv.py index 2250335a7b..ac0e41c49c 100644 --- a/src/openenv/cli/importers/agentenv.py +++ b/src/openenv/cli/importers/agentenv.py @@ -80,9 +80,16 @@ def _with_framework_floor(dependencies: list[str]) -> list[str]: except InvalidRequirement: continue if canonicalize_name(requirement.name) == _FRAMEWORK: + if requirement.url is not None: + raise ValueError( + f"Cannot verify {_FRAMEWORK}{_FRAMEWORK_FLOOR} from a direct " + "reference. In the bundle's pyproject.toml or requirements.txt, " + "replace the framework direct reference with a version constraint " + f"such as {_FRAMEWORK}[{_FRAMEWORK_EXTRA}]{_FRAMEWORK_FLOOR}, " + "preserving any additional extras and environment marker." + ) requirement.extras.add(_FRAMEWORK_EXTRA) - if requirement.url is None: - requirement.specifier &= SpecifierSet(_FRAMEWORK_FLOOR) + requirement.specifier &= SpecifierSet(_FRAMEWORK_FLOOR) dependencies[index] = str(requirement) return dependencies return [*dependencies, f"{_FRAMEWORK}[{_FRAMEWORK_EXTRA}]{_FRAMEWORK_FLOOR}"] @@ -105,6 +112,7 @@ def generate( detected: DetectedEnvironment, ) -> None: bundle = detected.file_path + dependencies = _with_framework_floor(collect_source_dependencies(bundle)) prepare_import_package(bundle, destination, env_name, vendor_dir=bundle.name) (destination / "server" / f"{env_name}_environment.py").unlink() write_text( @@ -137,6 +145,6 @@ def generate( append_dependency_files( destination, env_name, - _with_framework_floor(collect_source_dependencies(bundle)), + dependencies, requires_python=_REQUIRES_PYTHON, ) diff --git a/tests/test_cli/test_import_env.py b/tests/test_cli/test_import_env.py index d3f6158408..b5d8ed7468 100644 --- a/tests/test_cli/test_import_env.py +++ b/tests/test_cli/test_import_env.py @@ -23,6 +23,7 @@ from openenv.cli.importers.ors import detect_ors_dependencies, detect_ors_environments from openenv.cli.importers.verifiers import detect_verifiers_environments from openenv.core.env_server.mcp_types import CallToolAction, ListToolsAction +from packaging.requirements import Requirement from typer.testing import CliRunner @@ -966,6 +967,96 @@ def test_import_command_detects_agentenv_bundle_and_generates_server( sys.modules.pop(name) +@pytest.mark.parametrize("dependency_file", ["requirements.txt", "pyproject.toml"]) +@pytest.mark.parametrize( + "dependency", + [ + "agentenv-framework @ https://example.invalid/agentenv_framework-0.9.1000-py3-none-any.whl", + "agentenv-framework @ https://example.invalid/agentenv_framework-0.9.1400-py3-none-any.whl", + "agentenv-framework @ https://example.invalid/framework.tar.gz", + "agentenv-framework @ git+https://example.invalid/framework.git@v0.9.1400", + "agentenv-framework @ file:///tmp/agentenv_framework-0.9.1400-py3-none-any.whl", + 'AgentEnv_Framework[tools] @ https://user:secret@example.invalid/framework.whl ; python_version >= "3.11"', + ], +) +def test_import_command_rejects_agentenv_direct_references( + tmp_path: Path, dependency_file: str, dependency: str +) -> None: + source = tmp_path / "bundle" + _write_agentenv_bundle(source) + content = ( + "[project]\ndependencies = " + json.dumps([dependency]) + "\n" + if dependency_file == "pyproject.toml" + else dependency + "\n" + ) + (source / dependency_file).write_text(content, encoding="utf-8") + output_dir = tmp_path / "out" + + with ( + patch("openenv.cli.commands.import_env._generate_uv_lock") as lock, + patch("socket.socket.connect", side_effect=AssertionError("network forbidden")), + ): + result = runner.invoke( + app, + ["import", str(source), "--name", "desk_env", "-o", str(output_dir)], + ) + + assert result.exit_code == 1, result.output + assert "Cannot verify agentenv-framework>=0.9.1364" in result.output + assert "direct reference" in result.output + assert "version constraint" in result.output + assert "secret" not in result.output + lock.assert_not_called() + assert not (output_dir / "desk_env").exists() + + +@pytest.mark.parametrize("dependency_file", ["requirements.txt", "pyproject.toml"]) +@pytest.mark.parametrize("version_range", [">=0.9.1000,<1", ">=0.9.1400"]) +def test_import_command_merges_agentenv_floor_preserving_extras_and_marker( + tmp_path: Path, dependency_file: str, version_range: str +) -> None: + import tomllib + + source = tmp_path / "bundle" + _write_agentenv_bundle(source) + dependency = f'agentenv-framework[tools]{version_range}; python_version >= "3.11"' + content = ( + "[project]\ndependencies = " + json.dumps([dependency]) + "\n" + if dependency_file == "pyproject.toml" + else dependency + "\n" + ) + (source / dependency_file).write_text(content, encoding="utf-8") + output_dir = tmp_path / "out" + + with ( + patch("openenv.cli.commands.import_env._generate_uv_lock", return_value=True), + patch("socket.socket.connect", side_effect=AssertionError("network forbidden")), + ): + result = runner.invoke( + app, + ["import", str(source), "--name", "desk_env", "-o", str(output_dir)], + ) + + assert result.exit_code == 0, result.output + env_dir = output_dir / "desk_env" + project = tomllib.loads((env_dir / "pyproject.toml").read_text(encoding="utf-8")) + requirements = (env_dir / "server" / "requirements.txt").read_text(encoding="utf-8") + for dependencies in (project["project"]["dependencies"], requirements.splitlines()): + matches = [ + Requirement(dep) for dep in dependencies if "agentenv-framework" in dep + ] + assert len(matches) == 1 + requirement = matches[0] + assert requirement.url is None + assert requirement.extras == {"tools", "openenv"} + assert requirement.marker == Requirement(dependency).marker + assert ( + requirement.specifier + == Requirement(f"agentenv-framework{version_range},>=0.9.1364").specifier + ) + assert "0.9.1000" not in requirement.specifier + + def test_import_command_vendors_only_the_agentenv_bundle_it_picks( tmp_path: Path, ) -> None: From a72456aa05227b4305edc8050ecef519480ade09 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 27 Oct 2026 12:08:13 -0700 Subject: [PATCH 8/9] Generate an MCPToolClient for AgentEnv bundles, read MAX_CONCURRENT_ENVS, and require agentenv-framework 0.9.1366 - client.py is now an MCPToolClient subclass, and __init__.py exports it with CallToolAction and ListToolsAction, as envs/echo_env does. The echo scaffolding's models.py is dropped. The README shows the client. - server/app.py passes MAX_CONCURRENT_ENVS to build_app, as OpenEnv's own env apps do (default 8). - The floor is agentenv-framework 0.9.1366, whose adapter raises on a step before the session's first reset, as RFC 015's wrapper contract requires, and ends an episode whose end_turn the proxy refuses in an error observation. Co-Authored-By: Claude Opus 5.5 --- docs/source/guides/first-environment.md | 2 +- docs/source/reference/cli.md | 4 +-- src/openenv/cli/importers/agentenv.py | 19 +++++++++-- .../templates/agentenv_README.md.tpl | 29 +++++++++++++--- .../importers/templates/agentenv_app.py.tpl | 5 ++- .../templates/agentenv_client.py.tpl | 19 +++++++++++ .../importers/templates/agentenv_init.py.tpl | 7 ++++ tests/test_cli/test_import_env.py | 34 +++++++++++++------ 8 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 src/openenv/cli/importers/templates/agentenv_client.py.tpl create mode 100644 src/openenv/cli/importers/templates/agentenv_init.py.tpl diff --git a/docs/source/guides/first-environment.md b/docs/source/guides/first-environment.md index 931af8efda..7954ccbf43 100644 --- a/docs/source/guides/first-environment.md +++ b/docs/source/guides/first-environment.md @@ -33,7 +33,7 @@ my_env/ Class names come from the environment name, with a trailing `_env` dropped: `my_env` gives `MyAction`, `MyObservation`, `MyEnvironment` and `MyEnv`. The steps below go through each file. Edit them to replace the echo logic with yours. If you're working inside the OpenEnv repo, move the folder under `envs/`. > [!TIP] -> If you already have an ORS/OpenReward or Prime Intellect Verifiers environment, or an AgentEnv bundle, run `openenv import SOURCE --name my_env --output-dir DIR` instead. It detects the source type and vendors non-secret data files and portable dependencies into the generated package. ORS/OpenReward and Verifiers use generated wrappers; AgentEnv delegates serving to `agent_env.openenv.build_app` and vendors only the selected bundle. AgentEnv requires local Docker, and the generated Dockerfile alone cannot provide a working deployment. Its generated echo client/models are template scaffolding and do not match the delegated API. See the generated README and [CLI reference](../reference/cli#openenv-import) for prerequisites. +> If you already have an ORS/OpenReward or Prime Intellect Verifiers environment, or an AgentEnv bundle, run `openenv import SOURCE --name my_env --output-dir DIR` instead. It detects the source type and vendors non-secret data files and portable dependencies into the generated package. ORS/OpenReward and Verifiers use generated wrappers; AgentEnv delegates serving to `agent_env.openenv.build_app` and vendors only the selected bundle. AgentEnv requires local Docker, and the generated Dockerfile alone cannot provide a working deployment. Its generated client is an `MCPToolClient`. See the generated README and [CLI reference](../reference/cli#openenv-import) for prerequisites. ## 2. Define the models diff --git a/docs/source/reference/cli.md b/docs/source/reference/cli.md index 8debff62dd..f88ca8596d 100644 --- a/docs/source/reference/cli.md +++ b/docs/source/reference/cli.md @@ -35,8 +35,8 @@ default deploys each episode using local Docker. The server needs a Docker CLI and a running daemon accessible for image builds and episode containers. The generated `server/Dockerfile` does not supply that runtime, so `openenv build` and `openenv push` alone do not produce a working deployment. The generated -README describes local startup and these prerequisites; the generated echo -client and models are template scaffolding and do not match the delegated API. See +README describes local startup and these prerequisites. The generated client is +an `MCPToolClient`, since the served environment takes MCP actions. See [AgentEnv's OpenEnv page](https://www.agentenvframework.com/docs/openenv) for what it serves and what it needs. diff --git a/src/openenv/cli/importers/agentenv.py b/src/openenv/cli/importers/agentenv.py index ac0e41c49c..05d8d3694b 100644 --- a/src/openenv/cli/importers/agentenv.py +++ b/src/openenv/cli/importers/agentenv.py @@ -23,7 +23,7 @@ _FRAMEWORK = "agentenv-framework" _FRAMEWORK_EXTRA = "openenv" -_FRAMEWORK_VERSION = "0.9.1364" +_FRAMEWORK_VERSION = "0.9.1366" _FRAMEWORK_FLOOR = f">={_FRAMEWORK_VERSION}" _ADAPTER_SOURCE = ( "https://github.com/scaleapi/agentenv-framework/blob/" @@ -113,8 +113,20 @@ def generate( ) -> None: bundle = detected.file_path dependencies = _with_framework_floor(collect_source_dependencies(bundle)) - prepare_import_package(bundle, destination, env_name, vendor_dir=bundle.name) + package = prepare_import_package( + bundle, destination, env_name, vendor_dir=bundle.name + ) + client = f"{package.class_name_prefix}Env" (destination / "server" / f"{env_name}_environment.py").unlink() + (destination / "models.py").unlink() + for name, template in ( + ("client.py", "agentenv_client.py.tpl"), + ("__init__.py", "agentenv_init.py.tpl"), + ): + write_text( + destination / name, + render_importer_template(template, env_name=env_name, client=client), + ) write_text( destination / "server" / "__init__.py", f'"""{env_name} environment server: an AgentEnv bundle."""\n', @@ -133,6 +145,7 @@ def generate( bundle=bundle.name, framework=f"{_FRAMEWORK}[{_FRAMEWORK_EXTRA}]{_FRAMEWORK_FLOOR}", adapter=_ADAPTER_SOURCE, + client=client, ), ) console.print( @@ -140,7 +153,7 @@ def generate( "and a running daemon accessible to the server for image builds and episode " "containers. The generated server/Dockerfile does not provide this " "runtime; openenv build / openenv push alone do not produce a working " - "deployment. See README.md for prerequisites and client limitations." + "deployment. See README.md for prerequisites." ) append_dependency_files( destination, diff --git a/src/openenv/cli/importers/templates/agentenv_README.md.tpl b/src/openenv/cli/importers/templates/agentenv_README.md.tpl index 20aaaee33b..30d33aa10b 100644 --- a/src/openenv/cli/importers/templates/agentenv_README.md.tpl +++ b/src/openenv/cli/importers/templates/agentenv_README.md.tpl @@ -22,9 +22,12 @@ From this generated package directory, after setting up Docker: ```bash uv sync -uv run server --host 127.0.0.1 --port 8000 +MAX_CONCURRENT_ENVS=8 uv run server --host 127.0.0.1 --port 8000 ``` +`MAX_CONCURRENT_ENVS` (default 8) caps the concurrent sessions. Each session +plays its own episode on its own deployment. + The `server` entry point runs `server/app.py`, which passes `vendor/${bundle}` to `build_app`. Task selection, splits, tools and rewards are defined by the bundle and agentenv-framework. The served environment is @@ -34,9 +37,25 @@ minimum supported version: [environment.py](${adapter}/environment.py) and [AgentEnv's OpenEnv documentation](https://www.agentenvframework.com/docs/openenv) for its API and client usage. -`client.py`, `models.py` and the exports in `__init__.py` are echo template -scaffolding. They do not match the delegated AgentEnv API and must not be used -as a working client for this server. +## Client + +`client.py` defines `${client}`, an `MCPToolClient`: the served environment +takes OpenEnv's MCP actions (`ListToolsAction`, `CallToolAction`), which +`__init__.py` also exports. + +```python +from ${env_name} import ${client} + +async with ${client}(base_url="http://127.0.0.1:8000") as env: + await env.reset(split="all", index=0) + tools = await env.list_tools() + ended = await env.call_tool("end_turn", response="Done.") + print(ended["done"], ended["reward"]) +``` + +The tools are the bundle environment's MCP tools plus `end_turn`, which ends +the turn with its `response`. Its result carries the next turn's prompt, or, +once `done`, the task's reward. A step before `reset` raises. ## Deployment limitations @@ -55,4 +74,4 @@ before choosing a deployment host. - `server/requirements.txt`: server dependencies. - `openenv.yaml`: generated OpenEnv manifest. - `server/Dockerfile`: shared image template with the limitation described above. -- `client.py`, `models.py`, `__init__.py`: echo template scaffolding. +- `client.py`, `__init__.py`: the client and the MCP action types it sends. diff --git a/src/openenv/cli/importers/templates/agentenv_app.py.tpl b/src/openenv/cli/importers/templates/agentenv_app.py.tpl index 60b592c0b3..4ea4d0b600 100644 --- a/src/openenv/cli/importers/templates/agentenv_app.py.tpl +++ b/src/openenv/cli/importers/templates/agentenv_app.py.tpl @@ -1,5 +1,6 @@ from __future__ import annotations +import os from pathlib import Path from agent_env.openenv import build_app @@ -7,7 +8,9 @@ from agent_env.openenv import build_app BUNDLE = Path(__file__).resolve().parents[1] / "vendor" / ${bundle} -app = build_app(BUNDLE) +app = build_app( + BUNDLE, max_concurrent_envs=int(os.getenv("MAX_CONCURRENT_ENVS", "8")) +) def main(): diff --git a/src/openenv/cli/importers/templates/agentenv_client.py.tpl b/src/openenv/cli/importers/templates/agentenv_client.py.tpl new file mode 100644 index 0000000000..4bc33ad798 --- /dev/null +++ b/src/openenv/cli/importers/templates/agentenv_client.py.tpl @@ -0,0 +1,19 @@ +"""${env_name} client. + +The served AgentEnv bundle takes OpenEnv's MCP actions, so this client is an +``MCPToolClient``. Reset to a task, call the bundle's tools, and end each turn +with ``end_turn``. Its result carries the next turn's prompt, or once ``done`` +the task's reward: + + >>> async with ${client}(base_url="http://localhost:8000") as env: + ... await env.reset(split="all", index=0) + ... tools = await env.list_tools() + ... ended = await env.call_tool("end_turn", response="Done.") + ... print(ended["done"], ended["reward"]) +""" + +from openenv.core.mcp_client import MCPToolClient + + +class ${client}(MCPToolClient): + """Client for the ${env_name} environment.""" diff --git a/src/openenv/cli/importers/templates/agentenv_init.py.tpl b/src/openenv/cli/importers/templates/agentenv_init.py.tpl new file mode 100644 index 0000000000..860105f5b3 --- /dev/null +++ b/src/openenv/cli/importers/templates/agentenv_init.py.tpl @@ -0,0 +1,7 @@ +"""${env_name} environment: an AgentEnv bundle served through agentenv-framework.""" + +from openenv.core.env_server.mcp_types import CallToolAction, ListToolsAction + +from .client import ${client} + +__all__ = ["${client}", "CallToolAction", "ListToolsAction"] diff --git a/tests/test_cli/test_import_env.py b/tests/test_cli/test_import_env.py index b5d8ed7468..2a4a57d252 100644 --- a/tests/test_cli/test_import_env.py +++ b/tests/test_cli/test_import_env.py @@ -23,6 +23,7 @@ from openenv.cli.importers.ors import detect_ors_dependencies, detect_ors_environments from openenv.cli.importers.verifiers import detect_verifiers_environments from openenv.core.env_server.mcp_types import CallToolAction, ListToolsAction +from openenv.core.mcp_client import MCPToolClient from packaging.requirements import Requirement from typer.testing import CliRunner @@ -246,8 +247,8 @@ def _write_fake_agentenv_sdk(root: Path) -> None: served = [] -def build_app(bundle): - served.append(bundle) +def build_app(bundle, max_concurrent_envs=8): + served.append((bundle, max_concurrent_envs)) return FastAPI() """.lstrip(), encoding="utf-8", @@ -939,7 +940,7 @@ def test_import_command_detects_agentenv_bundle_and_generates_server( pyproject = (env_dir / "pyproject.toml").read_text(encoding="utf-8") requirements = (env_dir / "server" / "requirements.txt").read_text(encoding="utf-8") for text in (pyproject, requirements): - assert "agentenv-framework[openenv]>=0.9.1364" in text + assert "agentenv-framework[openenv]>=0.9.1366" in text assert text.count("agentenv-framework") == 1 assert "numpy>=1.26" in text assert 'requires-python = ">=3.11.4"' in pyproject @@ -950,12 +951,16 @@ def test_import_command_detects_agentenv_bundle_and_generates_server( calls = [] monkeypatch.setattr(uvicorn, "run", lambda app, **kwargs: calls.append(kwargs)) monkeypatch.setattr(sys, "argv", ["server", "--port", "8001"]) + monkeypatch.setenv("MAX_CONCURRENT_ENVS", "64") sys.path[:0] = [str(sdk), str(output_dir)] try: generated = importlib.import_module("desk_env.server.app") served = importlib.import_module("agent_env.openenv").served - assert served == [bundle.resolve()] + assert served == [(bundle.resolve(), 64)] + package = importlib.import_module("desk_env") + assert issubclass(package.DeskEnv, MCPToolClient) + assert package.__all__ == ["DeskEnv", "CallToolAction", "ListToolsAction"] assert TestClient(generated.app).get("/docs").status_code == 200 generated.main() assert calls == [{"host": "0.0.0.0", "port": 8001}] @@ -1002,7 +1007,7 @@ def test_import_command_rejects_agentenv_direct_references( ) assert result.exit_code == 1, result.output - assert "Cannot verify agentenv-framework>=0.9.1364" in result.output + assert "Cannot verify agentenv-framework>=0.9.1366" in result.output assert "direct reference" in result.output assert "version constraint" in result.output assert "secret" not in result.output @@ -1052,7 +1057,7 @@ def test_import_command_merges_agentenv_floor_preserving_extras_and_marker( assert requirement.marker == Requirement(dependency).marker assert ( requirement.specifier - == Requirement(f"agentenv-framework{version_range},>=0.9.1364").specifier + == Requirement(f"agentenv-framework{version_range},>=0.9.1366").specifier ) assert "0.9.1000" not in requirement.specifier @@ -1095,7 +1100,7 @@ def test_import_command_vendors_only_the_agentenv_bundle_it_picks( app_source = (env_dir / "server" / "app.py").read_text(encoding="utf-8") assert '"vendor" / "train"' in app_source requirements = (env_dir / "server" / "requirements.txt").read_text(encoding="utf-8") - assert "agentenv-framework[openenv]>=0.9.1364,>=0.9.1400" in requirements + assert "agentenv-framework[openenv]>=0.9.1366,>=0.9.1400" in requirements assert "numpy>=1.26" in requirements assert "torch" not in requirements @@ -1142,19 +1147,26 @@ def test_agentenv_generated_readme_matches_paths_and_prerequisites( assert "echoed_message" not in readme project = tomli.loads((env_dir / "pyproject.toml").read_text())["project"] - assert "uv sync\nuv run server --host 127.0.0.1 --port 8000" in readme + assert ( + "uv sync\nMAX_CONCURRENT_ENVS=8 uv run server --host 127.0.0.1 --port 8000" + in readme + ) assert project["scripts"]["server"] == f"{env_name}.server.app:main" assert project["requires-python"] == ">=3.11.4" assert "Python >=3.11.4" in readme - requirement = "agentenv-framework[openenv]>=0.9.1364" + requirement = "agentenv-framework[openenv]>=0.9.1366" assert requirement in project["dependencies"] assert requirement in (env_dir / "server/requirements.txt").read_text() assert requirement in readme assert "agent_env.openenv.build_app" in readme - adapter = "https://github.com/scaleapi/agentenv-framework/blob/v0.9.1364/src/agent_env/openenv" + adapter = "https://github.com/scaleapi/agentenv-framework/blob/v0.9.1366/src/agent_env/openenv" assert f"({adapter}/environment.py)" in readme assert f"({adapter}/app.py)" in readme - assert "do not match the delegated AgentEnv API" in readme + client = {"desk_env": "DeskEnv", "picked_env": "PickedEnv"}[env_name] + assert f"from {env_name} import {client}" in readme + assert f"class {client}(MCPToolClient):" in (env_dir / "client.py").read_text() + assert not (env_dir / "models.py").exists() + assert "scaffolding" not in readme for text in (readme, result.output): normalized = " ".join(text.split()) assert "Docker CLI and a running daemon accessible to the server" in normalized From 598514f8ea20dbce6feda97518483f0cc5e63102 Mon Sep 17 00:00:00 2001 From: burtenshaw Date: Tue, 27 Oct 2026 20:18:36 +0100 Subject: [PATCH 9/9] Fix generated AgentEnv client discovery across tasks and turns (#1493) The generated subclass inherited MCPToolClient's persistent discovery cache, so reset and turn changes kept old tools. Default to fresh discovery while preserving explicit use_cache arguments. Cover reset, end_turn via HTTP and WebSocket, and dynamic tool changes with generated-client transport regressions. --- .../templates/agentenv_README.md.tpl | 5 + .../templates/agentenv_client.py.tpl | 10 ++ tests/test_cli/test_agentenv_client.py | 138 ++++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 tests/test_cli/test_agentenv_client.py diff --git a/src/openenv/cli/importers/templates/agentenv_README.md.tpl b/src/openenv/cli/importers/templates/agentenv_README.md.tpl index 30d33aa10b..f2ad1c69d7 100644 --- a/src/openenv/cli/importers/templates/agentenv_README.md.tpl +++ b/src/openenv/cli/importers/templates/agentenv_README.md.tpl @@ -57,6 +57,11 @@ The tools are the bundle environment's MCP tools plus `end_turn`, which ends the turn with its `response`. Its result carries the next turn's prompt, or, once `done`, the task's reward. A step before `reset` raises. +`list_tools()` discovers fresh tools by default because tools can change after +reset, `end_turn`, or a bundle tool call. `get_tool()` and `has_tool()` also use +fresh discovery. Pass `list_tools(use_cache=True)` explicitly to reuse the last +discovery result. + ## Deployment limitations **Warning:** `server/Dockerfile` does not provide the Docker runtime required diff --git a/src/openenv/cli/importers/templates/agentenv_client.py.tpl b/src/openenv/cli/importers/templates/agentenv_client.py.tpl index 4bc33ad798..c98c527b1a 100644 --- a/src/openenv/cli/importers/templates/agentenv_client.py.tpl +++ b/src/openenv/cli/importers/templates/agentenv_client.py.tpl @@ -12,8 +12,18 @@ the task's reward: ... print(ended["done"], ended["reward"]) """ +from openenv.core.env_server.mcp_types import Tool from openenv.core.mcp_client import MCPToolClient class ${client}(MCPToolClient): """Client for the ${env_name} environment.""" + + async def list_tools(self, use_cache: bool = False) -> list[Tool]: + """Discover the current task and turn's tools. + + Args: + use_cache (`bool`, *optional*, defaults to `False`): + Set to `True` to reuse the last discovery result. + """ + return await super().list_tools(use_cache=use_cache) diff --git a/tests/test_cli/test_agentenv_client.py b/tests/test_cli/test_agentenv_client.py new file mode 100644 index 0000000000..bdd3db1800 --- /dev/null +++ b/tests/test_cli/test_agentenv_client.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: BSD-3-Clause + +"""Generated AgentEnv client discovery over real HTTP and WebSocket transports.""" + +import asyncio +import importlib.util +import socket + +import pytest +import uvicorn +from openenv.cli.importers.agentenv import AgentEnvImporter, detect_agentenv_bundles +from openenv.core.env_server.http_server import create_app +from openenv.core.env_server.interfaces import Environment +from openenv.core.env_server.mcp_types import ( + CallToolAction, + CallToolObservation, + ListToolsAction, + ListToolsObservation, + Tool, +) +from openenv.core.env_server.types import Observation, State + + +class ChangingToolsEpisode(Environment): + """Minimal episode with task-local tools that can also change during a turn.""" + + SUPPORTS_CONCURRENT_SESSIONS = True + + def reset(self, *, index=0, **kwargs): + self.tool = ("echo", "other")[index] + return Observation() + + def step(self, action, **kwargs): + if isinstance(action, ListToolsAction): + return ListToolsObservation( + tools=[ + Tool( + name=name, + description="Episode tool", + input_schema={"type": "object"}, + ) + for name in (self.tool, "end_turn") + ] + ) + assert isinstance(action, CallToolAction) + assert action.tool_name in (self.tool, "end_turn") + self.tool = "next" + return CallToolObservation(tool_name=action.tool_name, result=action.arguments) + + @property + def state(self): + return State() + + +@pytest.fixture +async def generated_client(tmp_path): + bundle = tmp_path / "bundle" + (bundle / "tasks").mkdir(parents=True) + (bundle / "tasks" / "task.json").write_text( + '[{"id": "turn", "type": "prompt_agent"}]' + ) + destination = tmp_path / "desk_env" + AgentEnvImporter().generate( + source=bundle, + destination=destination, + env_name="desk_env", + detected=detect_agentenv_bundles(bundle)[0], + ) + spec = importlib.util.spec_from_file_location( + "generated_agentenv_client", destination / "client.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + app = create_app( + ChangingToolsEpisode, + CallToolAction, + CallToolObservation, + reset_observation_cls=Observation, + max_concurrent_envs=8, + ) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + server = uvicorn.Server( + uvicorn.Config(app, log_level="critical", lifespan="off") + ) + task = asyncio.create_task(server.serve(sockets=[sock])) + try: + async with asyncio.timeout(10): + while not server.started: + if task.done(): + await task + await asyncio.sleep(0.01) + async with module.DeskEnv(base_url=f"http://127.0.0.1:{port}") as client: + yield client + finally: + server.should_exit = True + await asyncio.wait_for(task, timeout=10) + + +def names(tools): + return [tool.name for tool in tools] + + +async def test_generated_client_discovers_new_task_after_reset(generated_client): + client = generated_client + await client.reset(split="all", index=0) + assert names(await client.list_tools()) == ["echo", "end_turn"] + await client.call_tool("end_turn", response="Done.") + await client.reset(split="all", index=1) + assert names(await client.list_tools()) == ["other", "end_turn"] + assert await client.has_tool("other") + assert not await client.has_tool("echo") + + +@pytest.mark.parametrize("transition", ["end_turn", "step", "tool"]) +async def test_generated_client_discovers_changed_turn_tools( + generated_client, transition +): + client = generated_client + await client.reset(split="all", index=0) + assert names(await client.list_tools()) == ["echo", "end_turn"] + if transition == "step": + await client.step( + CallToolAction(tool_name="end_turn", arguments={"response": "Done."}) + ) + else: + name = "end_turn" if transition == "end_turn" else "echo" + result = await client.call_tool( + name, response="Done.", use_cache="tool argument" + ) + assert result["result"] == {"response": "Done.", "use_cache": "tool argument"} + # Explicit cache requests retain the inherited API, including positional use. + assert names(await client.list_tools(use_cache=True)) == ["echo", "end_turn"] + assert names(await client.list_tools(True)) == ["echo", "end_turn"] + assert names(await client.list_tools()) == ["next", "end_turn"] + assert names(await client.list_tools(use_cache=False)) == ["next", "end_turn"] + assert (await client.get_tool("next")).name == "next"