diff --git a/AGENTS.md b/AGENTS.md index 36204fd0a..7a896ca64 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,9 +107,9 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`: files, a `python` requirements file for a venv, and the services the base's init runs; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile and build context so any env with the same inputs reuses it; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver). An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`: files, a `python` requirements file for a venv, and the services the base's init runs; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile and build context so any env with the same inputs reuses it; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name). `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 8d8dc1182..27a9a4635 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -537,6 +537,49 @@ leasing, and unverified for one with it until the probe has lease checks. The pr default role; the tool-access check hides a tool from a role of its own, `agentenv-probe-`, and gives it back. +## Computer apps + +A computer app is software an agent uses on a computer, beside the driver: a desktop app, a service +or both. Write one as a `ComputerAppEnvironment` subclass from `agentenv_protocol.computer`, an +`AgentEnvEnvironment` whose class also says what it is to a computer. It needs only the plain +package: `agentenv_protocol.computer` imports what a driver needs (mcp, uvicorn, Pillow, PyAV) only +when a driver's names are used. + +```python +from pydantic import BaseModel, SecretStr +from agentenv_protocol.computer import ComputerAppEnvironment, Launch, Linux, Service, computer_app + +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +class MailApp(ComputerAppEnvironment): + linux = Linux(python="requirements.txt", services=[Service(name="imap", command="venv/bin/python imap.py")]) + launch = Launch(command=["thunderbird"], window={"class": "thunderbird"}) + skill = "skill.md" + + class Settings(BaseModel): + signature: str = "" + password: SecretStr = SecretStr("") +``` + +- **`@computer_app`** names the platforms the app supports, as a driver names them, and the + `permissions` it needs of the computer. Its env.toml names the rest: the app itself + (`environment_name`) and the `computer` and `driver` it deploys alone on. +- **`linux = Linux(...)`** is how a Linux computer installs it: `files`, a folder of the app's + copied under its prefix; `python`, a requirements file there installed into a venv beside them; + `services`, programs the computer's init runs from the prefix as the session user. +- **`launch = Launch(...)`** says how it starts in the desktop session, with it (`at="boot"`) or when + the agent opens it (`"demand"`), and `ready=Ready(...)` when it's ready; `@ready_check` marks a + method for readiness that needs logic, and `@install_step(scope=...)` one a computer runs as it + installs the app. +- **Its skill** is a file of its folder (`skill = "skill.md"`) or an `@agent_skill` method, not both; + its data hooks are the usual `@add_data`, `@get_data` and `@reset_data`. +- **`Settings`** is typed; a computer hands each app the values a deploy gives it, as one JSON value + in `APP_SETTINGS_ENV` (`AGENTENV_APP_SETTINGS`, the apps' values by name), and the base class reads + its own, by `ENVIRONMENT_NAME`, as `self.settings`. A `SecretStr` field is a secret. + +`MailApp.declaration()` returns what the class declares, an `AppDeclaration`, which a `computer_app` +env stores, and `python -m agentenv_protocol.computer._declare ` prints it as +JSON, or a driver's `DriverDeclaration`, without the class's own dependencies installed. + ## A2A agent framework The distribution exposes two unrelated decorators named `extension`: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py index 66afa4c69..45af04bf8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/__init__.py @@ -1,7 +1,25 @@ -"""The computer driver SDK: subclass ComputerDriverEnvironment, mark the methods that capture and send input, and -serve() it. Needs the ``computer`` extra: ``pip install 'agentenv-framework-protocol[computer]'``.""" +"""The computer SDK. A driver subclasses ComputerDriverEnvironment, marks the methods that capture and send input, and +serve()s it, which needs the ``computer`` extra: ``pip install 'agentenv-framework-protocol[computer]'``. An app +subclasses ComputerAppEnvironment, which needs only the plain package, so what a driver needs is imported on first +use.""" -from .driver import DEFAULT_IDENTITY, ComputerDriverEnvironment, computer_driver +from __future__ import annotations + +import importlib +from typing import TYPE_CHECKING, Any + +from .app import ( + APP_SETTINGS_ENV, + AppDeclaration, + ComputerAppEnvironment, + Launch, + Linux, + Ready, + Service, + computer_app, + install_step, + ready_check, +) from .errors import ( BackgroundUnavailable, ComputerError, @@ -18,16 +36,6 @@ Timeout, Unsupported, ) -from .card import ( - COMPUTER, - COMPUTER_DISPLAY, - COMPUTER_EXEC, - COMPUTER_KEYBOARD, - COMPUTER_POINTER, - COMPUTER_READY, - COMPUTER_RECORDING, - COMPUTER_SPACES, -) from .operations import ( Display, Effect, @@ -54,11 +62,43 @@ ) from .manifest import ManifestSegment, RecordingManifest from .platform import Platform -from .recording import RecordedFrame, RecordedSegment, RecordHints, Recording, frames_in_thread from .settings import SETTINGS_ENV, ComputerSettings +if TYPE_CHECKING: + from .card import ( + COMPUTER, + COMPUTER_DISPLAY, + COMPUTER_EXEC, + COMPUTER_KEYBOARD, + COMPUTER_POINTER, + COMPUTER_READY, + COMPUTER_RECORDING, + COMPUTER_SPACES, + ) + from .driver import DEFAULT_IDENTITY, ComputerDriverEnvironment, DriverDeclaration, computer_driver + from .recording import RecordedFrame, RecordedSegment, RecordHints, Recording, frames_in_thread + +# The names whose modules need the computer extra (mcp, av and Pillow), by module. +_ON_USE = { + "driver": ("ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", "DriverDeclaration"), + "card": ("COMPUTER", "COMPUTER_READY", "COMPUTER_DISPLAY", "COMPUTER_SPACES", "COMPUTER_POINTER", + "COMPUTER_KEYBOARD", "COMPUTER_EXEC", "COMPUTER_RECORDING"), + "recording": ("Recording", "RecordHints", "RecordedFrame", "RecordedSegment", "frames_in_thread"), +} + + +def __getattr__(name: str) -> Any: + module = next((module for module, names in _ON_USE.items() if name in names), None) + if module is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + return getattr(importlib.import_module(f"{__name__}.{module}"), name) + + __all__ = [ - "ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", + "ComputerDriverEnvironment", "computer_driver", "DEFAULT_IDENTITY", "DriverDeclaration", + "ComputerAppEnvironment", "computer_app", "install_step", "ready_check", "Linux", "Service", "Launch", "Ready", + "AppDeclaration", + "APP_SETTINGS_ENV", "Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV", "display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press", "pointer_move", "pointer_down", "pointer_up", "pointer_position", "key_down", "key_up", diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py index 812cd0e44..8f8d723ac 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/_declare.py @@ -1,8 +1,8 @@ -"""Read a computer driver class where its own dependencies may not be installed, as a ``computer_driver`` env's put -does: ``python -m agentenv_protocol.computer._declare `` imports the file with every module -not installed here standing in as an inert stub, and prints what the class declares for each platform it supports as -JSON. Nothing runs beyond defining the module and its classes, and a class that can't be defined even so exits 1 -with why.""" +"""Read a computer driver or app class where its own dependencies may not be installed, as a ``computer_driver`` or +``computer_app`` env's put does: ``python -m agentenv_protocol.computer._declare `` imports the +file with every module not installed here standing in as an inert stub, and prints what the class declares as JSON, +as its base class says. Nothing runs beyond defining the module and its classes, and a class that can't be defined +even so exits 1 with why.""" from __future__ import annotations import contextlib @@ -14,10 +14,14 @@ import types from pathlib import Path +from .app import AppDeclaration, ComputerAppEnvironment from .card import COMPUTER -from .driver import ComputerDriverEnvironment +from .driver import ComputerDriverEnvironment, DriverDeclaration, DriverPlatform from .platform import supported +#: The env type that stores each kind of declaration, which the printed JSON carries as ``type``. +_ENV_TYPES = {AppDeclaration: "computer_app", DriverDeclaration: "computer_driver"} + class _StubMeta(type): def __getattr__(cls, name: str): @@ -58,32 +62,35 @@ def exec_module(self, module) -> None: return None -def declare(folder: Path, file: str, name: str) -> dict: - """What the class ``name`` in ``folder/file`` declares: its card's name, its ``supports``, and for each platform - those name, the extensions it provides, its tools and the computer/v1 features. Raises ValueError for a class - that isn't a driver or declares no platform.""" +def declare(folder: Path, file: str, name: str) -> AppDeclaration | DriverDeclaration: + """What the class ``name`` in ``folder/file`` declares: an app's ``ComputerAppEnvironment.declaration()``; a + driver's card name, its ``supports``, and for each platform those name, the extensions it provides, its tools and + the computer/v1 features. Raises ValueError for a class that is neither, or a driver that declares no platform.""" sys.path.insert(0, str(folder)) sys.meta_path.append(_Stubs()) spec = importlib.util.spec_from_file_location("computer_driver_module", folder / file) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) cls = getattr(module, name, None) + if isinstance(cls, type) and issubclass(cls, ComputerAppEnvironment): + return cls.declaration() if not (isinstance(cls, type) and issubclass(cls, ComputerDriverEnvironment)): - raise ValueError(f"{file} has no ComputerDriverEnvironment subclass named {name}") + raise ValueError(f"{file} has no ComputerDriverEnvironment or ComputerAppEnvironment subclass named {name}") platforms = [platform for entry in cls.supports for platform in supported(entry)] if not platforms: raise ValueError(f"{name} declares no platform: name them with @computer_driver(supports=[...])") cards = [(platform, cls.declaration(platform)) for platform in platforms] - return {"name": cards[0][1].name, "supports": list(cls.supports), "platforms": [ - {"platform": platform.as_dict(), "provides": [e.uri for e in card.capabilities.extensions], - "tools": [t.name for t in card.capabilities.tools], - "features": next({k: v for k, v in e.params.items() if k not in ("endpoint", "methods")} - for e in card.capabilities.extensions if e.uri == COMPUTER)} - for platform, card in cards]} + return DriverDeclaration(name=cards[0][1].name, supports=list(cls.supports), platforms=[ + DriverPlatform(platform=platform, provides=[e.uri for e in card.capabilities.extensions], + tools=[t.name for t in card.capabilities.tools], + features=next({k: v for k, v in e.params.items() if k not in ("endpoint", "methods")} + for e in card.capabilities.extensions if e.uri == COMPUTER)) + for platform, card in cards]) def main(argv: list[str]) -> int: - """Print the declaration as the only output; what the module prints while it loads goes to stderr.""" + """Print the declaration as the only output, its env type as ``type``; what the module prints while it loads goes + to stderr.""" folder, file, name = argv try: with contextlib.redirect_stdout(sys.stderr): @@ -91,7 +98,7 @@ def main(argv: list[str]) -> int: except Exception as e: print(f"{type(e).__name__}: {e}", file=sys.stderr) return 1 - print(json.dumps(declared)) + print(json.dumps({"type": _ENV_TYPES[type(declared)], **declared.model_dump(mode="json", exclude_none=True)})) return 0 diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py new file mode 100644 index 000000000..d72a78738 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/app.py @@ -0,0 +1,178 @@ +"""ComputerAppEnvironment: software on a computer, an AgentEnvEnvironment whose class also declares what it is for a +computer: the platforms it supports and the permissions it needs (``@computer_app``), how each OS installs it +(``linux = Linux(...)``), how it launches (``launch``), its install steps and ready checks, its skill, and its typed +``Settings``. A ``computer_app`` env's put reads that declaration from the class; its data hooks are the base class's. +Its name is its env's ``environment_name``.""" +from __future__ import annotations + +import inspect +import json +import os +from typing import Any, Callable, ClassVar, Literal, Optional + +from pydantic import BaseModel, ConfigDict, Field + +from ..agent_env_environment import _EXT_ATTR, _OP_ATTR, AgentEnvEnvironment +from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, SKILL_EXTENSION_URI + +#: The environment variable a computer delivers its apps' settings in: JSON of each app's values by its name. +APP_SETTINGS_ENV = "AGENTENV_APP_SETTINGS" +_INSTALL_STEP_ATTR = "_aee_install_step" +_READY_CHECK_ATTR = "_aee_ready_check" +_DATA_METHODS = {"reset_data": METHOD_RESET, "add_data": METHOD_ADD, "get_data": METHOD_GET} + + +class Service(BaseModel): + """A program the computer's init runs from the app's folder as the session user; a relative program names one + there.""" + + model_config = ConfigDict(extra="forbid") + + name: str + command: str + + +class Linux(BaseModel): + """How a Linux computer installs the app: ``files``, a folder of the app's copied under its prefix; ``python``, a + requirements file in that folder installed into a venv beside them; ``services``, programs its init runs.""" + + model_config = ConfigDict(extra="forbid") + + files: str = "." + python: Optional[str] = None + services: list[Service] = Field(default_factory=list) + + +class Ready(BaseModel): + """When the app is ready: its window shown, a port answering or a command exiting 0, within ``timeout_s``.""" + + model_config = ConfigDict(extra="forbid") + + window: bool = False + port: Optional[int] = None + command: Optional[list[str]] = None + timeout_s: float = 60 + + +class Launch(BaseModel): + """How the app starts in the desktop session: with it (``boot``) or when the agent opens it (``demand``).""" + + model_config = ConfigDict(extra="forbid") + + at: Literal["boot", "demand"] = "boot" + command: list[str] + window: Optional[dict[str, str]] = None + ready: Optional[Ready] = None + + +class AppSkill(BaseModel): + """Where an app's skill for agents comes from: a ``file`` of its folder, or the ``@agent_skill`` ``method`` that + writes it.""" + + model_config = ConfigDict(extra="forbid") + + file: Optional[str] = None + method: Optional[str] = None + + +class InstallStep(BaseModel): + """An ``@install_step`` method, by name, and whom it runs as.""" + + model_config = ConfigDict(extra="forbid") + + name: str + scope: Literal["system", "session"] + + +class AppDeclaration(BaseModel): + """What a ComputerAppEnvironment class declares, as a ``computer_app`` env stores it: the platforms it ``supports``, + as a driver names them; its ``install`` by OS type; ``settings``, the JSON schema of its ``Settings``; and the + ``data`` methods, ``install_steps`` and ``ready_checks`` it serves, by name.""" + + model_config = ConfigDict(extra="forbid") + + supports: list[dict[str, Any]] + permissions: list[str] + install: dict[str, Linux] + launch: Optional[Launch] = None + settings: dict[str, Any] + skill: Optional[AppSkill] = None + data: list[str] + install_steps: list[InstallStep] + ready_checks: list[str] + + +def computer_app(*, supports: Optional[list] = None, permissions: Optional[list[str]] = None) -> Callable: + """Declare what an app is: the platforms it supports, as a driver names them, and the permissions it needs of the + computer. Its env.toml names it and the computer and driver it deploys alone on.""" + def deco(cls: type) -> type: + if not (isinstance(cls, type) and issubclass(cls, ComputerAppEnvironment)): + raise TypeError("@computer_app decorates a ComputerAppEnvironment subclass") + cls.supports, cls.permissions = list(supports or []), list(permissions or []) + return cls + return deco + + +def install_step(scope: Literal["system", "session"] = "system") -> Callable: + """Mark a method a computer runs as it installs the app, after its files, in the order the class defines them: as + root, or for ``session``, as the session user.""" + def deco(fn: Callable) -> Callable: + setattr(fn, _INSTALL_STEP_ATTR, scope) + return fn + return deco + + +def ready_check(fn: Callable) -> Callable: + """Mark a method that says whether the app is ready, when ``launch``'s ``ready`` can't.""" + setattr(fn, _READY_CHECK_ATTR, True) + return fn + + +class ComputerAppEnvironment(AgentEnvEnvironment): + """Subclass it, declare it with ``@computer_app``, and set ``linux``, ``launch`` and ``skill`` as class attributes; + a deploy's settings arrive as ``self.settings``.""" + + supports: ClassVar[list] = [] + permissions: ClassVar[list[str]] = [] + linux: ClassVar[Optional[Linux]] = None + launch: ClassVar[Optional[Launch]] = None + #: A file of the app's folder holding its skill for agents, when no ``@agent_skill`` method writes it. + skill: ClassVar[Optional[str]] = None + + class Settings(BaseModel): + """The app's settings: an app declares its own; a ``SecretStr`` field is a secret.""" + + def __init__(self) -> None: + values = json.loads(os.environ.get(APP_SETTINGS_ENV) or "{}").get(os.environ.get("ENVIRONMENT_NAME"), {}) + self.settings = self.Settings.model_validate(values) + + @classmethod + def declaration(cls) -> AppDeclaration: + """What the class declares. Raises ValueError for a skill declared twice or an install that isn't a + ``Linux``.""" + if cls.linux is not None and not isinstance(cls.linux, Linux): + raise ValueError(f"{cls.__name__}.linux is a Linux(...), not {cls.linux!r}") + methods = _methods(cls) + skill_methods = [fn.__name__ for fn in methods + if getattr(getattr(fn, _EXT_ATTR, None), "uri", None) == SKILL_EXTENSION_URI] + if cls.skill is not None and skill_methods: + raise ValueError(f"{cls.__name__} declares its skill twice: skill = {cls.skill!r} and @agent_skill " + f"{skill_methods[0]}") + return AppDeclaration( + supports=cls.supports, permissions=cls.permissions, + install={"linux": cls.linux} if cls.linux is not None else {}, + launch=cls.launch, settings=cls.Settings.model_json_schema(), + skill=AppSkill(file=cls.skill) if cls.skill is not None else + AppSkill(method=skill_methods[0]) if skill_methods else None, + data=sorted({_DATA_METHODS[op] for fn in methods if (op := getattr(fn, _OP_ATTR, None)) in _DATA_METHODS}), + install_steps=[InstallStep(name=fn.__name__, scope=getattr(fn, _INSTALL_STEP_ATTR)) for fn in methods + if hasattr(fn, _INSTALL_STEP_ATTR)], + ready_checks=[fn.__name__ for fn in methods if hasattr(fn, _READY_CHECK_ATTR)], + ) + + +def _methods(cls: type) -> list[Callable]: + """The class's methods, each as the class resolves its name, in the order they were first defined.""" + names = dict.fromkeys(name for klass in reversed(cls.__mro__) for name, member in vars(klass).items() + if inspect.isfunction(member)) + return [method for name in names if inspect.isfunction(method := getattr(cls, name))] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py index b70320e9d..172b57703 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/driver.py @@ -5,6 +5,8 @@ import asyncio from typing import Any, Callable, ClassVar +from pydantic import BaseModel, ConfigDict + from ..agent_env_environment import _CARD_CONFIG_ATTR, AgentEnvEnvironment, AgentEnvFastMCPApplication, environment_card from ..tool_context import ToolContext from ..types import EnvironmentCard, EnvironmentExtension, EnvironmentTool @@ -23,6 +25,29 @@ DEFAULT_IDENTITY = "default" +class DriverPlatform(BaseModel): + """What a driver serves on one platform its ``supports`` names: the extensions it ``provides``, its ``tools`` and + its computer/v1 ``features``.""" + + model_config = ConfigDict(extra="forbid") + + platform: Platform + provides: list[str] + tools: list[str] + features: dict[str, Any] + + +class DriverDeclaration(BaseModel): + """What a ComputerDriverEnvironment class declares, as a ``computer_driver`` env stores it: its card's ``name``, the + platforms it ``supports``, as predicates, and what it serves on each of them.""" + + model_config = ConfigDict(extra="forbid") + + name: str + supports: list[dict[str, Any]] + platforms: list[DriverPlatform] + + class ComputerDriverEnvironment(Tools, Extensions, DataPlane, AgentEnvEnvironment): """Subclass it and mark the methods that capture and send input (see ``operations``); the class is checked when it is defined. Features are class attributes, on the card. A subclass's ``__init__`` calls ``super().__init__()``. diff --git a/packages/agentenv-protocol/tests/test_computer_declare.py b/packages/agentenv-protocol/tests/test_computer_declare.py index ac4ac2b0b..a0fbccda3 100644 --- a/packages/agentenv-protocol/tests/test_computer_declare.py +++ b/packages/agentenv-protocol/tests/test_computer_declare.py @@ -1,9 +1,12 @@ -"""Reading a driver class where its own dependencies aren't installed: a module no finder finds imports as an inert -stub, the class is read without running it, and what it declares for each platform it supports comes back.""" +"""Reading a driver or app class where its own dependencies aren't installed: a module no finder finds imports as an +inert stub, the class is read without running it, and what it declares comes back: a driver's for each platform it +supports, an app's as ComputerAppEnvironment says. An app's code needs only the plain package.""" from __future__ import annotations import json +import os import re +import subprocess import sys import pytest @@ -11,7 +14,9 @@ pytest.importorskip("mcp.server.fastmcp") pytest.importorskip("PIL") +from agentenv_protocol.computer import AppDeclaration # noqa: E402 from agentenv_protocol.computer._declare import _Stub, _StubModule, declare, main # noqa: E402 +from agentenv_protocol.computer.platform import Platform # noqa: E402 DRIVER = '''import not_installed_here from not_installed_here.input import Base, Keyboard @@ -52,6 +57,45 @@ def type_in(self, text): pass @press def press_keys(self, keys): pass ''' +APP = '''import not_installed_here +from pydantic import BaseModel, SecretStr + +from agentenv_protocol import add_data, agent_skill, reset_data +from agentenv_protocol.computer import (ComputerAppEnvironment, Launch, Linux, Ready, Service, computer_app, + install_step, ready_check) + + +@computer_app(supports={supports}, permissions=["camera"]) +class Mail(ComputerAppEnvironment): + linux = Linux(files="linux", python="requirements.txt", services=[Service(name="imap", command="bin/imap")]) + launch = Launch(at="demand", command=["thunderbird"], ready=Ready(window=True, timeout_s=30)) + + class Settings(BaseModel): + account: str = "user@example.com" + password: SecretStr = SecretStr("") + + @install_step() + def packages(self): not_installed_here.apt() + + @install_step(scope="session") + def profile(self): pass + + @ready_check + def inbox_open(self): return True + + @agent_skill + def guide(self): return "Mail", "Read mail." + + @add_data + async def load(self, parts): pass + + @reset_data + async def reset(self): pass + + +class Twice(Mail): + skill = "skill.md" +''' LINUX = [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}] @@ -74,12 +118,12 @@ def test_a_class_whose_imports_arent_installed_declares_what_it_serves_on_each_p declared = declare(folder, "driver.py", "Desk") - assert declared["name"] == "desk" and declared["supports"] == LINUX - assert [p["platform"] for p in declared["platforms"]] == [{"os": "linux", "arch": "amd64", "display": "x11"}, - {"os": "linux", "arch": "arm64", "display": "x11"}] - linux = declared["platforms"][0] - assert {"urn:agentenv:computer/v1", "urn:agentenv:computer-exec/v1"} <= set(linux["provides"]) - assert len(linux["tools"]) == 7 and linux["features"]["buttons"] == ["left", "right"] + assert declared.name == "desk" and declared.supports == LINUX + assert [p.platform for p in declared.platforms] == [Platform(os="linux", arch="amd64", display="x11"), + Platform(os="linux", arch="arm64", display="x11")] + linux = declared.platforms[0] + assert {"urn:agentenv:computer/v1", "urn:agentenv:computer-exec/v1"} <= set(linux.provides) + assert len(linux.tools) == 7 and linux.features["buttons"] == ["left", "right"] def test_a_stub_is_itself_again_except_for_dunders(folder): @@ -93,8 +137,8 @@ def test_a_stub_is_itself_again_except_for_dunders(folder): @pytest.mark.parametrize("name, supports, problem", [ - ("Connection", LINUX, "driver.py has no ComputerDriverEnvironment subclass named Connection"), - ("Missing", LINUX, "driver.py has no ComputerDriverEnvironment subclass named Missing"), + ("Connection", LINUX, "driver.py has no ComputerDriverEnvironment or ComputerAppEnvironment subclass named Connection"), + ("Missing", LINUX, "driver.py has no ComputerDriverEnvironment or ComputerAppEnvironment subclass named Missing"), ("Desk", [], "Desk declares no platform: name them with @computer_driver(supports=[...])"), ]) def test_a_name_that_isnt_a_driver_or_declares_no_platform_is_refused(folder, name, supports, problem): @@ -108,9 +152,71 @@ def test_main_prints_only_the_declaration_or_exits_1_with_why_it_cant(folder, ca _driver(folder) assert main([str(folder), "driver.py", "Desk"]) == 0 printed = capsys.readouterr() - assert json.loads(printed.out)["name"] == "desk" and printed.err == "loading the desk driver\n" + printed_declaration = json.loads(printed.out) + assert (printed_declaration["type"], printed_declaration["name"]) == ("computer_driver", "desk") + assert printed_declaration["platforms"][0]["platform"] == {"os": "linux", "arch": "amd64", "display": "x11"} + assert printed.err == "loading the desk driver\n" _driver(folder, buttons=("thumb",)) assert main([str(folder), "driver.py", "Desk"]) == 1 printed = capsys.readouterr() assert printed.out == "" and printed.err.splitlines()[-1].startswith("TypeError: Desk: buttons include left") + + +def test_an_app_class_whose_imports_arent_installed_declares_what_it_is_to_a_computer(folder): + (folder / "app.py").write_text(APP.format(supports=LINUX)) + + declared = declare(folder, "app.py", "Mail") + + assert isinstance(declared, AppDeclaration) + settings = declared.settings + assert declared.model_dump(mode="json", exclude_none=True, exclude={"settings"}) == { + "supports": LINUX, "permissions": ["camera"], + "install": {"linux": {"files": "linux", "python": "requirements.txt", + "services": [{"name": "imap", "command": "bin/imap"}]}}, + "launch": {"at": "demand", "command": ["thunderbird"], "ready": {"window": True, "timeout_s": 30.0}}, + "skill": {"method": "guide"}, "data": ["data/add", "data/reset"], + "install_steps": [{"name": "packages", "scope": "system"}, {"name": "profile", "scope": "session"}], + "ready_checks": ["inbox_open"]} + assert list(settings["properties"]) == ["account", "password"] and settings["properties"]["password"]["writeOnly"] + + +def test_an_app_declaring_its_skill_twice_is_refused(folder): + (folder / "app.py").write_text(APP.format(supports=LINUX)) + + with pytest.raises(ValueError, match=re.escape("Twice declares its skill twice: skill = 'skill.md' and " + "@agent_skill guide")): + declare(folder, "app.py", "Twice") + + +def test_an_app_reads_the_settings_its_computer_gives_it_by_its_name(monkeypatch): + from pydantic import BaseModel + + from agentenv_protocol.computer import APP_SETTINGS_ENV, ComputerAppEnvironment + + class Notes(ComputerAppEnvironment): + class Settings(BaseModel): + signature: str = "" + lines: int = 1 + + monkeypatch.setenv("ENVIRONMENT_NAME", "notes") + monkeypatch.setenv(APP_SETTINGS_ENV, json.dumps({"notes": {"signature": "-- Ada"}, "mail": {"lines": 9}})) + + assert Notes().settings == Notes.Settings(signature="-- Ada", lines=1) + + +def test_an_apps_code_needs_only_the_plain_package(): + blocked = """ +import importlib.abc, sys +class Blocked(importlib.abc.MetaPathFinder): + def find_spec(self, name, path, target=None): + if name.split(".")[0] in ("mcp", "av", "PIL", "uvicorn"): + raise ImportError(name) +sys.meta_path.insert(0, Blocked()) +from agentenv_protocol.computer import ComputerAppEnvironment, Linux, computer_app +print(sorted(name for name in sys.modules if name.startswith("agentenv_protocol.computer."))) +""" + done = subprocess.run([sys.executable, "-c", blocked], capture_output=True, text=True, env=os.environ) + + assert done.returncode == 0, done.stderr + assert "agentenv_protocol.computer.driver" not in done.stdout diff --git a/pyproject.toml b/pyproject.toml index 4c45765a7..87382d35d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "click>=8.0.0", "cryptography>=42", # the local transfer CA "httpx>=0.27.0", + "jsonschema>=4.20", # checks an app's settings against its Settings' schema before a deploy "litellm>=1.96.0,<1.97", "mcp>=1.25,<2", # same floor as env/gateway/requirements.txt and agentenv-protocol; 2.0.0 removed mcp.server.fastmcp, imported at module scope by env/gateway "packaging>=22", # evaluates a plugin's requirement markers with no extra diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index fd6d09a1a..fabdeb7c3 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -35,7 +35,8 @@ from agent_env.env.env import Env from agent_env.env.envs._deployment import provider_refusal from agent_env.env.envs.computer.env import ComputerEnv -from agent_env.env.envs.computer_driver import ComputerDriverEnv +from agent_env.env.envs.computer.app_env import ComputerAppEnv +from agent_env.env.envs.computer.driver_env import ComputerDriverEnv from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv from agent_env.env.envs.website import WebsiteEnv @@ -98,9 +99,9 @@ def _stored_env(id: str, version: int | None) -> Env: _FOLDER_HOOKS = {"written_besides": Artifact, "seeded_environments": SeedsEnvironments} # The env types whose from_toml reads only the toml and what it names, so a type built by one of them is written from # its env.toml and the ledger can list what it's made from. -_ENV_FROM_TOMLS = (Env, MCPServerEnv, WebsiteEnv, MultiEnv, ComputerDriverEnv, ComputerEnv) +_ENV_FROM_TOMLS = (Env, MCPServerEnv, WebsiteEnv, MultiEnv, ComputerDriverEnv, ComputerEnv, ComputerAppEnv) # The env types whose from_toml also reads every file in the env's folder, which the ledger hashes as an artifact's. -_ENV_FROM_FOLDERS = (ComputerDriverEnv,) +_ENV_FROM_FOLDERS = (ComputerDriverEnv, ComputerAppEnv) _NOTHING_TO_RUN = "this bundle has no tasks or evals to run" diff --git a/src/agent_env/cli/env/build.py b/src/agent_env/cli/env/build.py index f2ac2a72b..f40a06951 100644 --- a/src/agent_env/cli/env/build.py +++ b/src/agent_env/cli/env/build.py @@ -4,7 +4,7 @@ import click from agent_env.env import Env -from agent_env.env.envs.computer import composer +from agent_env.env.envs.computer.utils import composer from agent_env.env.envs.computer.env import ComputerEnv from agent_env.store.base import NotFoundError @@ -21,6 +21,8 @@ def build(env_id: str, version: int | None): raise click.ClickException(str(e)) from None if not isinstance(env, ComputerEnv): raise click.ClickException(f"{env_id} is a {env.type} env; only a computer env is built") + if env.computer is None: + raise click.ClickException(f"{env_id} names no computer and driver, so it has nothing to build alone") if (pairs := composer.pending(env)) or not composer.current(env): composer.compose(env, pairs) failed = False diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index aaa54ccc3..3808fcfc7 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -25,5 +25,13 @@ A computer env composes a driver onto it at put: an `envs//env.toml` of `t machine's Docker can build, then boots the one this machine runs natively and probes it. The shipped `desktop` example is that env, and `agent-env run desktop` deploys it. +A computer app env (`type = "computer_app"`) deployed alone is composed onto it the same way, with the app's files +under `/opt/agentenv/apps//` and its services as programs beside the driver's, run as `user` like everything else. + +Everything in a computer runs as that one user, so any code in it, an app's, a driver's, a `run_code` script or a +command run through `computer-exec`, can read the environment of any other process there, settings included. A secret +setting, the base's or an app's, is kept out of the stored env, the task and the run's record, which hold only its +`secret:` or `env:` reference, but not from code in the computer: give a computer only secrets its own code may see. + This bundle has no tasks of its own: other bundles pull its computers by alias, and `agent-env run computers` has nothing to run. diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py new file mode 100644 index 000000000..a05ca8719 --- /dev/null +++ b/src/agent_env/env/envs/computer/app_env.py @@ -0,0 +1,147 @@ +"""Computer app env: software on a computer, a ``ComputerAppEnvironment`` subclass, with its files and what it declares, +read from the class at put. Deployed alone it is a one-app computer env: the computer and driver its env.toml names, +composed with the app installed under its prefix, its card named after it.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self + +from agentenv_protocol.computer import AppDeclaration + +from agent_env.artifact import Artifact +from agent_env.artifact.artifacts.computer import ComputerArtifact +from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse +from agent_env.env.env import Env +from agent_env.env.envs.computer.utils import composer, gate +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer.utils.install import AppInstall, installs, missing +from agent_env.env.envs.computer.driver_env import ComputerDriverEnv, declared +from agent_env.env.store import get_env_store +from agent_env.store.ids import derive_id + +if TYPE_CHECKING: + from collections.abc import Callable + + from agent_env.bundle.authoring import AuthoringContext + + +class ComputerAppEnv(ComputerEnv): + type: ClassVar[str] = "computer_app" + description = ("A computer app: software and its settings on a computer, read from its ComputerAppEnvironment " + "class at put; deployed alone, a computer of its own") + + def __init__(self, id: str, version: Optional[int], environment_name: str, module: str, files: FileArtifactUniverse, + declaration: AppDeclaration, computer: Optional[ComputerArtifact] = None, + driver: Optional[ComputerDriverEnv] = None, metadata: Optional[dict[str, Any]] = None): + super().__init__(id, version, environment_name, computer, driver, metadata=metadata) + self.module = module + self.files = files + self.declaration = declaration + + @property + def install(self) -> dict[str, AppInstall]: + return _installs(self.declaration)[0] + + def composed_apps(self) -> list[ComputerAppEnv]: + return [self] + + def secret_settings(self) -> set[str]: + """The names of its settings that are secrets: a ``SecretStr``, optional or not, which its schema marks + ``writeOnly``.""" + return {name for name, schema in self.declaration.settings.get("properties", {}).items() + if any(option.get("writeOnly") for option in [schema, *schema.get("anyOf", [])])} + + def to_dict(self) -> dict: + return {**Env.to_dict(self), "service_name": self.environment_name, + "environment_name": self.environment_name, # dual-write with service_name during the rename + "module": self.module, + "files": {"id": self.files.id, "version": self.files.version, "type": self.files.type}, + "declaration": self.declaration.model_dump(mode="json", exclude_none=True), + "computer": {"id": self.computer.id, "version": self.computer.version} if self.computer else None, + "driver": {"id": self.driver.id, "version": self.driver.version} if self.driver else None} + + @classmethod + def from_dict(cls, data: dict) -> ComputerAppEnv: + driver = Env.get(data["driver"]["id"], data["driver"]["version"]) if data.get("driver") else None + if driver is not None and not isinstance(driver, ComputerDriverEnv): + raise TypeError(f"{data['driver']['id']} is a {driver.type} env, not a computer_driver env") + computer = ComputerArtifact.get(data["computer"]["id"], data["computer"]["version"]) if data.get("computer") else None + return cls(id=data["id"], version=data.get("version"), environment_name=data["environment_name"], + module=data["module"], files=Artifact.get(data["files"]["id"], version=data["files"]["version"]), + declaration=AppDeclaration.model_validate(data["declaration"]), computer=computer, driver=driver, + metadata=data.get("metadata") or {}) + + @classmethod + def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifactUniverse, + declaration: AppDeclaration, computer: Optional[ComputerArtifact] = None, + driver: Optional[ComputerDriverEnv] = None, compose: bool = True, + on_build: Optional[Callable[[], None]] = None) -> Self: + """Write the app once what it declares checks out, and with a computer and a driver, once it fits them (gate + 1); then compose it with them unless ``compose`` is False. Refused, naming every problem, before anything is + stored.""" + app = cls(id=id, version=None, environment_name=environment_name, module=module, files=files, + declaration=declaration, computer=computer, driver=driver) + _, problems = _installs(declaration) + if (computer is None) != (driver is None): + problems.append("a computer app deploys alone on a computer and a driver; name both or neither") + elif computer is not None and not problems: + _, refused = gate.check(computer, driver, apps=[], display=None, network=None, computer_settings={}) + problems += refused + gate.check_app(computer, driver, app) + if problems: + raise ValueError(f"computer app env {id!r}: " + "; ".join(problems)) + app = get_env_store().put_document(app) + if computer is not None and compose: + composer.compose(app, on_build=on_build) + return app + + @classmethod + def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerAppEnv: + """Write the app an env.toml declares, its class read in a process of its own (``declared``), its files stored + as a file universe, with the computer and driver env.toml names; its environment_name the folder's when it sets + none.""" + fields = cls.accept_toml(data, ctx) + declaration = AppDeclaration.model_validate(declared(ctx, fields["module"], cls.type)) + files = ctx.files() + found, problems = _installs(declaration) + problems += [problem for os_type, install in found.items() for problem in missing(os_type, install, files)] + if (skill := declaration.skill and declaration.skill.file) is not None and skill not in files: + problems.append(f"skill: {skill!r} isn't in this folder") + if problems: + ctx.refuse(list(map(ctx.config_problem, problems))) + computer = ctx.artifact(fields["computer"], ComputerArtifact) if "computer" in fields else None + driver = ctx.env(fields["driver"], ComputerDriverEnv) if "driver" in fields else None + universe = FileArtifactUniverse.put_bundled(derive_id(ctx.id, "files"), files=files) + try: + return cls.put(id=ctx.id, environment_name=fields.get("environment_name", ctx.name), + module=fields["module"], files=universe, declaration=declaration, computer=computer, + driver=driver, on_build=ctx.on_build) + except ValueError as e: + ctx.refuse([str(e)]) + + @classmethod + def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: + """The keys of ``data``, an env.toml, a computer app env takes: ``module = ":"``, naming a file of + its folder; ``environment_name``; and the ``computer`` and ``driver`` it deploys alone on. Raises BundleError + listing every problem.""" + fields, refused = ctx.accepted(data, environment_name=str, module=str, computer=object, driver=object) + files, module, problems = ctx.files(), fields.get("module"), [] + if module is None and "module" not in data: + problems.append('module is required: the app\'s file and class, such as "app.py:MailApp"') + elif module is not None and (not module.partition(":")[2] or module.partition(":")[0] not in files): + problems.append(f'module {module!r} names no ":" in this folder; it has ' + f"{', '.join(sorted(files))}") + if refused or problems: + ctx.refuse([*refused, *map(ctx.config_problem, problems)]) + return fields + + def deploy_refusal(self, *, sandbox_type: str | None = None, env_params: Optional[dict] = None, + **options) -> str | None: + if self.computer is None: + return (f"computer app env {self.id!r} names no computer and driver to deploy alone on; name them in its " + "env.toml") + return super().deploy_refusal(sandbox_type=sandbox_type, env_params=env_params, **options) + + +def _installs(declaration: AppDeclaration) -> tuple[dict[str, AppInstall], list[str]]: + """Its installs by OS type, each checked as the composer installs it, and the problems with them.""" + return installs({os_type: linux.model_dump() for os_type, linux in declaration.install.items()}, AppInstall) diff --git a/src/agent_env/env/envs/computer_driver.py b/src/agent_env/env/envs/computer/driver_env.py similarity index 70% rename from src/agent_env/env/envs/computer_driver.py rename to src/agent_env/env/envs/computer/driver_env.py index be2b2c05e..08b4a73bf 100644 --- a/src/agent_env/env/envs/computer_driver.py +++ b/src/agent_env/env/envs/computer/driver_env.py @@ -7,13 +7,14 @@ import json import subprocess import sys -from pathlib import PurePosixPath from typing import TYPE_CHECKING, Any, ClassVar, NoReturn, Optional +from agentenv_protocol.computer.driver import DriverDeclaration + from agent_env.artifact import Artifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.env.env import Env -from agent_env.env.envs.computer.install import Install, installs +from agent_env.env.envs.computer.utils.install import Install, installs, missing from agent_env.store.ids import derive_id if TYPE_CHECKING: @@ -31,20 +32,19 @@ class ComputerDriverEnv(Env): type: ClassVar[str] = "computer_driver" description = "A computer driver: the code that serves a computer's tools, and what it declares" - def __init__(self, id: str, version: Optional[int], module: str, files: FileArtifactUniverse, declaration: dict, - install: Optional[dict[str, Install]] = None, port: int = DEFAULT_PORT, + def __init__(self, id: str, version: Optional[int], module: str, files: FileArtifactUniverse, + declaration: DriverDeclaration, install: Optional[dict[str, Install]] = None, port: int = DEFAULT_PORT, metadata: Optional[dict[str, Any]] = None): super().__init__(id, version, metadata=metadata) self.module = module self.files = files - #: The card's ``name``, the class's ``supports``, and per platform those name the extensions it ``provides``, - #: its ``tools`` and its computer/v1 ``features``. self.declaration = declaration self.install = install or {} self.port = port def to_dict(self) -> dict: - return {**super().to_dict(), "module": self.module, "declaration": self.declaration, + return {**super().to_dict(), "module": self.module, + "declaration": self.declaration.model_dump(mode="json", exclude_none=True), "files": {"id": self.files.id, "version": self.files.version, "type": self.files.type}, "install": {os_type: install.model_dump(exclude_none=True) for os_type, install in self.install.items()}, "port": self.port} @@ -54,16 +54,14 @@ def from_dict(cls, data: dict) -> ComputerDriverEnv: files = Artifact.get(data["files"]["id"], version=data["files"]["version"]) install = {os_type: Install.model_validate(table) for os_type, table in (data.get("install") or {}).items()} return cls(id=data["id"], version=data.get("version"), module=data["module"], files=files, - declaration=data["declaration"], install=install, port=data.get("port", DEFAULT_PORT), - metadata=data.get("metadata") or {}) + declaration=DriverDeclaration.model_validate(data["declaration"]), install=install, + port=data.get("port", DEFAULT_PORT), metadata=data.get("metadata") or {}) @classmethod def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerDriverEnv: - """The driver's class is read in a process of its own (``agentenv_protocol.computer._declare``), so the - driver's own dependencies need not be installed here.""" + """The driver's class is read in a process of its own (``declared``).""" fields = cls.accept_toml(data, ctx) - file, _, name = fields["module"].partition(":") - declaration = _declare(ctx, file, name) + declaration = DriverDeclaration.model_validate(declared(ctx, fields["module"], cls.type)) universe = FileArtifactUniverse.put_bundled(derive_id(ctx.id, "files"), files=ctx.files()) return cls.put(id=ctx.id, module=fields["module"], files=universe, declaration=declaration, install=fields["install"], port=fields.get("port", DEFAULT_PORT)) @@ -85,7 +83,7 @@ def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: problems.append(f"port must be 1 to {MAX_PORT}, not {fields['port']}") fields["install"], invalid = installs(fields.get("install", {})) problems += invalid + [problem for os_type, install in fields["install"].items() - for problem in _missing(os_type, install, files)] + for problem in missing(os_type, install, files)] if refused or problems: ctx.refuse([*refused, *map(ctx.config_problem, problems)]) return fields @@ -95,26 +93,22 @@ async def deploy(self, **kwargs) -> DeployedEnv: "own: name it as a computer env's driver and deploy that") -def _missing(os_type: str, install: Install, files: dict) -> list[str]: - """What ``install`` names that the driver's folder, ``files`` by path, doesn't hold.""" - folder = PurePosixPath(install.files) - problems = [] - if folder != PurePosixPath(".") and not any(PurePosixPath(path).is_relative_to(folder) for path in files): - problems.append(f"install.{os_type}.files: this folder has no {install.files!r} folder") - if install.python is not None and (folder / install.python).as_posix() not in files: - problems.append(f"install.{os_type}.python: {(folder / install.python).as_posix()!r} isn't in this folder") - return problems - - -def _declare(ctx: AuthoringContext, file: str, name: str) -> dict: +def declared(ctx: AuthoringContext, module: str, env_type: str) -> dict: + """What the class ``module`` names in the folder declares, read in a process of its own + (``agentenv_protocol.computer._declare``), so its own dependencies need not be installed here; refused unless an + env of ``env_type`` stores it.""" + file, _, name = module.partition(":") try: done = subprocess.run([sys.executable, "-m", "agentenv_protocol.computer._declare", str(ctx.dir), file, name], capture_output=True, text=True, timeout=DECLARE_TIMEOUT_S, check=False) except subprocess.TimeoutExpired: - _refuse(ctx, f"reading {file}:{name} took over {DECLARE_TIMEOUT_S} s") + _refuse(ctx, f"reading {module} took over {DECLARE_TIMEOUT_S} s") if done.returncode != 0: - _refuse(ctx, f"{file}:{name} can't be read: {(done.stderr.strip().splitlines() or ['no output'])[-1]}") - return json.loads(done.stdout) + _refuse(ctx, f"{module} can't be read: {(done.stderr.strip().splitlines() or ['no output'])[-1]}") + declaration = json.loads(done.stdout) + if (declares := declaration.pop("type")) != env_type: + _refuse(ctx, f"{module} is a class a {declares} env declares, not a {env_type} env") + return declaration def _refuse(ctx: AuthoringContext, problem: str) -> NoReturn: diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 940edd22e..da811607b 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -5,13 +5,14 @@ from __future__ import annotations import logging +import posixpath from collections.abc import Callable from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self from agent_env.artifact.artifacts.computer import ComputerArtifact, SettingValue from agent_env.attribution import Attribution from agent_env.entity.refs import EntityRef -from agent_env.env.env import Env, gateway_url_of +from agent_env.env.env import Env, LoadFileArtifactUniverseResult, gateway_url_of from agent_env.env.envs._deployment import ( as_builtin, builtin_provider_for, @@ -20,12 +21,13 @@ deploy_through_provider, provider_or_class, ) -from agent_env.env.envs.computer import composer, gate -from agent_env.env.envs.computer import params as computer_params -from agent_env.env.envs.computer_driver import ComputerDriverEnv +from agent_env.env.envs.computer.utils import composer, gate, inside +from agent_env.env.envs.computer.utils import params as computer_params +from agent_env.env.envs.computer.driver_env import ComputerDriverEnv from agent_env.env.gateway import GatewayMode from agent_env.env.store import get_env_store from agent_env.providers.env_providers.env_computer_provider import end_recordings +from agent_env.utils.paths import validate_relative_filename if TYPE_CHECKING: from agent_env.bundle.authoring import AuthoringContext @@ -34,6 +36,9 @@ logger = logging.getLogger(__name__) +#: Where load_artifact puts files in a computer by default: one of the folders data/get returns by default. +DEFAULT_DESTINATION = "Downloads" + class ComputerEnv(Env): type: ClassVar[str] = "computer" @@ -110,6 +115,10 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: except ValueError as e: ctx.refuse([str(e)]) + def composed_apps(self) -> list: + """The computer app envs composed into the env's computer, each installed under its prefix.""" + return [] + def computer_artifact(self) -> ComputerArtifact: """The env's computer: its base composed with its driver, over the images built.""" return ComputerArtifact.get(self.metadata["computer"]["id"], self.metadata["computer"]["version"]) @@ -148,7 +157,7 @@ def deploy_refusal(self, *, sandbox_type: str | None = None, env_params: Optiona return refusal _, _, problems = computer_params.checked(self, env_params) sandbox_provider = build_sandbox_provider(sandbox_type) if sandbox_type else get_env_sandbox_provider() - links, arches = sandbox_provider.links, gate.arches(self.computer, self.driver) + links, arches = sandbox_provider.links, gate.arches(self.computer, self.driver, self.composed_apps()) if not any(f"linux/{arch}" in link.native_platforms() for link in links for arch in arches): problems.append(f"{', '.join(type(link).__name__ for link in links)} runs none of its arches " f"({', '.join(arches)}) natively") @@ -168,6 +177,24 @@ async def from_deployed_env(cls, deployed: DeployedEnv) -> ComputerEnv: env._deployed = deployed return env + async def load_file_artifact_universe(self, file_artifact_universe: Any, + destination_path: Optional[str] = None) -> LoadFileArtifactUniverseResult: + """Write each file of ``file_artifact_universe`` under ``destination_path``, a folder relative to the session + user's home (``Downloads`` by default), through the driver's ``data/add``, as that user; the result's paths are + relative to the home too, since the home's path differs by OS.""" + if destination_path is not None and posixpath.isabs(destination_path): + raise ValueError(f"{self.id}: a computer's destination_path is relative to its session user's home, not " + f"{destination_path!r}") + destination = posixpath.normpath(destination_path or DEFAULT_DESTINATION) + validate_relative_filename(destination) + files = file_artifact_universe.get_file_artifacts() + for filename in files: + validate_relative_filename(filename) + paths = {filename: posixpath.normpath(posixpath.join(destination, filename)) for filename in files} + await inside.add_artifacts(self._deployed, {paths[filename]: artifact for filename, artifact in files.items()}) + logger.info(f"Loaded {len(files)} file(s) of '{file_artifact_universe.id}' into {self.id}'s ~/{destination}") + return LoadFileArtifactUniverseResult(destination_path=destination, files=paths) + async def reset(self, deployed: DeployedEnv) -> None: """Return the computer to boot state: its recordings ended, its container replaced with a fresh one of the same image, on the same sandbox and port, so its address stays. An agent's MCP session ends with the old driver; one diff --git a/src/agent_env/env/envs/computer/utils/__init__.py b/src/agent_env/env/envs/computer/utils/__init__.py new file mode 100644 index 000000000..4bd04817d --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/__init__.py @@ -0,0 +1 @@ +"""What the computer env types share: install formats, fit checks, composition, boot, params and driver I/O.""" diff --git a/src/agent_env/env/envs/computer/boot.py b/src/agent_env/env/envs/computer/utils/boot.py similarity index 86% rename from src/agent_env/env/envs/computer/boot.py rename to src/agent_env/env/envs/computer/utils/boot.py index 37bee0cfd..2f114e46b 100644 --- a/src/agent_env/env/envs/computer/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING, Any, Optional from agentenv_protocol import client as protocol_v1 -from agentenv_protocol.computer import COMPUTER_READY, SETTINGS_ENV +from agentenv_protocol.computer import APP_SETTINGS_ENV, COMPUTER_READY, SETTINGS_ENV from agent_env.config import get_config from agent_env.providers.env_providers.env_provider import _size_kwargs @@ -21,6 +21,7 @@ if TYPE_CHECKING: from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.attribution import Attribution + from agent_env.env.envs.computer.app_env import ComputerAppEnv from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy, Sandbox from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider @@ -62,6 +63,17 @@ def settings_environment(computer: ComputerArtifact, values: dict[str, Any], *, return environment +def app_settings_environment(apps: list[ComputerAppEnv], app_settings: dict[str, dict]) -> dict[str, str]: + """The environment that hands each of ``apps`` the values ``app_settings`` gives it, by its name, in + ``APP_SETTINGS_ENV``, a secret's resolved through the secret store; nothing when none has any.""" + given = [(app, app_settings[app.environment_name]) for app in apps if app_settings.get(app.environment_name)] + if not given: + return {} + return {APP_SETTINGS_ENV: json.dumps({app.environment_name: { + name: get_config().resolve_reference(value) if name in app.secret_settings() and value is not None else value + for name, value in values.items()} for app, values in given})} + + def recorded_settings(computer: ComputerArtifact, values: dict[str, Any]) -> dict[str, Any]: """The deploy-phase settings a boot applies as a record keeps them, a secret's and a generated one's withheld.""" return {name: WITHHELD if setting.secret or setting.generate else values.get(name, setting.default) diff --git a/src/agent_env/env/envs/computer/composer.py b/src/agent_env/env/envs/computer/utils/composer.py similarity index 81% rename from src/agent_env/env/envs/computer/composer.py rename to src/agent_env/env/envs/computer/utils/composer.py index 053f93412..495f682b5 100644 --- a/src/agent_env/env/envs/computer/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -1,6 +1,6 @@ -"""The composer: a computer env's base and driver built into one image for each form and arch with this machine's Docker, -tagged in the env's image store by a key of everything that goes into it, so an image of that key, built for any env, is -reused. +"""The composer: a computer env's base, driver and apps built into one image for each form and arch with this machine's +Docker, tagged in the env's image store by a key of everything that goes into it, so an image of that key, built for any +env, is reused. Each result is recorded in the env's metadata as it finishes (``composed``), and the arches that built are written as the env's computer, a ``computer`` artifact over an index of their images.""" @@ -21,6 +21,7 @@ import threading import urllib.parse from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path, PurePosixPath from collections.abc import Callable @@ -31,11 +32,12 @@ from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact, docker_login, push_local_image from agent_env.artifact.artifacts.file import FileArtifact +from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.config import get_config from agent_env.entity.export import Unwritable, checked_path from agent_env.env.env import Env -from agent_env.env.envs.computer import boot, gate -from agent_env.env.envs.computer.install import Install +from agent_env.env.envs.computer.utils import boot, gate +from agent_env.env.envs.computer.utils.install import Install from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag from agent_env.store.base import ConcurrentModificationError from agent_env.store.ids import derive_id @@ -49,14 +51,16 @@ from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider from agent_env.env.envs.computer.env import ComputerEnv - from agent_env.env.envs.computer_driver import ComputerDriverEnv + from agent_env.env.envs.computer.app_env import ComputerAppEnv + from agent_env.env.envs.computer.driver_env import ComputerDriverEnv logger = logging.getLogger(__name__) #: The image store repository composed computers are pushed to, tagged by their keys. REPOSITORY = "agentenv-computers" -#: Where a driver's files go in a composed computer, in a folder named for the driver. +#: Where a driver's files go in a composed computer, in a folder named for the driver; an app's, under APPS. DRIVERS = "/opt/agentenv/drivers" +APPS = "/opt/agentenv/apps" KEY_LABEL = "dev.agentenv.computer-key" # What a build installed and whether it was emulated, kept on its image for an env that reuses it. INSTALLED_LABEL, EMULATED_LABEL = "dev.agentenv.installed", "dev.agentenv.emulated" @@ -84,7 +88,8 @@ def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, here; reusing them all calls it never.""" base, driver = env.computer, env.driver building = _once(on_build) - pairs = pairs if pairs is not None else [(form, arch) for form in gate.forms(base) for arch in gate.arches(base, driver)] + pairs = pairs if pairs is not None else [(form, arch) for form in gate.forms(base) + for arch in gate.arches(base, driver, env.composed_apps())] record = _Record(env) record.started(pairs) with ThreadPoolExecutor(max_workers=max(len(pairs), 1)) as pool: @@ -114,7 +119,8 @@ def pending(env: ComputerEnv) -> list[tuple[str, str]]: composed, provider = env.metadata.get("composed", {}), get_env_sandbox_provider() statuses = {(form, arch): composed.get(form, {}).get(arch, {}).get("status") - for form in gate.forms(env.computer) for arch in gate.arches(env.computer, env.driver)} + for form in gate.forms(env.computer) + for arch in gate.arches(env.computer, env.driver, env.composed_apps())} return [(form, arch) for (form, arch), status in statuses.items() if status not in _BUILT or (status == "unverified" and boot.native_links(provider, arch))] @@ -184,8 +190,8 @@ def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, context = Path(work) source, base_identity = _base(base, arch) layer = _stage_driver(context, driver, base) - dockerfile = _dockerfile(source, layer, driver.install[base.os.type], base, _build_settings(env, base), - _stage_protocol(context)) + apps = [_stage_app(context, app, base) for app in env.composed_apps()] + dockerfile = _dockerfile(source, [layer, *apps], base, _build_settings(env, base), _stage_protocol(context)) key = "sha256:" + hashlib.sha256(json.dumps({ "base": base_identity, "dockerfile": dockerfile, "context": _digest(context), "form": form, "arch": arch, "composer": _composer_digest(), @@ -208,7 +214,7 @@ def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, (context / "Dockerfile").write_text(f"{dockerfile}LABEL {KEY_LABEL}={key}\n") local = f"agentenv-computer:{key[7:23]}-{arch}" build_image(context / "Dockerfile", context, local, platform=platform, labels={KEY_LABEL: key}) - installed = _installed(local, layer) + installed = _installed(local, layer.prefix) _label(local, platform, {INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false"}) stage = "push" @@ -260,38 +266,69 @@ def _ready_base(base: ComputerArtifact, arch: str) -> None: build_for(image, f"linux/{arch}") -def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifact) -> str: - """Write the driver's install files under ``context/driver/`` and its supervisord programs beside them; the name - of its folder under ``DRIVERS``, its id's last part.""" - install = driver.install[base.os.type] - layer = re.sub(r"[^a-z0-9-]+", "-", driver.id.rsplit("/", 1)[-1].lower()).strip("-") or "driver" - folder = PurePosixPath(install.files) - for name, artifact in driver.files.get_file_artifacts().items(): +@dataclass(frozen=True) +class _Layer: + """A driver's or an app's part of a build: its folder in the build context, its prefix in the image, what it + installs, and the file of its supervisord programs.""" + + folder: str + prefix: str + install: Install + conf: str + + +def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifact) -> _Layer: + """Write the driver's install files under ``context/driver/`` and its supervisord programs beside them, served on + its port, its folder under ``DRIVERS`` named for its id's last part.""" + install, name = driver.install[base.os.type], _name(driver.id, "driver") + layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-{name}.conf") + _stage(context, layer, f"driver {driver.id}", driver.files, name, {"MCP_PORT": driver.port}) + return layer + + +def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact) -> _Layer: + """Write the app's install files under ``context/apps//`` and its supervisord programs beside them, run under + its name, its folder under ``APPS`` named for its id's last part.""" + install, name = app.install[base.os.type], _name(app.id, "app") + layer = _Layer(f"apps/{name}", f"{APPS}/{name}", install, f"agentenv-app-{name}.conf") + _stage(context, layer, f"app {app.id}", app.files, f"app-{name}", {"ENVIRONMENT_NAME": app.environment_name}) + return layer + + +def _name(env_id: str, fallback: str) -> str: + return re.sub(r"[^a-z0-9-]+", "-", env_id.rsplit("/", 1)[-1].lower()).strip("-") or fallback + + +def _stage(context: Path, layer: _Layer, what: str, files: FileArtifactUniverse, programs: str, + environment: dict[str, Any]) -> None: + """Write the ``files`` of ``what``'s folder its install names to ``layer``'s folder, a script or binary executable, + and its services as supervisord programs named for ``programs``, with ``environment``.""" + folder = PurePosixPath(layer.install.files) + for name, artifact in files.get_file_artifacts().items(): try: path = PurePosixPath(checked_path(name)) except Unwritable as e: - raise ValueError(f"driver {driver.id}: {e}") from None + raise ValueError(f"{what}: {e}") from None if folder == PurePosixPath(".") or path.is_relative_to(folder): - target = context / "driver" / path.relative_to(folder) + target = context / layer.folder / path.relative_to(folder) target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(data := artifact.load()) if data.startswith(_RUN_ONLY): target.chmod(_EXECUTABLE) - (context / "driver").mkdir(exist_ok=True) - (context / f"agentenv-{layer}.conf").write_text(_programs(layer, install, driver.port)) - return layer + (context / layer.folder).mkdir(parents=True, exist_ok=True) + (context / layer.conf).write_text(_programs(programs, layer.prefix, layer.install, environment)) -def _programs(layer: str, install: Install, port: int) -> str: - prefix = f"{DRIVERS}/{layer}" +def _programs(name: str, prefix: str, install: Install, environment: dict[str, Any]) -> str: + variables = ",".join(f'{key}="{value}"' for key, value in environment.items()) programs = [] for service in install.services: program, _, rest = service.command.partition(" ") if "/" in program and not program.startswith("/"): program = f"{prefix}/{program}" - programs.append(f"[program:{layer}-{service.name}]\ncommand={program} {rest}".rstrip() + "\n" - f"directory={prefix}\nenvironment=MCP_PORT=\"{port}\"\nautorestart=true\nredirect_stderr=true\n" - f"stdout_logfile=/var/log/agentenv/{layer}-{service.name}.log\n") + programs.append(f"[program:{name}-{service.name}]\ncommand={program} {rest}".rstrip() + "\n" + f"directory={prefix}\nenvironment={variables}\nautorestart=true\nredirect_stderr=true\n" + f"stdout_logfile=/var/log/agentenv/{name}-{service.name}.log\n") return "\n".join(programs) @@ -328,34 +365,36 @@ def _values(env: ComputerEnv) -> dict[str, Any]: return {**env.computer_settings, **({"resolution": f"{display['width']}x{display['height']}"} if display else {})} -def _dockerfile(source: str, layer: str, install: Install, base: ComputerArtifact, build_settings: dict, - protocol: bool) -> str: - """The composed image's Dockerfile but its key's label, which the key is a digest of.""" - prefix = f"{DRIVERS}/{layer}" - lines = [f"FROM {source}", "USER root", f"COPY driver/ {prefix}/"] - if install.python is not None: - packages = f"/tmp/agentenv-protocol -r {prefix}/{install.python}" if protocol else f"-r {prefix}/{install.python}" - if protocol: - lines.append("COPY protocol /tmp/agentenv-protocol") - lines.append(f"RUN python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip install --no-cache-dir {packages} " - f"&& {prefix}/venv/bin/pip list --format=json > {prefix}/installed.json" - + (" && rm -rf /tmp/agentenv-protocol" if protocol else "")) - lines.append(f"COPY agentenv-{layer}.conf /etc/supervisor/conf.d/agentenv-{layer}.conf") - checks = [f"test -d {prefix}", f"test -f /etc/supervisor/conf.d/agentenv-{layer}.conf"] - if install.python is not None: - checks.append(f"test -x {prefix}/venv/bin/python") - lines.append(f"RUN {' && '.join(checks)}") +def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build_settings: dict, protocol: bool) -> str: + """The composed image's Dockerfile but its key's label, which the key is a digest of: each layer's files under its + prefix, its venv beside them, with this checkout's protocol when ``protocol``, and its programs.""" + venvs = [layer for layer in layers if layer.install.python is not None] + lines = [f"FROM {source}", "USER root", *(["COPY protocol /tmp/agentenv-protocol"] if protocol and venvs else [])] + for layer in layers: + prefix, install = layer.prefix, layer.install + lines.append(f"COPY {layer.folder}/ {prefix}/") + if install.python is not None: + packages = f"/tmp/agentenv-protocol -r {prefix}/{install.python}" if protocol else f"-r {prefix}/{install.python}" + lines.append(f"RUN python3 -m venv {prefix}/venv && {prefix}/venv/bin/pip install --no-cache-dir {packages} " + f"&& {prefix}/venv/bin/pip list --format=json > {prefix}/installed.json") + lines.append(f"COPY {layer.conf} /etc/supervisor/conf.d/{layer.conf}") + checks = [f"test -d {prefix}", f"test -f /etc/supervisor/conf.d/{layer.conf}"] + if install.python is not None: + checks.append(f"test -x {prefix}/venv/bin/python") + lines.append(f"RUN {' && '.join(checks)}") + if protocol and venvs: + lines.append("RUN rm -rf /tmp/agentenv-protocol") lines += [f"ENV AGENTENV_SETTING_{name.upper()}={json.dumps(str(value))}" for name, value in build_settings.items()] lines.append(f"USER {base.session.user}") return "\n".join(lines) + "\n" -def _installed(local: str, layer: str) -> dict[str, str]: - """The packages ``local``'s build installed into the driver's venv, by name, read from the image without running it, - so on any arch; none for a driver with no venv.""" +def _installed(local: str, prefix: str) -> dict[str, str]: + """The packages ``local``'s build installed into the venv of the driver at ``prefix``, by name, read from the image + without running it, so on any arch; none for a driver with no venv.""" created = subprocess.run(["docker", "create", local], capture_output=True, text=True, check=True).stdout.strip() try: - copied = subprocess.run(["docker", "cp", f"{created}:{DRIVERS}/{layer}/installed.json", "-"], + copied = subprocess.run(["docker", "cp", f"{created}:{prefix}/installed.json", "-"], capture_output=True) finally: subprocess.run(["docker", "rm", created], capture_output=True) diff --git a/src/agent_env/env/envs/computer/gate.py b/src/agent_env/env/envs/computer/utils/gate.py similarity index 61% rename from src/agent_env/env/envs/computer/gate.py rename to src/agent_env/env/envs/computer/utils/gate.py index 187be2d7a..4aed65c25 100644 --- a/src/agent_env/env/envs/computer/gate.py +++ b/src/agent_env/env/envs/computer/utils/gate.py @@ -1,11 +1,13 @@ -"""Gate 1, run at a computer env's put before anything is stored or built: its base and driver are matched against each -other and against what the env declares, and every mismatch is named.""" +"""Gate 1, run at a computer env's put before anything is stored or built: its base, driver and apps are matched against +each other and against what the env declares, and every mismatch is named.""" from __future__ import annotations +from collections.abc import Sequence from typing import TYPE_CHECKING, Any from agentenv_protocol.computer.platform import Platform, supported +from jsonschema import Draft202012Validator from agent_env.artifact.artifacts.computer import FORMS, ComputerArtifact, SettingValue from agent_env.config.loader import ENV_REF_PREFIX, SECRET_REF_PREFIX @@ -13,7 +15,8 @@ from agent_env.store.image_store.oci_registry_credentials import names_registry if TYPE_CHECKING: - from agent_env.env.envs.computer_driver import ComputerDriverEnv + from agent_env.env.envs.computer.app_env import ComputerAppEnv + from agent_env.env.envs.computer.driver_env import ComputerDriverEnv #: The largest screen side X can address: its coordinates are signed 16-bit. MAX_SIDE = 32767 @@ -39,13 +42,53 @@ def forms(base: ComputerArtifact) -> list[str]: return [form for form in FORMS if getattr(base.forms, form, None)] -def arches(base: ComputerArtifact, driver: ComputerDriverEnv) -> list[str]: - """The architectures a composition of ``base`` and ``driver`` runs on: the base's that a platform of the driver's - matching the base's OS names, in the base's order.""" - matching = {platform.arch for platform in _matching(base, driver)} +def arches(base: ComputerArtifact, driver: ComputerDriverEnv, apps: Sequence[ComputerAppEnv] = ()) -> list[str]: + """The architectures a composition of ``base``, ``driver`` and ``apps`` runs on: the base's that a platform of the + driver's and of each app's matching the base's OS names, in the base's order.""" + matching = {platform.arch for platform in _matching(base, driver.declaration.supports)} + for app in apps: + matching &= {platform.arch for platform in _matching(base, app.declaration.supports)} return [arch for arch in base.arch if arch in matching] +def check_app(base: ComputerArtifact, driver: ComputerDriverEnv, app: ComputerAppEnv) -> list[str]: + """Every way ``app`` doesn't fit a composition of ``base`` and ``driver``: a platform it doesn't support, no + architecture in common, or no install for the base's OS.""" + os = base.os + if not _matching(base, app.declaration.supports): + named = ", ".join(_shown(platform) for entry in app.declaration.supports for platform in supported(entry)) + return [f"app: {app.id} supports no platform {base.id} is ({os.type} {os.distribution} {os.version} " + f"{os.display}); it supports {named or 'none'}"] + problems = [] + if not arches(base, driver, [app]): + problems.append(f"app: no architecture {base.id}, {driver.id} and {app.id} all run") + if os.type not in app.install: + problems.append(f"app: {app.id} declares no install for {os.type}") + return problems + + +def check_app_settings(app: ComputerAppEnv, values: dict[str, Any], at: str) -> list[str]: + """The problems with ``values``, of ``app``'s settings by name, each named from ``at``: a setting it doesn't declare, + a secret's given as anything but a bare reference (or null, where it's optional), a required one missing, and what + the schema of its ``Settings`` refuses of the rest. A secret's reference never meets the constraints on its value, + and its value never reaches a message.""" + schema, secrets = app.declaration.settings, app.secret_settings() + properties, required = schema.get("properties", {}), schema.get("required", []) + problems = [f"{at}.{name}: {app.id} declares no setting {name!r}" for name in values if name not in properties] + problems += [f"{at}.{name} is a secret, so it takes a {SECRET_REF_PREFIX}KEY or {ENV_REF_PREFIX}NAME reference " + "with no ?default, not its value" for name in values if name in secrets + and not (_names_a_secret(values[name]) or values[name] is None and _optional(properties[name]))] + problems += [f"{at}: {name!r} is a required property" for name in required if name in secrets and name not in values] + plain = {**schema, "properties": {name: setting for name, setting in properties.items() if name not in secrets}, + "required": [name for name in required if name not in secrets]} + return problems + [f"{'.'.join([at, *map(str, error.absolute_path)])}: {error.message}" for error in + Draft202012Validator(plain).iter_errors({k: v for k, v in values.items() if k not in secrets})] + + +def _optional(setting: dict) -> bool: + return any(option.get("type") == "null" for option in setting.get("anyOf", [])) + + def check_computer_settings(base: ComputerArtifact, values: Any) -> tuple[dict[str, SettingValue], list[str]]: """``values``, of the base's settings by name, and the problems with them: a setting the base doesn't declare, a value of another type than its default's, or a secret's given as anything but a bare reference, since the env, the @@ -78,14 +121,14 @@ def _names_a_secret(value: Any) -> bool: def _platform(base: ComputerArtifact, driver: ComputerDriverEnv) -> list[str]: os = base.os - if not _matching(base, driver): - named = ", ".join(_shown(platform) for entry in driver.declaration["supports"] for platform in supported(entry)) + if not _matching(base, driver.declaration.supports): + named = ", ".join(_shown(platform) for entry in driver.declaration.supports for platform in supported(entry)) return [f"driver: {driver.id} supports no platform {base.id} is ({os.type} {os.distribution} {os.version} " f"{os.display}); it supports {named}"] problems = [] if not arches(base, driver): problems.append(f"driver: no architecture both run: {base.id} runs {', '.join(base.arch)}, and {driver.id} " - f"{', '.join(sorted({platform.arch for platform in _matching(base, driver)}))}") + f"{', '.join(sorted({p.arch for p in _matching(base, driver.declaration.supports)}))}") if (install := driver.install.get(os.type)) is None: problems.append(f"driver: {driver.id} declares no install for {os.type}") else: @@ -95,9 +138,10 @@ def _platform(base: ComputerArtifact, driver: ComputerDriverEnv) -> list[str]: return problems -def _matching(base: ComputerArtifact, driver: ComputerDriverEnv) -> list[Platform]: +def _matching(base: ComputerArtifact, supports: list) -> list[Platform]: + """The platforms of ``supports``, as a driver or app declares them, that the base's OS is.""" os = base.os - return [platform for entry in driver.declaration["supports"] for platform in supported(entry) + return [platform for entry in supports for platform in supported(entry) if platform.os == os.type and platform.display == os.display and platform.distribution in (None, os.distribution) and platform.version in (None, os.version)] diff --git a/src/agent_env/env/envs/computer/utils/inside.py b/src/agent_env/env/envs/computer/utils/inside.py new file mode 100644 index 000000000..f1033e23c --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/inside.py @@ -0,0 +1,128 @@ +"""What reaches inside a deployed computer through its driver, as its session user, at paths relative to the home: +``data/add`` writes files, in batches the driver holds in memory one at a time, each file's bytes coming through a read +grant when the object store's grants reach the computer and its network allows everything, else inline; ``data/get`` +reads one back; computer-exec/v1 ``run`` runs a command.""" + +from __future__ import annotations + +import asyncio +import base64 +import uuid +from typing import TYPE_CHECKING, Optional + +import httpx +from agentenv_protocol import DATA_OBJECTS_EXTENSION_URI, METHOD_ADD, METHOD_GET, FilePart +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.computer import COMPUTER_EXEC +from agentenv_protocol.types import FileWithBytes, FileWithUri + +from agent_env.a2a_agent.object_transfer import read_object +from agent_env.config import get_config +from agent_env.env.gateway.constants import data_plane_load_timeout_s +from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy +from agent_env.store.object_store.object_store import issues_grants_to + +if TYPE_CHECKING: + from agent_env.artifact import FileArtifact + from agent_env.env.env import DeployedSandboxEnv + +#: The most bytes one ``data/add`` carries: the driver reads a call's files into memory before it writes them. A file +#: bigger than this goes alone. +BATCH_BYTES = 64 << 20 +#: How much longer than a command's own timeout its call waits for the answer. +CALL_MARGIN_S = 30 +_READ_TIMEOUT_S = 120 + + +def missing(deployed: DeployedSandboxEnv) -> list[str]: + """What the env's card doesn't serve of computer-exec/v1 ``run``, ``data/add`` and ``data/get``.""" + operations = ((deployed.environment_card or {}).get("capabilities") or {}).get("operations") + absent = [] if deployed.supports(COMPUTER_EXEC, "run") else [f"{COMPUTER_EXEC} run"] + return absent + [op for op in (METHOD_ADD, METHOD_GET) if operations is not None and op not in operations] + + +def max_timeout_s(deployed: DeployedSandboxEnv) -> float: + """The longest a command may run, as the card's computer-exec/v1 says.""" + return protocol_v1.extension_params(deployed.environment_card, COMPUTER_EXEC)["max_timeout_s"] + + +def loopback(deployed: DeployedSandboxEnv) -> dict: + """The env's id, address and MCP endpoint as code inside the computer reaches them: the driver on loopback.""" + url = f"http://127.0.0.1:{deployed.metadata['computer']['port']}" + return {"id": deployed.env_id, "url": url, "mcp_url": url + deployed.mcp_url.removeprefix(deployed.environment_url)} + + +async def add_artifacts(deployed: DeployedSandboxEnv, files: dict[str, FileArtifact]) -> None: + """Write each file artifact at its path.""" + network = (deployed.metadata or {}).get("computer", {}).get("network") + granted = not (network and NetworkPolicy.from_dict(network).restricts_egress) + batch: list[FilePart] = [] + size = 0 + for path, artifact in files.items(): + part, part_size = await asyncio.to_thread(_part, path, artifact, deployed.sandbox_type, granted) + if batch and size + part_size > BATCH_BYTES: + await _add(deployed, batch, size) + batch, size = [], 0 + batch.append(part) + size += part_size + if batch: + await _add(deployed, batch, size) + + +async def add_files(deployed: DeployedSandboxEnv, files: dict[str, bytes]) -> None: + """Write each file's bytes at its path, inline, in one call.""" + await _add(deployed, [_inline(path, data) for path, data in files.items()], sum(map(len, files.values()))) + + +async def read_file(deployed: DeployedSandboxEnv, path: str) -> bytes: + """The bytes of the file at ``path``; FileNotFoundError when there is none.""" + try: + answer = await protocol_v1.get_data(deployed.environment_url, timeout=_READ_TIMEOUT_S, paths=[path]) + except httpx.TransportError as e: + raise _broken(deployed, "data/get", e) from e + for part in answer.parts: + if (part.metadata or {}).get(DATA_OBJECTS_EXTENSION_URI, {}).get("path") == path: + return base64.b64decode(part.file.bytes) + raise FileNotFoundError(f"no file {path} in the computer") + + +async def run(deployed: DeployedSandboxEnv, argv: list[str], *, timeout_s: float, + env: Optional[dict[str, str]] = None) -> dict: + """computer-exec/v1 ``run``'s answer for ``argv``, under a request id of its own. A call whose connection breaks + fails rather than being sent again: a driver that restarted would run the command a second time, beside the first, + which outlives the driver that started it.""" + params = {"argv": argv, "request_id": uuid.uuid4().hex, "timeout_s": timeout_s, **({"env": env} if env else {})} + try: + return await deployed.invoke(COMPUTER_EXEC, "run", params, timeout=timeout_s + CALL_MARGIN_S) + except httpx.TransportError as e: + raise _broken(deployed, "computer-exec/v1 run", e) from e + + +async def _add(deployed: DeployedSandboxEnv, parts: list[FilePart], size: int) -> None: + try: + await protocol_v1.add_data(deployed.environment_url, parts, timeout=data_plane_load_timeout_s(size)) + except httpx.TransportError as e: + raise _broken(deployed, "data/add, so some of the files it carried may be written, each whole", e) from e + + +def _broken(deployed: DeployedSandboxEnv, during: str, error: httpx.TransportError) -> RuntimeError: + """What a call to the computer's driver failed by when its connection broke, which httpx often says nothing of.""" + return RuntimeError(f"env '{deployed.env_id}': the connection to its driver broke during {during} " + f"({type(error).__name__}{f': {error}' if str(error) else ''})") + + +def _part(path: str, artifact: FileArtifact, sandbox_type: Optional[str], granted: bool) -> tuple[FilePart, int]: + """The ``data/add`` part writing ``artifact`` at ``path``, and how many bytes the driver reads for it.""" + store = get_config().get_object_store_at(artifact.object_url) + if granted and issues_grants_to(store, sandbox_type): + grant = read_object(store, artifact.object_url) + return FilePart(file=FileWithUri(uri=path, name=artifact.filename, mimeType=artifact.content_type), + metadata={DATA_OBJECTS_EXTENSION_URI: {"path": path, "read": grant.model_dump(mode="json")}} + ), grant.size_bytes + data = artifact.load() + return _inline(path, data, artifact.filename, artifact.content_type), len(data) + + +def _inline(path: str, data: bytes, name: Optional[str] = None, mime_type: Optional[str] = None) -> FilePart: + return FilePart(file=FileWithBytes(bytes=base64.b64encode(data).decode("ascii"), name=name, mimeType=mime_type), + metadata={DATA_OBJECTS_EXTENSION_URI: {"path": path}}) diff --git a/src/agent_env/env/envs/computer/install.py b/src/agent_env/env/envs/computer/utils/install.py similarity index 59% rename from src/agent_env/env/envs/computer/install.py rename to src/agent_env/env/envs/computer/utils/install.py index b37bc5daf..6691c922c 100644 --- a/src/agent_env/env/envs/computer/install.py +++ b/src/agent_env/env/envs/computer/utils/install.py @@ -1,6 +1,7 @@ -"""What a computer's layer installs on one OS, as a driver's env.toml declares it under ``[install.]``: ``files``, a -folder of its own copied under the layer's prefix; ``python``, a requirements file in that folder installed into a -venv beside them; and ``services``, the programs the base's init runs from the prefix.""" +"""What a computer's layer installs on one OS, as a driver's env.toml declares it under ``[install.]`` or an app's +class as ``linux = Linux(...)``: ``files``, a folder of its own copied under the layer's prefix; ``python``, a +requirements file in that folder installed into a venv beside them; and ``services``, the programs the base's init +runs from the prefix.""" from __future__ import annotations @@ -46,9 +47,15 @@ def _named_once(cls, value: list[Service]) -> list[Service]: return value -def installs(data: Any) -> tuple[dict[str, Install], list[str]]: - """The installs ``data``, an env.toml's ``install`` table, declares by OS type, and the problems with it, each - named by its key.""" +class AppInstall(Install): + """An app's install, which may run no service: a desktop app runs when it's launched.""" + + services: list[Service] = Field(default_factory=list) + + +def installs(data: Any, model: type[Install] = Install) -> tuple[dict[str, Install], list[str]]: + """The installs ``data``, an ``install`` table, declares by OS type, each a ``model``, and the problems with it, + each named by its key.""" if not isinstance(data, dict): return {}, [f"install must be a table of OS types ({', '.join(OS_TYPES)}), not {data!r}"] found, problems = {}, [] @@ -57,8 +64,19 @@ def installs(data: Any) -> tuple[dict[str, Install], list[str]]: problems.append(f"install.{os_type}: an install is for an OS type agent-env composes: {', '.join(OS_TYPES)}") continue try: - found[os_type] = Install.model_validate(table) + found[os_type] = model.model_validate(table) except ValidationError as e: problems.extend(f"install.{'.'.join(map(str, (os_type, *error['loc'])))}: {error['msg']}" for error in e.errors()) return found, problems + + +def missing(os_type: str, install: Install, files: dict) -> list[str]: + """What ``install`` names that the layer's folder, ``files`` by path, doesn't hold.""" + folder = PurePosixPath(install.files) + problems = [] + if folder != PurePosixPath(".") and not any(PurePosixPath(path).is_relative_to(folder) for path in files): + problems.append(f"install.{os_type}.files: this folder has no {install.files!r} folder") + if install.python is not None and (folder / install.python).as_posix() not in files: + problems.append(f"install.{os_type}.python: {(folder / install.python).as_posix()!r} isn't in this folder") + return problems diff --git a/src/agent_env/env/envs/computer/params.py b/src/agent_env/env/envs/computer/utils/params.py similarity index 73% rename from src/agent_env/env/envs/computer/params.py rename to src/agent_env/env/envs/computer/utils/params.py index a49bd1f38..4edc8067d 100644 --- a/src/agent_env/env/envs/computer/params.py +++ b/src/agent_env/env/envs/computer/utils/params.py @@ -1,6 +1,7 @@ """What a deploy of a computer env takes as ``deploy_env`` ``env_params``: the form to boot, ``computer_settings``, -values of the base's deploy-phase settings, and ``driver_settings``, the driver's ``ComputerSettings``, such as the -default coordinate space its tools describe. The env's display and network are its own, which no deploy overrides.""" +values of the base's deploy-phase settings, ``driver_settings``, the driver's ``ComputerSettings``, such as the default +coordinate space its tools describe, and ``app_settings``, values of each app's ``Settings`` by its name. The env's +display and network are its own, which no deploy overrides.""" from __future__ import annotations @@ -10,7 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError from agent_env.artifact.artifacts.computer import SettingValue -from agent_env.env.envs.computer import gate +from agent_env.env.envs.computer.utils import gate if TYPE_CHECKING: from agent_env.env.envs.computer.env import ComputerEnv @@ -22,6 +23,7 @@ class ComputerParams(BaseModel): form: Optional[str] = None computer_settings: dict[str, Any] = Field(default_factory=dict) driver_settings: dict[str, Any] = Field(default_factory=dict) + app_settings: dict[str, dict[str, Any]] = Field(default_factory=dict) def checked(env: ComputerEnv, env_params: Optional[dict]) -> tuple[ComputerParams, dict[str, SettingValue], list[str]]: @@ -47,6 +49,11 @@ def checked(env: ComputerEnv, env_params: Optional[dict]) -> tuple[ComputerParam ComputerSettings.model_validate(params.driver_settings) except ValidationError as e: problems += _problems("env_params.driver_settings", e) + apps = {app.environment_name: app for app in env.composed_apps()} + problems += [f"env_params.app_settings.{name}: {env.id} has no app {name!r}" + + (f"; its apps are {', '.join(apps)}" if apps else "") for name in params.app_settings if name not in apps] + for name, app in apps.items(): + problems += gate.check_app_settings(app, params.app_settings.get(name, {}), f"env_params.app_settings.{name}") return params, values, problems diff --git a/src/agent_env/env/registry.py b/src/agent_env/env/registry.py index 50b85bfa1..966d3f5a2 100644 --- a/src/agent_env/env/registry.py +++ b/src/agent_env/env/registry.py @@ -22,7 +22,8 @@ def _get_type(cls: type["Env"]) -> str: def _builtin_registry() -> dict[str, type["Env"]]: """The built-in env types, by type.""" from agent_env.env.envs.computer.env import ComputerEnv - from agent_env.env.envs.computer_driver import ComputerDriverEnv + from agent_env.env.envs.computer.app_env import ComputerAppEnv + from agent_env.env.envs.computer.driver_env import ComputerDriverEnv from agent_env.env.envs.gateway_server import GatewayEnv from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv @@ -31,6 +32,7 @@ def _builtin_registry() -> dict[str, type["Env"]]: return { _get_type(ComputerEnv): ComputerEnv, + _get_type(ComputerAppEnv): ComputerAppEnv, _get_type(ComputerDriverEnv): ComputerDriverEnv, _get_type(GatewayEnv): GatewayEnv, _get_type(MCPServerEnv): MCPServerEnv, diff --git a/src/agent_env/examples/desktop/README.md b/src/agent_env/examples/desktop/README.md index f7ca52e3a..aa2d7eae5 100644 --- a/src/agent_env/examples/desktop/README.md +++ b/src/agent_env/examples/desktop/README.md @@ -8,8 +8,14 @@ time; a later run reuses the image. Deploying it boots that image in a container `tasks/probe.json` deploys it and runs the conformance probe at `full` against it, with `artifacts/check_probe/`. It needs Docker and the local registry on `localhost:5000`, and no model. -`tasks/agent.json` has Claude Code, given the computer's skill (`add_skills` `env_ids`), open the file -manager and make a folder named `Reports` in `Documents`, while the display is recorded; `artifacts/check_folder/` -checks the folder through the driver. It needs a model endpoint as well. +`tasks/files.json` reaches inside the computer through its driver, as its session user: `load_artifact` puts +`artifacts/figures/` in `~/Documents`, `run_code` runs `artifacts/read_figures/` in the computer, where it reads the +file and asks the driver for its card on loopback, and `run_container_unit_tests_verifier` checks the file with a +command run there. It needs no model either. + +`tasks/agent.json` loads the figures into `~/Downloads`, the default, then has Claude Code, given the computer's skill +(`add_skills` `env_ids`), make a folder named `Reports` in `Documents` with the file manager and move the file into +it, while the display is recorded; `artifacts/check_folder/` checks through the driver that it moved. It needs a model +endpoint as well. Run them with `agent-env run desktop`, or one with `agent-env run desktop --task probe`. diff --git a/src/agent_env/examples/desktop/artifacts/check_folder/verify.py b/src/agent_env/examples/desktop/artifacts/check_folder/verify.py index 31d7de83d..718e67352 100644 --- a/src/agent_env/examples/desktop/artifacts/check_folder/verify.py +++ b/src/agent_env/examples/desktop/artifacts/check_folder/verify.py @@ -1,15 +1,17 @@ -"""Whether Documents/Reports is a folder in the session user's home, asked through the driver's computer-exec.""" +"""Whether the figures moved from Downloads into a Reports folder in Documents, asked through the driver's +computer-exec.""" import uuid from agentenv_protocol import client as protocol EXEC = "urn:agentenv:computer-exec/v1" +MOVED = "test -f Documents/Reports/figures.csv && test ! -e Downloads/figures.csv" -async def verify(mcp_url): - base = mcp_url.rsplit("/mcp", 1)[0] - card = await protocol.get_card(base) - out = await protocol.invoke_extension(base, card, EXEC, {"argv": ["test", "-d", "Documents/Reports"], - "request_id": uuid.uuid4().hex}, method="run") - return [{"id": "folder", "result": out["exit_code"] == 0, - "description": "Documents/Reports is a folder" if out["exit_code"] == 0 else "no folder Documents/Reports"}] +async def verify(mcp_url, env_url): + card = await protocol.get_card(env_url) + out = await protocol.invoke_extension(env_url, card, EXEC, {"argv": ["sh", "-c", MOVED], + "request_id": uuid.uuid4().hex}, method="run") + moved = out["exit_code"] == 0 + return [{"id": "folder", "result": moved, "description": "the figures are in Documents/Reports" if moved else + "the figures aren't in Documents/Reports, or are still in Downloads"}] diff --git a/src/agent_env/examples/desktop/artifacts/check_probe/verify.py b/src/agent_env/examples/desktop/artifacts/check_probe/verify.py index bf1ac8a34..066d7cf08 100644 --- a/src/agent_env/examples/desktop/artifacts/check_probe/verify.py +++ b/src/agent_env/examples/desktop/artifacts/check_probe/verify.py @@ -3,7 +3,7 @@ from agentenv_protocol.computer.conformance import run_probe -async def verify(mcp_url): - report = await run_probe(mcp_url.rsplit("/mcp", 1)[0], "full") +async def verify(mcp_url, env_url): + report = await run_probe(env_url, "full") return [{"id": f"{check.check} {check.name}", "result": check.status in ("passed", "not_applicable"), "description": check.detail or check.status} for check in report.checks] diff --git a/src/agent_env/examples/desktop/artifacts/figures/figures.csv b/src/agent_env/examples/desktop/artifacts/figures/figures.csv new file mode 100644 index 000000000..0da140d75 --- /dev/null +++ b/src/agent_env/examples/desktop/artifacts/figures/figures.csv @@ -0,0 +1,4 @@ +quarter,revenue +Q1,120 +Q2,135 +Q3,150 diff --git a/src/agent_env/examples/desktop/artifacts/read_figures/run.py b/src/agent_env/examples/desktop/artifacts/read_figures/run.py new file mode 100644 index 000000000..249709680 --- /dev/null +++ b/src/agent_env/examples/desktop/artifacts/read_figures/run.py @@ -0,0 +1,12 @@ +"""What the computer holds where load_artifact put the figures, and the name its driver's card gives, asked on +loopback from inside the computer.""" +import json +import urllib.request + + +def run(input): + with open("Documents/figures.csv") as figures: + header = figures.readline().strip() + with urllib.request.urlopen(f"{input['env']['url']}/.well-known/agent-env.json", timeout=10) as response: + card = json.load(response) + return {"header": header, "card": card["name"]} diff --git a/src/agent_env/examples/desktop/tasks/agent.json b/src/agent_env/examples/desktop/tasks/agent.json index 09b1e89f1..ccdb502ad 100644 --- a/src/agent_env/examples/desktop/tasks/agent.json +++ b/src/agent_env/examples/desktop/tasks/agent.json @@ -1,10 +1,11 @@ [ {"id": "env", "type": "deploy_env", "env_id": "desktop"}, + {"id": "files", "type": "load_artifact", "env_id": "desktop", "artifact_id": "figures", "depends_on": ["env"]}, {"id": "agent", "type": "deploy_agent", "env_ids": ["desktop"]}, {"id": "skills", "type": "add_skills", "env_ids": ["desktop"]}, {"id": "record", "type": "start_computer_recording", "env_id": "desktop", "name": "agent", "depends_on": ["env"]}, - {"id": "task", "type": "prompt_agent", "depends_on": ["skills", "record"], - "prompt": "Open the file manager on this computer and make a new folder named Reports in the Documents folder. Use only the computer_* tools, and reply DONE once the folder is there."}, + {"id": "task", "type": "prompt_agent", "depends_on": ["files", "skills", "record"], + "prompt": "Make a new folder named Reports in the Documents folder on this computer, then move the CSV file you'll find in Downloads into it. Use only the computer_* tools, and reply DONE once the file is in the folder."}, {"id": "check", "type": "env_outcome_verifier", "env_id": "desktop", "file_artifact_id": "check_folder", "verifier_id": "folder", "depends_on": ["task"]}, {"id": "end", "type": "end_computer_recording", "name": "agent", "depends_on": ["task"]} diff --git a/src/agent_env/examples/desktop/tasks/files.json b/src/agent_env/examples/desktop/tasks/files.json new file mode 100644 index 000000000..7ef8b2c64 --- /dev/null +++ b/src/agent_env/examples/desktop/tasks/files.json @@ -0,0 +1,8 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desktop"}, + {"id": "load", "type": "load_artifact", "env_id": "desktop", "artifact_id": "figures", + "destination_path": "Documents"}, + {"id": "inside", "type": "run_code", "env_id": "desktop", "script_artifact_id": "read_figures"}, + {"id": "tests", "type": "run_container_unit_tests_verifier", "env_id": "desktop", "verifier_id": "figures", + "command": "grep -qx 'Q3,150' Documents/figures.csv"} +] diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index ba5fbdad9..ce0794fc3 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -13,8 +13,16 @@ from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.attribution import Attribution from agent_env.env.env import DeployedSandboxEnv, EnvCapabilityUnsupported -from agent_env.env.envs.computer import params as computer_params -from agent_env.env.envs.computer.boot import READY_TIMEOUT_S, boot, discard, ready, recorded_settings, settings_environment +from agent_env.env.envs.computer.utils import params as computer_params +from agent_env.env.envs.computer.utils.boot import ( + READY_TIMEOUT_S, + app_settings_environment, + boot, + discard, + ready, + recorded_settings, + settings_environment, +) from agent_env.env.gateway.constants import WELL_KNOWN_PATH from agent_env.providers.env_providers.env_provider import DeployRefused, _SandboxEnvironmentProvider from agent_env.providers.sandbox_providers.sandbox import NetworkPolicy @@ -55,6 +63,8 @@ async def deploy( if not isinstance(env, ComputerEnv): raise TypeError(f"env_provider_type '{self.type}' deploys a computer env, not a {env.type} env") + if env.computer is None: # an app that names no computer to deploy alone on + raise DeployRefused(env.deploy_refusal()) params, values, problems = computer_params.checked(env, env_params) if problems: raise DeployRefused(f"{env.id}: " + "; ".join(problems)) @@ -100,7 +110,7 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par metadata={"computer": { "form": form, "arch": arch, "provider": type(sandbox_provider).__name__, "image": build["image"], "computer": {"id": env.metadata["computer"]["id"], "version": env.metadata["computer"]["version"]}, - "display": values.get("resolution"), "network": env.network, + "port": env.driver.port, "display": values.get("resolution"), "network": env.network, "computer_settings": recorded_settings(computer, values), "env_params": params.model_dump(exclude_defaults=True)}}, ) @@ -149,8 +159,9 @@ def _values(computer: ComputerArtifact, values: dict) -> dict: def _environment(env: ComputerEnv, computer: ComputerArtifact, values: dict, params: computer_params.ComputerParams) -> dict[str, str]: - return settings_environment(computer, values, environment_name=env.environment_name, - driver_settings=params.driver_settings) + return {**settings_environment(computer, values, environment_name=env.environment_name, + driver_settings=params.driver_settings), + **app_settings_environment(env.composed_apps(), params.app_settings)} def _require_name(env: ComputerEnv, card: dict) -> None: diff --git a/src/agent_env/task_step/task_steps/load_artifact.py b/src/agent_env/task_step/task_steps/load_artifact.py index cba850321..b45f135bf 100644 --- a/src/agent_env/task_step/task_steps/load_artifact.py +++ b/src/agent_env/task_step/task_steps/load_artifact.py @@ -13,12 +13,15 @@ from typing import Callable, ClassVar, Optional from urllib.parse import unquote, urlparse +from agent_env.artifact import Artifact, EnvironmentArtifact, EnvironmentUniverseArtifact +from agent_env.env.env import Env from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.providers.sandbox_providers.sandbox_provider import reachable_url from agent_env.task_step.context import DeployedAgent, TaskStepContext from agent_env.entity.refs import EntityRef from agent_env.entity.scopes import FieldScope +from agent_env.store.base import NotFoundError from agent_env.task_step.task_step import TaskStep, TaskStepDependency from agent_env.utils.paths import validate_relative_filename @@ -121,6 +124,11 @@ def _url_downloads(step_id: str, urls: list[UrlEntry]) -> list[tuple[str, str]]: return downloads +# The method an env type defines when it loads each kind of environment artifact. +_ENVIRONMENT_LOADERS = {EnvironmentArtifact: "load_environment_artifact", + EnvironmentUniverseArtifact: "load_environment_universe_artifact"} + + @dataclass class _LoadInputs: """What a load_artifact step loads and where, after per-run overrides.""" @@ -213,6 +221,24 @@ def config_problems(self) -> list[str]: """A step naming nothing to load.""" return [] if self._names_a_source() else [self._nothing_to_load()] + def preflight(self) -> list[str]: + """An environment artifact this step names for an env whose type loads none.""" + if self.env_id is None: + return [] + try: + env = Env.get(self.env_id) + artifacts = [Artifact.get(ref["id"], ref.get("version")) for ref in self.artifacts] + except NotFoundError: # what the run reports + return [] + return [refusal for artifact in artifacts if (refusal := self._unloadable(env, self.env_id, artifact))] + + def _unloadable(self, env: Env, env_id: str, artifact: Artifact) -> Optional[str]: + """Why ``env`` can't load ``artifact``, an environment artifact its type has no loader for; None when it can.""" + loader = next((loader for kind, loader in _ENVIRONMENT_LOADERS.items() if isinstance(artifact, kind)), None) + if loader is None or hasattr(env, loader): + return None + return f"load_artifact '{self.id}': {env.type} env '{env_id}' loads no {artifact.type} artifacts, so not '{artifact.id}'" + def _names_a_source(self) -> bool: return bool(self.artifacts or self.urls or self.collected_artifacts_step_id or self.artifact_from_step_id) @@ -589,6 +615,8 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if env is None: env_obj = Env.get(deployed.env_id, deployed.env_version) env = await type(env_obj).from_deployed_env(deployed) + if refusal := self._unloadable(env, deployed.env_id, artifact): + raise RuntimeError(refusal) if isinstance(artifact, EnvironmentArtifact): await env.load_environment_artifact(artifact) else: diff --git a/src/agent_env/task_step/task_steps/run_code.py b/src/agent_env/task_step/task_steps/run_code.py index bc8258f1a..98b773526 100644 --- a/src/agent_env/task_step/task_steps/run_code.py +++ b/src/agent_env/task_step/task_steps/run_code.py @@ -2,9 +2,10 @@ The script defines ``run(input) -> JSON``, where ``input = {"args", "results"}``. The result is stored at ``context.metadata["script_results"][result_id]``. -A script on an env's host also gets ``input["env"] = {"id", "url", "mcp_url"}``: the env's -address, where its card is at ``/.well-known/agent-env.json``, and its MCP endpoint, as that -host reaches them (``Sandbox.url_on_host``), since a host can't always call its own public URL. +A script run for an env also gets ``input["env"] = {"id", "url", "mcp_url"}``: the env's +address, where its card is at ``/.well-known/agent-env.json``, and its MCP endpoint, as the script +reaches them: on the env's host as that host does (``Sandbox.url_on_host``), since a host can't +always call its own public URL; inside a computer, the driver on loopback. ``script_artifact_id`` is either a single-file ``FileArtifact`` or a ``FileArtifactUniverse`` whose ``.py`` members are staged side by side so the entry @@ -12,7 +13,10 @@ the entry, defaulting to the sole ``.py`` member or to ``run.py``. Target: with ``env_id`` set, the script runs on that env's VM host, or in its container for an -env in a container sandbox, whose image needs python3 (no agent, no LLM key); with +env in a container sandbox, whose image needs python3 (no agent, no LLM key); for an env whose card +serves computer-exec/v1 (a computer), inside the computer as its session user, through the driver: +staged under ``~/.agentenv/run_code//`` with ``data/add``, run with ``run``, its result read +with ``data/get``, and the folder then removed, since the agent can read the home; with ``sandbox_name`` set, on a ``deploy_sandbox`` sandbox (its VM host, or its container for a container-mode sandbox); otherwise it runs inside the agent container (``agent_name``), where it can see the agent's workspace. @@ -24,14 +28,18 @@ import json import logging import os +import re import shlex -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path, PurePosixPath from typing import Any, ClassVar, Optional, Protocol +from agentenv_protocol.computer import COMPUTER_EXEC + from agent_env.artifact import Artifact, FileArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.env.env import DeployedSandboxEnv, require_sandbox +from agent_env.env.envs.computer.utils import inside from agent_env.providers.sandbox_providers.sandbox import into_container from agent_env.task_step.context import TaskStepContext from agent_env.entity.refs import EntityRef @@ -41,6 +49,12 @@ _RUN_DIR = "/tmp/agent_env_run_code" +# Inside a computer, relative to the session user's home. +_COMPUTER_RUN_DIR = ".agentenv/run_code" +# How much longer than the script's timeout a computer lets the runner run, so the runner stops the script first. +_RUNNER_SLACK_S = 10 +# What a POSIX extended regular expression, as `pkill -f` reads one, gives a meaning to outside brackets. +_ERE_SPECIAL = re.compile(r"[.\[\\()*+?{|^$]") # The bootstrap we run in the sandbox: `python3 runner.py `. _RUNNER_SOURCE = (Path(__file__).parent / "run_code_runner.py").read_text(encoding="utf-8") @@ -170,9 +184,13 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: target = await self._resolve_target(context) run_dir = target.run_dir(self.id) - await self._stage(target, run_dir, files, self._build_input(context, target)) - await self._run(target, run_dir, script_file) - result = await self._read_output(target, run_dir) + try: + await self._stage(target, run_dir, files, self._build_input(context, target)) + await self._run(target, run_dir, script_file) + result = await self._read_output(target, run_dir) + finally: + if isinstance(target, _DriverTarget): + await self._remove(target, run_dir) context.metadata.setdefault("script_results", {})[self.result_id] = result logger.info( @@ -300,7 +318,7 @@ def _build_input(self, context: TaskStepContext, target: _ExecTarget) -> dict: "results": context.metadata.get("script_results", {}), } if self.env_id is not None: - script_input["env"] = _env_on_host(self._deployed_env(context), target.sandbox) + script_input["env"] = target.endpoints(self._deployed_env(context)) return script_input def _target_kind(self) -> str: @@ -360,14 +378,22 @@ async def _read_output(self, target: _ExecTarget, run_dir: str) -> Any: raise self._validation_error( f"result exceeds {self.MAX_OUTPUT_BYTES}-byte cap" ) - exit_code, stdout, stderr = await target.run("cat", path) - if exit_code != 0: - raise self._exec_error(f"could not read result file: {self._stderr_tail(stderr)}") try: - return json.loads(stdout) + content = await target.read_file(path) + except OSError as exc: + raise self._exec_error(f"could not read result file: {self._stderr_tail(str(exc))}") from exc + try: + return json.loads(content) except json.JSONDecodeError as exc: raise self._validation_error(f"result is not valid JSON: {exc}") from exc + async def _remove(self, target: _DriverTarget, run_dir: str) -> None: + """Remove a computer's run folder; a failure is logged, not raised over the run's own.""" + try: + await target.remove_dir(run_dir) + except Exception as e: + logger.warning("run_code '%s': couldn't remove %s: %s", self.id, run_dir, e) + # --- run targets --- def _deployed_env(self, context: TaskStepContext) -> DeployedSandboxEnv: @@ -392,6 +418,8 @@ async def _host_target(self, context: TaskStepContext) -> _ExecTarget: ) deployed = self._deployed_env(context) + if deployed.supports(COMPUTER_EXEC, "run"): + return self._driver_target(deployed) # env sandbox provider, honoring a custom sandbox_type provider = ( build_sandbox_provider(deployed.sandbox_type) @@ -402,6 +430,19 @@ async def _host_target(self, context: TaskStepContext) -> _ExecTarget: # An env in a container sandbox has no host to run on: the script runs in the container, beside its server. return _HostTarget(sandbox) if isinstance(sandbox, VmSandbox) else _AgentTarget(sandbox, prefix=()) + def _driver_target(self, deployed: DeployedSandboxEnv) -> _DriverTarget: + """Inside the computer, through its driver, once its card serves what the run needs and allows its timeout.""" + if missing := inside.missing(deployed): + raise self._validation_error(f"env '{self.env_id}' serves computer-exec/v1 but not {', '.join(missing)}, " + "which running code in it needs") + limit = inside.max_timeout_s(deployed) + if self.timeout_seconds > limit - _RUNNER_SLACK_S: + raise self._validation_error(f"timeout_seconds {self.timeout_seconds} is above the " + f"{limit - _RUNNER_SLACK_S} s a script in env '{self.env_id}' may run: it " + f"runs a command for at most {limit} s, and the runner keeps " + f"{_RUNNER_SLACK_S} s of that to stop the script") + return _DriverTarget(deployed, self.timeout_seconds + _RUNNER_SLACK_S) + async def _sandbox_target(self, context: TaskStepContext) -> _ExecTarget: from agent_env.providers.sandbox_providers.sandbox import VmSandbox from agent_env.providers.sandbox_providers.sandbox_provider import ( @@ -504,12 +545,19 @@ def _env_on_host(deployed: DeployedSandboxEnv, sandbox: Any) -> dict: } -class _ExecTarget(Protocol): - sandbox: Any +async def _cat(target: _ExecTarget, path: str) -> str: + exit_code, stdout, stderr = await target.run("cat", path) + if exit_code != 0: + raise OSError(stderr) + return stdout + +class _ExecTarget(Protocol): def run_dir(self, step_id: str) -> str: ... + def endpoints(self, deployed: DeployedSandboxEnv) -> dict: ... async def reset_dir(self, path: str) -> None: ... async def write_file(self, path: str, content: str) -> None: ... + async def read_file(self, path: str) -> str: ... async def run(self, *command: str) -> tuple[int, str, str]: ... @@ -523,6 +571,9 @@ def run_dir(self, step_id: str) -> str: # The sandbox's own: runs on the local provider share this machine's /tmp, and teardown removes it there. return f"{self.sandbox.scoped_name(_RUN_DIR)}/{step_id}" + def endpoints(self, deployed: DeployedSandboxEnv) -> dict: + return _env_on_host(deployed, self.sandbox) + async def reset_dir(self, path: str) -> None: await self.sandbox.exec_script( f"rm -rf {shlex.quote(path)} && mkdir -p {shlex.quote(path)}" @@ -531,6 +582,9 @@ async def reset_dir(self, path: str) -> None: async def write_file(self, path: str, content: str) -> None: await self.sandbox.write_host_file(content.encode("utf-8"), path) + async def read_file(self, path: str) -> str: + return await _cat(self, path) + async def run(self, *command: str) -> tuple[int, str, str]: # Files are written as root, so run as root too. return await self.sandbox.exec_with_output("sudo", *command) @@ -548,6 +602,9 @@ class _AgentTarget: def run_dir(self, step_id: str) -> str: return f"{_RUN_DIR}/{step_id}" + def endpoints(self, deployed: DeployedSandboxEnv) -> dict: + return _env_on_host(deployed, self.sandbox) + async def reset_dir(self, path: str) -> None: await self.run("rm", "-rf", path) await self.run("mkdir", "-p", path) @@ -556,5 +613,48 @@ async def write_file(self, path: str, content: str) -> None: # write_file_from_text takes (content, path); we expose (path, content). await self.sandbox.write_file_from_text(content, path, **into_container(self.container)) + async def read_file(self, path: str) -> str: + return await _cat(self, path) + async def run(self, *command: str) -> tuple[int, str, str]: return await self.sandbox.exec_with_output(*self.prefix, *command) + + +@dataclass +class _DriverTarget: + """Runs the script inside a computer, as its session user, through its driver: files written with ``data/add`` and + read with ``data/get``, commands run with computer-exec/v1 ``run`` for at most ``timeout_s``, with ``env`` added + to the driver's environment. Paths are relative to the home, where commands run.""" + + deployed: DeployedSandboxEnv + timeout_s: float + env: dict[str, str] = field(default_factory=dict) + + def run_dir(self, step_id: str) -> str: + return f"{_COMPUTER_RUN_DIR}/{step_id}" + + def endpoints(self, deployed: DeployedSandboxEnv) -> dict: + return inside.loopback(deployed) + + async def reset_dir(self, path: str) -> None: + """End whatever still runs from the folder, as a run whose driver died leaves going, then remove it.""" + await self.run("pkill", "-f", _ERE_SPECIAL.sub(r"\\\g<0>", f"{path}/runner.py")) + await self.remove_dir(path) + + async def remove_dir(self, path: str) -> None: + exit_code, _, stderr = await self.run("rm", "-rf", path) + if exit_code != 0: + raise OSError(stderr) + + async def write_file(self, path: str, content: str) -> None: + await inside.add_files(self.deployed, {path: content.encode("utf-8")}) + + async def read_file(self, path: str) -> str: + return (await inside.read_file(self.deployed, path)).decode("utf-8") + + async def run(self, *command: str) -> tuple[int, str, str]: + """``command``'s exit status, stdout and stderr, each cut at the driver's output cap; a command stopped at the + timeout exits as the runner's does.""" + answer = await inside.run(self.deployed, list(command), timeout_s=self.timeout_s, env=self.env) + exit_code = RunCodeTaskStep._TIMEOUT_EXIT_CODE if answer["timed_out"] else answer["exit_code"] + return exit_code, answer["stdout"], answer["stderr"] diff --git a/src/agent_env/task_step/task_steps/run_code_runner.py b/src/agent_env/task_step/task_steps/run_code_runner.py index cf54bdb9e..e78fe7294 100644 --- a/src/agent_env/task_step/task_steps/run_code_runner.py +++ b/src/agent_env/task_step/task_steps/run_code_runner.py @@ -94,7 +94,8 @@ def forward(signum, _frame): import json import os -run_dir, entrypoint, script_file = sys.argv[1], sys.argv[2], sys.argv[3] +# Absolute, as a computer's is not: the script may change directory before the runner writes its result. +run_dir, entrypoint, script_file = os.path.abspath(sys.argv[1]), sys.argv[2], sys.argv[3] script_path = os.path.join(run_dir, script_file) # The entry's own directory first: a nested entry (`src/run.py`) imports siblings by diff --git a/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py b/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py index 3a0c9649c..1e8c89d1b 100644 --- a/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/env_outcome_verifier.py @@ -24,7 +24,8 @@ class EnvOutcomeVerifierTaskStep(TaskStep): - """Runs the artifact's ``verify(mcp_url)``; a ``run`` parameter also gets the ``Run`` of ``prompt_id``.""" + """Runs the artifact's ``verify(mcp_url)``; a ``run`` parameter also gets the ``Run`` of ``prompt_id``, and an + ``env_url`` parameter the env's address, which the protocol client takes as it is.""" type: ClassVar[str] = "env_outcome_verifier" entity_refs = ( @@ -119,11 +120,11 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: spec.loader.exec_module(module) logger.info(f"Running verify() from artifact '{self.file_artifact_id}' against {deployed.mcp_url}") - if "run" in inspect.signature(module.verify).parameters: - run = await load_run_context(context, self._graded_prompt(context)) - results = await module.verify(deployed.mcp_url, run=run) - else: - results = await module.verify(deployed.mcp_url) + parameters = inspect.signature(module.verify).parameters + keywords = {"env_url": deployed.environment_url} if "env_url" in parameters else {} + if "run" in parameters: + keywords["run"] = await load_run_context(context, self._graded_prompt(context)) + results = await module.verify(deployed.mcp_url, **keywords) score = aggregate_score(results, self.score_aggregator) if "verifications" not in context.metadata: diff --git a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py index f289bba51..5e3580e9d 100644 --- a/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/run_container_unit_tests_verifier.py @@ -1,9 +1,13 @@ -"""Run a unit-tests-style verifier inside a deployed task container. +"""Run a unit-tests-style verifier inside a deployed task container, or inside a computer env. Optionally runs setup_commands first (to install verifier dependencies the agent's container didn't have). The main command's exit code, stdout/stderr, and any extracted result files land on context.metadata["verifications"]. +With ``env_id``, the commands run inside that env's computer as its session user, through its +driver's computer-exec/v1 ``run``, with ``env_vars`` in their environment; ``result_paths`` and +``reward_path`` are relative to the home and read with ``data/get``. + stdout and stderr are each persisted as their own FileArtifact in S3 (so the verification entry stays small and the full output is addressable via the artifact API). Result files declared via `result_paths` are docker-cp'd out of @@ -26,14 +30,20 @@ import json import logging import os +import posixpath import re import shlex import tempfile import time import uuid +from collections.abc import Awaitable, Callable from typing import Any, ClassVar, Optional +from agent_env.config import ConfigError, get_config from agent_env.config.model import MODEL_HEADERS_ENV +from agent_env.entity.refs import EntityRef +from agent_env.env.env import require_sandbox +from agent_env.env.envs.computer.utils import inside from agent_env.providers.sandbox_providers.local_docker.provider import host_url_for from agent_env.providers.sandbox_providers.sandbox_provider import ( all_sandbox_container_env, @@ -42,6 +52,7 @@ from agent_env.store.ids import derive_id, is_local_id, key_segment, validate_local_id from agent_env.task_step.context import TaskStepContext, dual_keyed from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.run_code import _DriverTarget logger = logging.getLogger(__name__) @@ -50,6 +61,8 @@ # convenience for `agent-env task run` / hub UI to show a head without an S3 # round-trip. _OUTPUT_PREVIEW_CHARS = 500 +# How much of a failed command's stderr an error quotes, from its end. +_ERROR_TAIL_CHARS = 1500 # Every env var key — stored, override or seed — reaches the `docker exec` string # unquoted (`-e 'K=v'` and `-e K=v` differ to docker), so it must be a POSIX name @@ -74,18 +87,20 @@ class RunContainerUnitTestsVerifierTaskStep(TaskStep): type: ClassVar[str] = "run_container_unit_tests_verifier" - entity_refs = () + entity_refs = (EntityRef.env("env_id"),) def __init__( self, id: str, version: Optional[int], - sandbox_name: str, - container_name: str, + sandbox_name: Optional[str] = None, + container_name: Optional[str] = None, + *, command: str, + env_id: Optional[str] = None, verifier_id: Optional[str] = None, setup_commands: Optional[list[str]] = None, - user: str = "root", + user: Optional[str] = None, timeout_sec: int = 300, env_vars: Optional[dict[str, str]] = None, result_paths: Optional[list[str]] = None, @@ -94,8 +109,16 @@ def __init__( fail_task_on_error: bool = True, ): super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + container = (sandbox_name, container_name) + if (env_id is None and None in container) or (env_id is not None and container != (None, None)): + raise ValueError(f"run_container_unit_tests_verifier '{id}': set env_id, or sandbox_name and " + "container_name") + if env_id is not None and user is not None: + raise ValueError(f"run_container_unit_tests_verifier '{id}': commands in env '{env_id}' run as its " + "computer's session user; leave user unset") self.sandbox_name = sandbox_name self.container_name = container_name + self.env_id = env_id self.command = command self.verifier_id = verifier_id or id self.setup_commands = list(setup_commands or []) @@ -115,15 +138,20 @@ def __init__( self.reward_path = reward_path if reward_path and reward_path not in self.result_paths: self.result_paths.append(reward_path) + if env_id is not None and (absolute := [path for path in self.result_paths if posixpath.isabs(path)]): + raise ValueError(f"run_container_unit_tests_verifier '{id}': result paths in env '{env_id}' are relative " + f"to its session user's home, not {absolute}") def to_dict(self) -> dict: base = super().to_dict() base["sandbox_name"] = self.sandbox_name base["container_name"] = self.container_name + base["env_id"] = self.env_id base["command"] = self.command base["verifier_id"] = self.verifier_id base["setup_commands"] = self.setup_commands - base["user"] = self.user + if self.user is not None: # a release before env_id targets reads a missing user as root, and null as None + base["user"] = self.user base["timeout_sec"] = self.timeout_sec base["env_vars"] = self.env_vars base["result_paths"] = self.result_paths @@ -134,12 +162,13 @@ def to_dict(self) -> dict: def from_dict(cls, data: dict) -> RunContainerUnitTestsVerifierTaskStep: return cls( **cls._base_from_dict(data), - sandbox_name=data["sandbox_name"], - container_name=data["container_name"], + sandbox_name=data.get("sandbox_name"), + container_name=data.get("container_name"), command=data["command"], + env_id=data.get("env_id"), verifier_id=data.get("verifier_id"), setup_commands=data.get("setup_commands"), - user=data.get("user", "root"), + user=data.get("user"), timeout_sec=data.get("timeout_sec", 300), env_vars=data.get("env_vars"), result_paths=data.get("result_paths"), @@ -203,101 +232,20 @@ def _resolve_command(self, context: TaskStepContext) -> tuple[str, dict[str, str return command, extra_env async def execute(self, context: TaskStepContext) -> TaskStepContext: - from agent_env.providers.sandbox_providers.sandbox_provider import ( - build_sandbox_provider, - get_sandbox_provider, - ) - from agent_env.config import ConfigError, get_config - # The outputs are named after the run: refuse an id they can't take before the tests run. if context.instance_id and is_local_id(context.instance_id): validate_local_id(derive_id(context.instance_id, f"verifier-stdout-{self.id}")) - # 1. Lookup container + sandbox, rehydrate VmSandbox - containers = context.metadata.get("deployed_docker_containers", []) - container_entry = next( - ( - c for c in containers - if c.get("container_name") == self.container_name - and c.get("sandbox_name") == self.sandbox_name - ), - None, - ) - if container_entry is None: - raise RuntimeError( - f"Container '{self.container_name}' not found on sandbox '{self.sandbox_name}' in " - f"context.metadata['deployed_docker_containers']" - ) - ds = next((s for s in context.deployed_sandboxes if s.sandbox_name == self.sandbox_name), None) - if ds is None: - raise RuntimeError(f"Sandbox '{self.sandbox_name}' not found in context.deployed_sandboxes") - - provider = ( - build_sandbox_provider(ds.sandbox_type) if ds.sandbox_type else get_sandbox_provider() - ) - sandbox = await provider.get_sandbox(ds.sandbox_id) - - # 2. Build the merged env (auto-injected from agent-env config + user overlay) config = get_config() - merged_env: dict[str, str] = {} - try: - litellm_key = config.get_litellm_api_key() - merged_env["LITELLM_API_KEY"] = litellm_key - merged_env["ANTHROPIC_API_KEY"] = litellm_key - except (KeyError, ConfigError): - logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds") - try: - litellm_url = config.get_litellm_base_url() - merged_env["LITELLM_BASE_URL"] = litellm_url - merged_env["ANTHROPIC_BASE_URL"] = litellm_url - except (KeyError, ConfigError): - logger.warning("LITELLM_BASE_URL unavailable; verifier won't have LLM base URL") - merged_env.update(all_sandbox_container_env()) command, extra_env = self._resolve_command(context) - merged_env.update(self.env_vars) - merged_env.update(extra_env) - # Unlike a missing key or URL, configured headers that don't resolve fail the step: a verifier that ran - # without them would grade against an endpoint that answers differently, or bills someone else. - if MODEL_HEADERS_ENV not in merged_env: - if headers := config.get_model_headers(merged_env.get("LITELLM_BASE_URL")): - merged_env[MODEL_HEADERS_ENV] = json.dumps(headers) - for key in _MODEL_URL_KEYS & merged_env.keys(): - merged_env[key] = host_url_for(merged_env[key], sandbox.type) - env_flags = " ".join(f"-e {k}={shlex.quote(v)}" for k, v in merged_env.items()) - - # 3. Run setup_commands (fail loud) - for i, setup_cmd in enumerate(self.setup_commands, 1): - logger.info( - f"[{self.verifier_id}] setup [{i}/{len(self.setup_commands)}]: " - f"{setup_cmd.splitlines()[0][:140]}" - ) - await sandbox.exec_script( - f"docker exec -u {shlex.quote(self.user)} {env_flags} " - f"{shlex.quote(sandbox.scoped_name(self.container_name))} bash -c {shlex.quote(setup_cmd)}" - ) - - # 4. Clear the reward file the score is read from, so one the agent left in this container - # isn't graded; a process the agent left running can still write it. Other result files - # stay: a step can extract what an earlier step's job wrote. `-w /` resolves a relative - # path as `docker cp` does, and uid 0 needs no passwd entry. - if self.reward_path: - await sandbox.exec_script( - f"docker exec -u 0 -w / " - f"{shlex.quote(sandbox.scoped_name(self.container_name))} " - f"rm -f -- {shlex.quote(self.reward_path)}" - ) - - # 5. Run main command, bounded by in-container timeout(1) - logger.info( - f"[{self.verifier_id}] running verifier (timeout={self.timeout_sec}s) " - f"in container '{self.container_name}': {command[:160]}" - ) - wrapped = ( - f"docker exec -u {shlex.quote(self.user)} {env_flags} " - f"{shlex.quote(sandbox.scoped_name(self.container_name))} " - f"timeout --kill-after=10 {self.timeout_sec} bash -c {shlex.quote(command)}" - ) - exit_code, stdout, stderr = await sandbox.exec_with_output("sudo", "bash", "-c", wrapped) + if self.env_id is not None: + target = self._computer(context) + target.env = self._merged_env(config, target.deployed.sandbox_type, extra_env) + exit_code, stdout, stderr, extracted_files = await self._run_in_computer(target, command) + else: + sandbox = await self._container_sandbox(context) + merged_env = self._merged_env(config, sandbox.type, extra_env) + exit_code, stdout, stderr, extracted_files = await self._run_in_container(sandbox, merged_env, command) timed_out = exit_code in (124, 137) logger.info( f"[{self.verifier_id}] exit_code={exit_code} timed_out={timed_out} " @@ -308,20 +256,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if stderr: logger.info(f"[{self.verifier_id}] stderr:\n{stderr}") - # 6. Extract result files (docker cp + cat + try JSON parse) - extracted_files: dict[str, Any] = {} - for result_path in self.result_paths: - try: - content = await self._extract_file(sandbox, result_path) - try: - extracted_files[result_path] = json.loads(content) - except json.JSONDecodeError: - extracted_files[result_path] = content - except Exception as e: - logger.warning(f"[{self.verifier_id}] failed to extract {result_path}: {e}") - extracted_files[result_path] = None - - # 7. Persist stdout + stderr as separate FileArtifacts + # Persist stdout + stderr as separate FileArtifacts # Every execution gets its own key: a retried step keeps its run's instance id, and the # outputs are write-once. The instance id and the time only group and order them. stamp = f"{int(time.time())}-{uuid.uuid4().hex[:12]}" @@ -344,7 +279,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: object_url=store.object_url(f"{outputs}/stderr.txt"), ) - # 8. Record on context (shape compatible with aggregate_verifiers). + # Record on context (shape compatible with aggregate_verifiers). # Harbor-style verifiers (reward_path set) grade by the reward file, not # the exit code: test.sh exits 0 and encodes pass/fail in the reward. passed_surrogate = exit_code == 0 and not timed_out @@ -395,7 +330,7 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: "extracted_files": extracted_files, } - # 9. Raise on timeout / non-zero exit AFTER recording. In reward mode a + # Raise on timeout / non-zero exit AFTER recording. In reward mode a # non-zero exit is graded by the reward file this run wrote (score 0 when # it wrote none), not a step error — only an unfinished (timed-out) run is # unreliable. @@ -406,11 +341,154 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: if exit_code != 0 and self.reward_path is None: raise RuntimeError( f"Verifier '{self.verifier_id}' exited {exit_code}; " - f"stderr (last 1500 chars):\n{stderr[-1500:]}" + f"stderr (last {_ERROR_TAIL_CHARS} chars):\n{stderr[-_ERROR_TAIL_CHARS:]}" ) return context + def _merged_env(self, config, sandbox_type: Optional[str], extra_env: dict[str, str]) -> dict[str, str]: + """The commands' environment: the model endpoint's key and URL from agent-env's config, every sandbox + provider's container env, ``env_vars`` and this run's ``extra_env``, then the endpoint's headers, its URL as + ``sandbox_type``'s containers reach it.""" + merged_env: dict[str, str] = {} + try: + litellm_key = config.get_litellm_api_key() + merged_env["LITELLM_API_KEY"] = litellm_key + merged_env["ANTHROPIC_API_KEY"] = litellm_key + except (KeyError, ConfigError): + logger.warning("LITELLM_API_KEY unavailable; verifier won't have LLM creds") + try: + litellm_url = config.get_litellm_base_url() + merged_env["LITELLM_BASE_URL"] = litellm_url + merged_env["ANTHROPIC_BASE_URL"] = litellm_url + except (KeyError, ConfigError): + logger.warning("LITELLM_BASE_URL unavailable; verifier won't have LLM base URL") + merged_env.update(all_sandbox_container_env()) + merged_env.update(self.env_vars) + merged_env.update(extra_env) + # Unlike a missing key or URL, configured headers that don't resolve fail the step: a verifier that ran + # without them would grade against an endpoint that answers differently, or bills someone else. + if MODEL_HEADERS_ENV not in merged_env: + if headers := config.get_model_headers(merged_env.get("LITELLM_BASE_URL")): + merged_env[MODEL_HEADERS_ENV] = json.dumps(headers) + for key in _MODEL_URL_KEYS & merged_env.keys(): + merged_env[key] = host_url_for(merged_env[key], sandbox_type) + return merged_env + + async def _container_sandbox(self, context: TaskStepContext): + """The sandbox that hosts the container, once the container is found in the run.""" + from agent_env.providers.sandbox_providers.sandbox_provider import ( + build_sandbox_provider, + get_sandbox_provider, + ) + + containers = context.metadata.get("deployed_docker_containers", []) + container_entry = next( + ( + c for c in containers + if c.get("container_name") == self.container_name + and c.get("sandbox_name") == self.sandbox_name + ), + None, + ) + if container_entry is None: + raise RuntimeError( + f"Container '{self.container_name}' not found on sandbox '{self.sandbox_name}' in " + f"context.metadata['deployed_docker_containers']" + ) + ds = next((s for s in context.deployed_sandboxes if s.sandbox_name == self.sandbox_name), None) + if ds is None: + raise RuntimeError(f"Sandbox '{self.sandbox_name}' not found in context.deployed_sandboxes") + + provider = ( + build_sandbox_provider(ds.sandbox_type) if ds.sandbox_type else get_sandbox_provider() + ) + return await provider.get_sandbox(ds.sandbox_id) + + async def _run_in_container(self, sandbox, merged_env: dict[str, str], + command: str) -> tuple[int, str, str, dict[str, Any]]: + """Run the setup commands and the command in the container; the command's exit code, stdout and stderr, and + the result files.""" + env_flags = " ".join(f"-e {k}={shlex.quote(v)}" for k, v in merged_env.items()) + user = shlex.quote(self.user or "root") + container = shlex.quote(sandbox.scoped_name(self.container_name)) + + # Setup commands fail loud. + for i, setup_cmd in enumerate(self.setup_commands, 1): + self._log_setup(i, setup_cmd) + await sandbox.exec_script(f"docker exec -u {user} {env_flags} {container} bash -c {shlex.quote(setup_cmd)}") + + # Clear the reward file the score is read from, so one the agent left in this container + # isn't graded; a process the agent left running can still write it. Other result files + # stay: a step can extract what an earlier step's job wrote. `-w /` resolves a relative + # path as `docker cp` does, and uid 0 needs no passwd entry. + if self.reward_path: + await sandbox.exec_script(f"docker exec -u 0 -w / {container} rm -f -- {shlex.quote(self.reward_path)}") + + # The command, bounded by in-container timeout(1). + logger.info( + f"[{self.verifier_id}] running verifier (timeout={self.timeout_sec}s) " + f"in container '{self.container_name}': {command[:160]}" + ) + wrapped = ( + f"docker exec -u {user} {env_flags} {container} " + f"timeout --kill-after=10 {self.timeout_sec} bash -c {shlex.quote(command)}" + ) + exit_code, stdout, stderr = await sandbox.exec_with_output("sudo", "bash", "-c", wrapped) + return exit_code, stdout, stderr, await self._extracted(lambda path: self._extract_file(sandbox, path)) + + def _computer(self, context: TaskStepContext) -> _DriverTarget: + """``env_id``'s computer, through its driver, once its card serves what the commands need and allows their + timeout.""" + deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) + if deployed is None: + raise RuntimeError(f"Env '{self.env_id}' not found in context.deployed_envs") + deployed = require_sandbox(deployed, "Running a verifier in an env") + if missing := inside.missing(deployed): + raise RuntimeError(f"env '{self.env_id}' doesn't serve {', '.join(missing)}, which running a verifier in " + "it needs") + if self.timeout_sec > (limit := inside.max_timeout_s(deployed)): + raise RuntimeError(f"timeout_sec {self.timeout_sec} is above the {limit} s env '{self.env_id}' lets a " + "command run") + return _DriverTarget(deployed, self.timeout_sec) + + async def _run_in_computer(self, target: _DriverTarget, command: str) -> tuple[int, str, str, dict[str, Any]]: + """Run the setup commands and the command in the computer as its session user; the command's exit code, stdout + and stderr, each cut at the driver's output cap, and the result files.""" + for i, setup_cmd in enumerate(self.setup_commands, 1): + self._log_setup(i, setup_cmd) + exit_code, _, stderr = await target.run("bash", "-c", setup_cmd) + if exit_code != 0: + raise RuntimeError(f"Verifier '{self.verifier_id}' setup command {i} exited {exit_code}: " + f"{stderr[-_ERROR_TAIL_CHARS:]}") + if self.reward_path: + exit_code, _, stderr = await target.run("rm", "-f", "--", self.reward_path) + if exit_code != 0: + raise RuntimeError(f"Verifier '{self.verifier_id}' couldn't clear the reward file {self.reward_path!r} " + f"before the tests, so one left there would be graded: {stderr.strip()}") + logger.info(f"[{self.verifier_id}] running verifier (timeout={self.timeout_sec}s) in env '{self.env_id}': " + f"{command[:160]}") + exit_code, stdout, stderr = await target.run("bash", "-c", command) + return exit_code, stdout, stderr, await self._extracted(target.read_file) + + def _log_setup(self, i: int, setup_cmd: str) -> None: + logger.info(f"[{self.verifier_id}] setup [{i}/{len(self.setup_commands)}]: {setup_cmd.splitlines()[0][:140]}") + + async def _extracted(self, read: Callable[[str], Awaitable[str]]) -> dict[str, Any]: + """Each result file as ``read`` reads it, parsed when it's JSON; None for one that couldn't be read.""" + extracted_files: dict[str, Any] = {} + for result_path in self.result_paths: + try: + content = await read(result_path) + try: + extracted_files[result_path] = json.loads(content) + except json.JSONDecodeError: + extracted_files[result_path] = content + except Exception as e: + logger.warning(f"[{self.verifier_id}] failed to extract {result_path}: {e}") + extracted_files[result_path] = None + return extracted_files + @staticmethod def _parse_reward(raw: Any) -> Optional[float]: """Coerce an extracted reward file's content to a float in [0, 1], or None. diff --git a/tst/integration/computer/app/artifacts/check_notes/run.py b/tst/integration/computer/app/artifacts/check_notes/run.py new file mode 100644 index 000000000..38b72f11b --- /dev/null +++ b/tst/integration/computer/app/artifacts/check_notes/run.py @@ -0,0 +1,17 @@ +"""What the notes app's service wrote, which it can only once its settings reached it, and the name the computer's +card gives, asked on loopback from inside the computer.""" +import json +import time +import urllib.request +from pathlib import Path + + +def run(input): + written = Path.home() / ".notes.json" + for _ in range(60): + if written.exists(): + break + time.sleep(0.5) + with urllib.request.urlopen(f"{input['env']['url']}/.well-known/agent-env.json", timeout=10) as response: + card = json.load(response) + return {**json.loads(written.read_text()), "card": card["name"]} diff --git a/tst/integration/computer/app/envs/notes/app.py b/tst/integration/computer/app/envs/notes/app.py new file mode 100644 index 000000000..d96ccfeef --- /dev/null +++ b/tst/integration/computer/app/envs/notes/app.py @@ -0,0 +1,15 @@ +"""A notes app for the computer app tests: its service writes the settings a deploy gave it to ~/.notes.json.""" +from pydantic import BaseModel, SecretStr + +from agentenv_protocol.computer import ComputerAppEnvironment, Launch, Linux, Service, computer_app + + +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +class Notes(ComputerAppEnvironment): + linux = Linux(python="requirements.txt", services=[Service(name="notes", command="venv/bin/python serve.py")]) + launch = Launch(at="demand", command=["xdg-open", ".notes.json"]) + skill = "skill.md" + + class Settings(BaseModel): + signature: str = "-- notes" + token: SecretStr = SecretStr("") diff --git a/tst/integration/computer/app/envs/notes/env.toml b/tst/integration/computer/app/envs/notes/env.toml new file mode 100644 index 000000000..61191b1ce --- /dev/null +++ b/tst/integration/computer/app/envs/notes/env.toml @@ -0,0 +1,4 @@ +type = "computer_app" +module = "app.py:Notes" +computer = "ubuntu-24.04" +driver = "reference" diff --git a/tst/integration/computer/app/envs/notes/requirements.txt b/tst/integration/computer/app/envs/notes/requirements.txt new file mode 100644 index 000000000..f671e2e13 --- /dev/null +++ b/tst/integration/computer/app/envs/notes/requirements.txt @@ -0,0 +1 @@ +agentenv-framework-protocol diff --git a/tst/integration/computer/app/envs/notes/serve.py b/tst/integration/computer/app/envs/notes/serve.py new file mode 100644 index 000000000..44620ad7a --- /dev/null +++ b/tst/integration/computer/app/envs/notes/serve.py @@ -0,0 +1,11 @@ +import json +import time +from pathlib import Path + +from app import Notes + +notes = Notes() +Path.home().joinpath(".notes.json").write_text(json.dumps( + {"signature": notes.settings.signature, "token": notes.settings.token.get_secret_value()})) +while True: + time.sleep(3600) diff --git a/tst/integration/computer/app/envs/notes/skill.md b/tst/integration/computer/app/envs/notes/skill.md new file mode 100644 index 000000000..6c9494109 --- /dev/null +++ b/tst/integration/computer/app/envs/notes/skill.md @@ -0,0 +1 @@ +The notes app keeps its settings in `~/.notes.json`. diff --git a/tst/integration/computer/app/tasks/alone.json b/tst/integration/computer/app/tasks/alone.json new file mode 100644 index 000000000..3f1fb6893 --- /dev/null +++ b/tst/integration/computer/app/tasks/alone.json @@ -0,0 +1,5 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "notes", + "env_params": {"app_settings": {"notes": {"signature": "-- Ada", "token": "env:NOTES_TOKEN"}}}}, + {"id": "service", "type": "run_code", "env_id": "notes", "script_artifact_id": "check_notes"} +] diff --git a/tst/integration/computer/computer_app_test.py b/tst/integration/computer/computer_app_test.py new file mode 100644 index 000000000..7a6903043 --- /dev/null +++ b/tst/integration/computer/computer_app_test.py @@ -0,0 +1,28 @@ +"""A computer app deployed alone through ``agent-env run``: a one-app computer of the computer and driver its env.toml +names, its card named after it, its service running from its folder in a venv of the plain protocol package, on the +settings the deploy gave it, a secret resolved at boot. Needs Docker and the local registry on localhost:5000.""" + +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from agent_env.cli import cli +from tst.integration.computer.conftest import run_context + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] + +APP = Path(__file__).parent / "app" + + +def test_an_app_deployed_alone_runs_its_service_on_its_settings_on_a_computer_named_after_it(state, monkeypatch): + monkeypatch.setenv("NOTES_TOKEN", "t0ken") + + ran = CliRunner().invoke(cli, ["run", str(APP), "--task", "alone"]) + + assert ran.exit_code == 0, ran.output + context = run_context(ran.output) + (deployed,) = context["deployed_envs"] + assert deployed["env_id"].endswith("/notes") and deployed["environment_card"]["name"] == "notes" + assert context["metadata"]["script_results"]["service"] == {"signature": "-- Ada", "token": "t0ken", + "card": "notes"} diff --git a/tst/integration/computer/computer_env_test.py b/tst/integration/computer/computer_env_test.py index e953f28b7..5463a3eac 100644 --- a/tst/integration/computer/computer_env_test.py +++ b/tst/integration/computer/computer_env_test.py @@ -2,8 +2,8 @@ builds an image for each arch this machine's Docker can build, pushed to the local registry and written as the env's computer, and gate 3 makes the image of this machine's arch ready; that image boots with everything running as the session user and passes the conformance probe at full three times in a row; another env that changes only what doesn't -change the image reuses it; one with no network, deployed, is reached but reaches nothing, comes back from a reset -fresh, and leaves nothing once torn down; and a recording open when the computer resets or is torn down is ended by +change the image reuses it; one with no network, deployed, is reached but reaches nothing, takes a file and runs +code that reaches its driver on loopback, comes back from a reset fresh, and leaves nothing once torn down; and a recording open when the computer resets or is torn down is ended by the computer and registered whole. Needs Docker and the local registry on localhost:5000.""" import asyncio @@ -18,6 +18,7 @@ from agentenv_protocol.computer.conformance import run_probe from agentenv_protocol.computer.conformance.client import Driver +from agent_env.artifact import FileArtifact from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.env import Env from agent_env.env.envs.computer.env import ComputerEnv @@ -31,13 +32,20 @@ EndComputerRecordingTaskStep, StartComputerRecordingTaskStep, ) +from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep +from agent_env.task_step.task_steps.run_code import RunCodeTaskStep from tst.integration.computer.conftest import docker pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] PORT = 18765 +READ_NOTE = b"""import json, urllib.request +def run(input): + with urllib.request.urlopen(input["env"]["url"] + "/.well-known/agent-env.json", timeout=10) as response: + return {"note": open("Downloads/note.txt").read(), "card": json.load(response)["name"]} +""" def test_each_arch_this_machine_builds_is_composed_pushed_and_written_as_the_envs_computer(desktop): @@ -99,6 +107,14 @@ async def test_with_no_network_it_is_reached_reaches_nothing_comes_back_fresh_an deployed = await Env.get(closed.id).deploy() context = TaskStepContext(deployed_envs=[deployed]) try: + note = FileArtifact.put_bytes(f"{closed.id}-note", description="", filename="note.txt", content=b"inside\n", + content_type="text/plain") + script = FileArtifact.put_bytes(f"{closed.id}-read", description="", filename="run.py", content=READ_NOTE, + content_type="text/x-python") + context = await LoadArtifactTaskStep(id="load", version=None, env_id=closed.id, + artifact_id=note.id).execute(context) + context = await RunCodeTaskStep(id="read", version=None, env_id=closed.id, + script_artifact_id=script.id).execute(context) driver = await Driver.connect(deployed.environment_url, 30) egress = await driver.run("python3", "-c", "import urllib.request; urllib.request.urlopen('http://1.1.1.1', timeout=5)") await driver.run("sh", "-c", "echo x > Desktop/left.txt") @@ -108,6 +124,7 @@ async def test_with_no_network_it_is_reached_reaches_nothing_comes_back_fresh_an report = await teardown_run(context) sandbox = deployed.sandbox_id + assert context.metadata["script_results"]["read"] == {"note": "inside\n", "card": "closed"} assert egress["exit_code"] != 0 and "Network is unreachable" in egress["stderr"], egress assert left["exit_code"] != 0 assert [s.sandbox_id for s in report.terminated] == [sandbox] and not report.failed diff --git a/tst/integration/computer/conftest.py b/tst/integration/computer/conftest.py index fa1bbe76e..64b9ac3e2 100644 --- a/tst/integration/computer/conftest.py +++ b/tst/integration/computer/conftest.py @@ -1,7 +1,9 @@ """The ubuntu-24.04 base computer, built from its folder once for the tests here and tagged for the local registry; the -shipped desktop example's computer env, composed once for a module's tests; and ``docker``, which runs the docker CLI -for them.""" +shipped desktop example's computer env, composed once for a module's tests; ``state``, local stores for a test that +runs a bundle; and ``docker``, which runs the docker CLI for them.""" +import logging +import re import subprocess import uuid from pathlib import Path @@ -17,7 +19,9 @@ from agent_env.bundle.resolve import BuiltImage, resolve_bundle from agent_env.config import configure, reset_config from agent_env.env import Env +from agent_env.store.object_store.local.grant_server import grant_server from agent_env.store.routing import namespace_routing +from agent_env.task.store import get_task_instance_store BASE = Path(agent_env.__file__).parent / "computers" / "artifacts" / "ubuntu-24.04" @@ -54,3 +58,31 @@ def desktop(tmp_path_factory): def docker(*args: str) -> subprocess.CompletedProcess: return subprocess.run(["docker", *args], check=True, capture_output=True) + + +@pytest.fixture +def state(monkeypatch, tmp_path): + """Local stores under this test's folder. Whatever ran, no sandbox work folder may be left behind.""" + sandboxes = tmp_path / "sandboxes" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + monkeypatch.setenv("AGENT_ENV_DOCUMENT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_OBJECT_STORE", "local") + monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(sandboxes)) + configure() + reset_artifact_store() + logging.disable(logging.CRITICAL) + try: + yield tmp_path + assert not sandboxes.exists() or not any(sandboxes.iterdir()), "a run left a sandbox work folder" + finally: + logging.disable(logging.NOTSET) + grant_server(None, None).close() + reset_artifact_store() + reset_config() + + +def run_context(output: str) -> dict: + """The context of the run whose ``agent-env run`` output ``output`` is.""" + with namespace_routing(): + return get_task_instance_store().get(re.search(r"instance (@local/\S+)", output).group(1)).context diff --git a/tst/integration/computer/desktop_example_test.py b/tst/integration/computer/desktop_example_test.py index 61afb2b7d..f5432737a 100644 --- a/tst/integration/computer/desktop_example_test.py +++ b/tst/integration/computer/desktop_example_test.py @@ -1,11 +1,10 @@ """The shipped desktop example through ``agent-env run``: its computer env, deployed on local Docker, passes the -conformance probe at full; a scripted drive of the conformance fixture on it is recorded on the clock its screenshots -carry; and an agent given the computer's skill makes a folder through the computer_* tools under a recording. Needs -Docker and the local registry; the agent's task, a model endpoint too.""" +conformance probe at full; a file loaded into it is read by code run inside it and checked by a command there; a +scripted drive of the conformance fixture on it is recorded on the clock its screenshots carry; and an agent given the +computer's skill files a loaded file in a new folder through the computer_* tools under a recording. Needs Docker and +the local registry; the agent's task, a model endpoint too.""" import json -import logging -import re from datetime import datetime from pathlib import Path @@ -15,12 +14,10 @@ from PIL import Image from agent_env.artifact import Artifact -from agent_env.artifact.store import reset_artifact_store from agent_env.cli import cli -from agent_env.config import configure, get_config, reset_config -from agent_env.store.object_store.local.grant_server import grant_server +from agent_env.config import get_config from agent_env.store.routing import namespace_routing -from agent_env.task.store import get_task_instance_store +from tst.integration.computer.conftest import run_context from tst.util.capabilities import skip_without_model_endpoint pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] @@ -28,28 +25,6 @@ RECORDING = Path(__file__).parent / "recording" -@pytest.fixture -def state(monkeypatch, tmp_path): - """Local stores under this test's folder. Whatever ran, no sandbox work folder may be left behind.""" - sandboxes = tmp_path / "sandboxes" - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) - monkeypatch.setenv("AGENT_ENV_DOCUMENT_STORE", "local") - monkeypatch.setenv("AGENT_ENV_OBJECT_STORE", "local") - monkeypatch.setenv("AGENT_ENV_LOCAL_SANDBOX_DIR", str(sandboxes)) - configure() - reset_artifact_store() - logging.disable(logging.CRITICAL) - try: - yield tmp_path - assert not sandboxes.exists() or not any(sandboxes.iterdir()), "a run left a sandbox work folder" - finally: - logging.disable(logging.NOTSET) - grant_server(None, None).close() - reset_artifact_store() - reset_config() - - def test_the_desktop_examples_computer_passes_the_probe_at_full(state): ran = _run("desktop", "probe") @@ -57,6 +32,17 @@ def test_the_desktop_examples_computer_passes_the_probe_at_full(state): assert "passed (probe: 1)" in ran.output, ran.output +def test_a_file_loaded_into_the_desktop_is_read_by_code_inside_it_and_checked_there(state): + ran = _run("desktop", "files") + + assert ran.exit_code == 0, ran.output + assert "passed (tests: 1)" in ran.output, ran.output + context = run_context(ran.output) + (loaded,) = context["metadata"]["loaded_file_artifact_universes"] + assert (loaded["destination_path"], loaded["files"]) == ("Documents", {"figures.csv": "Documents/figures.csv"}) + assert context["metadata"]["script_results"]["inside"] == {"header": "quarter,revenue", "card": "desktop"} + + def test_a_scripted_drive_is_recorded_on_the_clock_its_screenshots_carry(state): ran = _run(str(RECORDING), "recording") @@ -87,7 +73,7 @@ def test_a_scripted_drive_is_recorded_on_the_clock_its_screenshots_carry(state): @skip_without_model_endpoint() -def test_an_agent_given_the_computers_skill_makes_a_folder_under_a_recording(state): +def test_an_agent_given_the_computers_skill_files_a_loaded_file_in_a_new_folder_under_a_recording(state): ran = _run("desktop", "agent") assert ran.exit_code == 0, ran.output @@ -105,10 +91,9 @@ def _run(bundle: str, task: str): def _recorded(output: str, name: str, tmp: Path) -> tuple[dict, dict, list]: """The run's context, the manifest of its recording ``name``, and every frame of the video with its time on the computer's clock.""" - instance_id = re.search(r"instance (@local/\S+)", output).group(1) + context = run_context(output) + entry = context["metadata"]["computer_recordings"][name] with namespace_routing(): - context = get_task_instance_store().get(instance_id).context - entry = context["metadata"]["computer_recordings"][name] universe = Artifact.get(entry["artifact"]["id"], version=entry["artifact"]["version"]) objects = get_config().get_object_store() files = {name: objects.get(Artifact.get(ref).object_url) for name, ref in universe.file_artifact_ids.items()} diff --git a/tst/integration/computer/recording/artifacts/drive_desk/verify.py b/tst/integration/computer/recording/artifacts/drive_desk/verify.py index 9eec904dd..2e7b811e4 100644 --- a/tst/integration/computer/recording/artifacts/drive_desk/verify.py +++ b/tst/integration/computer/recording/artifacts/drive_desk/verify.py @@ -21,8 +21,8 @@ def _thumb(result): return Image.open(io.BytesIO(base64.b64decode(data))).convert("L").resize(THUMB).tobytes().hex() -async def verify(mcp_url): - driver = await Driver.connect(mcp_url.rsplit("/mcp", 1)[0], 60) +async def verify(mcp_url, env_url): + driver = await Driver.connect(env_url, 60) await driver.add_file("desk/fixture.py", FIXTURE) await driver.run("sh", "-c", "python3 desk/fixture.py fixture > desk/fixture.log 2>&1 &") for _ in range(60): # the Fixture Desk writes its layout once it is up diff --git a/tst/unit/bundle/computer_app_env_test.py b/tst/unit/bundle/computer_app_env_test.py new file mode 100644 index 000000000..2f9fe7b5a --- /dev/null +++ b/tst/unit/bundle/computer_app_env_test.py @@ -0,0 +1,111 @@ +"""Computer app envs written from a bundle's env folders: the app's files stored with each version, its class read where +its own dependencies aren't installed, what it declares, refusals before any write, and a deploy refused until it names +a computer and a driver.""" + +import json + +import pytest +from agentenv_protocol.computer import Launch, Linux, Service +from agentenv_protocol.computer.app import AppSkill, InstallStep + +from agent_env.bundle import BundleError +from agent_env.bundle.materialize import materialize +from agent_env.env import Env +from agent_env.env.envs.computer.app_env import ComputerAppEnv +from agent_env.providers.env_providers.env_provider import DeployRefused +from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep +from tst.unit.bundle._support import layout, plan_of +from tst.unit.bundle.computer_driver_env_test import DRIVER, LINUX + +ROOT = "@local/~/apps" +APP = '''import not_installed_here +from pydantic import BaseModel, SecretStr + +from agentenv_protocol import add_data, get_data +from agentenv_protocol.computer import ComputerAppEnvironment, Launch, Linux, Service, computer_app, install_step + + +@computer_app(supports={supports}, permissions=["camera"]) +class Mail(ComputerAppEnvironment): + linux = Linux(python="requirements.txt", services=[Service(name="imap", command="venv/bin/python imap.py")]) + launch = Launch(command=["thunderbird"], window={{"class": "thunderbird"}}) + skill = {skill!r} + + class Settings(BaseModel): + signature: str = "" + password: SecretStr = SecretStr("") + + @install_step(scope="session") + def profile(self): not_installed_here.profile() + + @add_data + async def load(self, parts): pass + + @get_data + async def export(self): return [] +''' +TOML = 'type = "computer_app"\nmodule = "app.py:Mail"\n' + + +@pytest.fixture +def bundle_dir(tmp_path, monkeypatch, local_stores, cli_routing): + monkeypatch.setenv("HOME", str(tmp_path)) + return tmp_path / "apps" + + +def _bundle(root, toml=TOML, supports=LINUX, skill="skill.md", **extra): + files = {"envs/mail/env.toml": toml, "envs/mail/app.py": APP.format(supports=supports, skill=skill), + "envs/mail/requirements.txt": "agentenv-framework-protocol\n", "envs/mail/skill.md": "# Mail\n"} + return layout(root, {**files, **extra, "tasks/t.json": json.dumps([{"id": "agent", "type": "deploy_agent", + "env_ids": ["mail"]}])}) + + +def _problems(root): + with pytest.raises(BundleError) as caught: + materialize(plan_of(root)) + return " ".join(caught.value.problems) + + +@pytest.mark.asyncio +async def test_an_app_folder_is_written_with_its_files_and_what_its_class_declares(bundle_dir): + _bundle(bundle_dir) + + materialize(plan_of(bundle_dir)) + env = Env.get(f"{ROOT}/mail") + + assert isinstance(env, ComputerAppEnv) and (env.environment_name, env.module) == ("mail", "app.py:Mail") + assert sorted(env.files.file_artifact_ids) == ["app.py", "requirements.txt", "skill.md"] + declared = env.declaration + assert (declared.supports, declared.permissions) == (LINUX, ["camera"]) + assert declared.install == {"linux": Linux(python="requirements.txt", + services=[Service(name="imap", command="venv/bin/python imap.py")])} + assert declared.launch == Launch(command=["thunderbird"], window={"class": "thunderbird"}) + assert declared.settings["properties"]["password"]["writeOnly"] is True + assert (declared.skill, declared.data) == (AppSkill(file="skill.md"), ["data/add", "data/get"]) + assert declared.install_steps == [InstallStep(name="profile", scope="session")] + assert (env.computer, env.driver) == (None, None) + assert ComputerAppEnv.from_dict(env.to_dict()).to_dict() == env.to_dict() + (refusal,) = DeployEnvTaskStep(id="env", version=None, env_id=env.id).preflight() + assert "names no computer and driver to deploy alone on; name them in its env.toml" in refusal + with pytest.raises(DeployRefused, match="names no computer and driver to deploy alone on"): + await env.deploy() + + +@pytest.mark.parametrize("toml, extra, skill, problem", [ + ('type = "computer_app"\n', {}, "skill.md", "module is required"), + ('type = "computer_app"\nmodule = "app.py:Desk"\n', {"envs/mail/app.py": DRIVER.format(supports=LINUX, + buttons=("left",))}, + "skill.md", "app.py:Desk is a class a computer_driver env declares, not a computer_app env"), + (TOML, {"envs/mail/requirements.txt": None}, "skill.md", + "install.linux.python: 'requirements.txt' isn't in this folder"), + (TOML, {}, "guide.md", "skill: 'guide.md' isn't in this folder"), + (f'{TOML}image = "x"\n', {}, "skill.md", "unknown key 'image'"), +], ids=["no-module", "a-driver", "install-python", "skill", "unknown-key"]) +def test_an_app_folder_that_cant_be_read_is_refused_before_any_write(bundle_dir, toml, extra, skill, problem): + _bundle(bundle_dir, toml=toml, skill=skill, **{path: data for path, data in extra.items() if data is not None}) + for path in (path for path, data in extra.items() if data is None): + (bundle_dir / path).unlink() + + assert problem in _problems(bundle_dir) + with pytest.raises(Exception): + Env.get(f"{ROOT}/mail") diff --git a/tst/unit/bundle/computer_driver_env_test.py b/tst/unit/bundle/computer_driver_env_test.py index cb49ac76c..c2562fbdc 100644 --- a/tst/unit/bundle/computer_driver_env_test.py +++ b/tst/unit/bundle/computer_driver_env_test.py @@ -6,6 +6,7 @@ import json import pytest +from agentenv_protocol.computer.platform import Platform from agent_env.bundle import BundleError from agent_env.bundle.installed import installed_bundles @@ -15,8 +16,8 @@ from agent_env.bundle.run import dry_run_bundle from agent_env.bundle import parse_bundle from agent_env.env import Env -from agent_env.env.envs import computer_driver -from agent_env.env.envs.computer_driver import ComputerDriverEnv +from agent_env.env.envs.computer import driver_env as computer_driver +from agent_env.env.envs.computer.driver_env import ComputerDriverEnv from tst.unit.bundle._support import layout, plan_of ROOT = "@local/~/desks" @@ -93,12 +94,12 @@ def test_a_driver_folder_is_written_with_its_files_and_what_its_class_declares(b assert isinstance(env, ComputerDriverEnv) and env.module == "driver.py:Desk" assert sorted(env.files.file_artifact_ids) == ["driver.py"] declared = env.declaration - assert declared["name"] == "desk" and declared["supports"] == LINUX - assert [p["platform"] for p in declared["platforms"]] == [{"os": "linux", "arch": "amd64", "display": "x11"}, - {"os": "linux", "arch": "arm64", "display": "x11"}] - linux = declared["platforms"][0] - assert "urn:agentenv:computer-exec/v1" in linux["provides"] and "urn:agentenv:computer-pointer/v1" not in linux["provides"] - assert len(linux["tools"]) == 7 and linux["features"]["buttons"] == ["left", "right"] + assert declared.name == "desk" and declared.supports == LINUX + assert [p.platform for p in declared.platforms] == [Platform(os="linux", arch="amd64", display="x11"), + Platform(os="linux", arch="arm64", display="x11")] + linux = declared.platforms[0] + assert "urn:agentenv:computer-exec/v1" in linux.provides and "urn:agentenv:computer-pointer/v1" not in linux.provides + assert len(linux.tools) == 7 and linux.features["buttons"] == ["left", "right"] assert ComputerDriverEnv.from_dict(env.to_dict()).to_dict() == env.to_dict() @@ -124,7 +125,7 @@ def test_a_driver_is_written_again_only_when_one_of_its_files_changes(bundle_dir rewritten = _run(bundle_dir) assert rewritten[f"{ROOT}/desk"] == (2, False, ("files changed: driver.py",)) - assert Env.get(f"{ROOT}/desk").declaration["platforms"][0]["features"]["buttons"] == ["left"] + assert Env.get(f"{ROOT}/desk").declaration.platforms[0].features["buttons"] == ["left"] @pytest.mark.parametrize("toml, supports, buttons, problem", [ @@ -132,7 +133,7 @@ def test_a_driver_is_written_again_only_when_one_of_its_files_changes(bundle_dir ('type = "computer_driver"\nmodule = "server.py:Desk"\n', LINUX, ("left",), "module 'server.py:Desk' names no \":\" in this folder; it has driver.py"), ('type = "computer_driver"\nmodule = "driver.py:Missing"\n', LINUX, ("left",), - "driver.py has no ComputerDriverEnvironment subclass named Missing"), + "driver.py has no ComputerDriverEnvironment or ComputerAppEnvironment subclass named Missing"), (TOML, LINUX, ("thumb",), "buttons include left"), (TOML, [], ("left",), "Desk declares no platform: name them with @computer_driver(supports=[...])"), ('type = "computer_driver"\nmodule = "driver.py:Desk"\nimage = "x"\n', LINUX, ("left",), "unknown key 'image'"), @@ -196,8 +197,8 @@ def test_a_run_naming_the_reference_driver_writes_the_shipped_one(tmp_path, monk env = Env.get("@local/agentenv-framework/drivers/reference") assert "@local/agentenv-framework/drivers/reference" in written - assert env.declaration["name"] == "reference" and len(env.declaration["platforms"]) == 2 + assert env.declaration.name == "reference" and len(env.declaration.platforms) == 2 assert {"urn:agentenv:computer-pointer/v1", "urn:agentenv:computer-keyboard/v1", - "urn:agentenv:computer-exec/v1"} <= set(env.declaration["platforms"][0]["provides"]) + "urn:agentenv:computer-exec/v1"} <= set(env.declaration.platforms[0].provides) assert (env.port, env.install["linux"].python) == (18765, "requirements.txt") assert env.install["linux"].services[0].command.endswith("agentenv_protocol.computer.serve driver.py:ReferenceDriver") diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index 4f3c38cc3..fc904fb04 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -13,7 +13,7 @@ from agent_env.bundle.plan import plan_bundle, plan_entities from agent_env.bundle.resolve import resolve_bundle from agent_env.env import Env -from agent_env.env.envs.computer import composer +from agent_env.env.envs.computer.utils import composer from agent_env.env.envs.computer.env import ComputerEnv from tst.unit.bundle._support import layout, local_builds diff --git a/tst/unit/bundle/materialize_test.py b/tst/unit/bundle/materialize_test.py index 8cdf8226f..ad80409b5 100644 --- a/tst/unit/bundle/materialize_test.py +++ b/tst/unit/bundle/materialize_test.py @@ -980,8 +980,9 @@ def test_a_dry_run_doesnt_preflight_a_step_reading_what_it_would_write_and_lists rewritten = _run(bundle_dir, dry_run=True) - assert [(write.id, step.id) for write, step in new.not_preflighted] == [(f"{ROOT}/t", "check")] - assert [(write.id, step.id) for write, step in rewritten.not_preflighted] == [(f"{ROOT}/t", "check")] + assert [(write.id, step.id) for write, step in new.not_preflighted] == [(f"{ROOT}/t", "greet"), (f"{ROOT}/t", "check")] + assert [(write.id, step.id) for write, step in rewritten.not_preflighted] == [(f"{ROOT}/t", "greet"), + (f"{ROOT}/t", "check")] def test_a_second_run_of_the_bundle_waits_for_the_first_to_finish_writing(bundle_dir, tmp_path): diff --git a/tst/unit/computer_home.py b/tst/unit/computer_home.py new file mode 100644 index 000000000..ae4348d5d --- /dev/null +++ b/tst/unit/computer_home.py @@ -0,0 +1,75 @@ +"""A computer at its driver's boundary, for steps that reach inside one: its home a folder on this machine, ``data/add`` +writing there, ``data/get`` reading there, and computer-exec/v1 ``run`` running commands there; each call recorded. +``pkill`` matches nothing, so one test never ends another's processes on this machine.""" + +from __future__ import annotations + +import asyncio +import base64 +import os +import subprocess +from pathlib import Path + +from agentenv_protocol import DATA_OBJECTS_EXTENSION_URI, FilePart +from agentenv_protocol import client as protocol_v1 +from agentenv_protocol.computer import COMPUTER_EXEC +from agentenv_protocol.types import FileWithBytes, GetDataResponse + +from agent_env.env.env import DeployedSandboxEnv + +PORT = 18765 +CARD = {"name": "desk", "capabilities": {"operations": ["data/reset", "data/add", "data/get"], "extensions": [ + {"uri": COMPUTER_EXEC, "params": {"endpoint": "/computer/exec", "methods": {"run": {}}, + "max_output_bytes": 1 << 20, "max_timeout_s": 3600}}]}} + + +class Computer: + def __init__(self, home: Path): + self.home = home + self.calls: list[tuple] = [] + self.add_fails: Exception | None = None + + def deployed(self, env_id: str = "desk", card: dict | None = None, **computer) -> DeployedSandboxEnv: + return DeployedSandboxEnv(env_id=env_id, env_version=1, env_provider_type="computer", + environment_card_url="http://127.0.0.1:20001/.well-known/agent-env.json", + environment_card=card or CARD, sandbox_id="local-1", sandbox_type="local", + metadata={"computer": {"port": PORT, **computer}}) + + def install(self, monkeypatch) -> Computer: + monkeypatch.setattr(protocol_v1, "add_data", self.add_data) + monkeypatch.setattr(protocol_v1, "get_data", self.get_data) + monkeypatch.setattr(protocol_v1, "invoke_extension", self.invoke_extension) + return self + + async def add_data(self, base_url, parts, timeout=120, verify=True): + self.calls.append(("data/add", [p.metadata[DATA_OBJECTS_EXTENSION_URI] for p in parts], timeout)) + if self.add_fails is not None: + raise self.add_fails + for part in parts: + if isinstance(part.file, FileWithBytes): + target = self.home / part.metadata[DATA_OBJECTS_EXTENSION_URI]["path"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(base64.b64decode(part.file.bytes)) + + async def get_data(self, base_url, timeout=30, verify=True, *, write_namespace=None, paths=None): + self.calls.append(("data/get", paths)) + return GetDataResponse(parts=[ + FilePart(file=FileWithBytes(bytes=base64.b64encode((self.home / path).read_bytes()).decode()), + metadata={DATA_OBJECTS_EXTENSION_URI: {"path": path}}) + for path in paths if (self.home / path).is_file()]) + + async def invoke_extension(self, base_url, card, uri, params=None, timeout=30, verify=True, *, method=None): + assert (uri, method) == (COMPUTER_EXEC, "run") + self.calls.append(("run", params, timeout)) + return await asyncio.to_thread(self._run, params) + + def _run(self, params: dict) -> dict: + if params["argv"][0] == "pkill": + return {"exit_code": 1, "stdout": "", "stderr": "", "truncated": False, "timed_out": False} + try: + done = subprocess.run(params["argv"], cwd=self.home, env={**os.environ, **(params.get("env") or {})}, + capture_output=True, text=True, timeout=params["timeout_s"]) + except subprocess.TimeoutExpired: + return {"exit_code": None, "stdout": "", "stderr": "", "truncated": False, "timed_out": True} + return {"exit_code": done.returncode, "stdout": done.stdout, "stderr": done.stderr, "truncated": False, + "timed_out": False} diff --git a/tst/unit/env/envs/test_artifact_staging.py b/tst/unit/env/envs/test_artifact_staging.py index 13cb6042e..84dbf61b8 100644 --- a/tst/unit/env/envs/test_artifact_staging.py +++ b/tst/unit/env/envs/test_artifact_staging.py @@ -14,7 +14,7 @@ from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.website import WebsiteEnv -from agent_env.providers.sandbox_providers.local_sandbox import LocalSandbox +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox _SHELL_NAMES = [ ("email", "x;touch pwned;#.json"), diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 8e54d46de..470b5d1cf 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -7,13 +7,15 @@ import io import json import logging +import re import subprocess import tarfile from types import SimpleNamespace from unittest.mock import AsyncMock +import httpx import pytest -from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING +from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING, AppDeclaration, DriverDeclaration from agentenv_protocol.computer.conformance import CheckResult, ProbeReport from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI from click.testing import CliRunner @@ -27,11 +29,12 @@ from agent_env.cli import cli from agent_env.env import Env from agent_env.env import env as env_module -from agent_env.env.envs.computer import boot, composer, gate -from agent_env.env.envs.computer.composer import _probe +from agent_env.env.envs.computer.utils import boot, composer, gate, inside +from agent_env.env.envs.computer.utils.composer import _probe from agent_env.env.envs.computer.env import ComputerEnv -from agent_env.env.envs.computer.install import Install, Service -from agent_env.env.envs.computer_driver import ComputerDriverEnv +from agent_env.env.envs.computer.app_env import ComputerAppEnv +from agent_env.env.envs.computer.utils.install import Install, Service +from agent_env.env.envs.computer.driver_env import ComputerDriverEnv from agent_env.plugins._bundles import ALIASES_FILE, InstalledBundle, installed_aliases from agent_env.providers.env_providers import env_computer_provider from agent_env.providers.env_providers.env_provider import DeployRefused @@ -45,6 +48,7 @@ from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore from agent_env.store.image_store.oci_registry_credentials import RegistryAuth from agent_env.utils.docker.build import DockerBuildError +from tst.unit.computer_home import Computer from tst.util.oci_registry import UBUNTU_LAYER, Registry OS = {"type": "linux", "distribution": "ubuntu", "version": "24.04", "display": "x11"} @@ -79,7 +83,8 @@ def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_ raise DockerBuildError(self.fail[platform]) self.builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), "files": sorted(p.relative_to(context).as_posix() for p in context.rglob("*") if p.is_file()), - "conf": next(context.glob("agentenv-*.conf")).read_text()}) + "conf": next(context.glob("agentenv-*.conf")).read_text(), + "confs": {conf.name: conf.read_text() for conf in context.glob("agentenv-*.conf")}}) def label(self, local, platform, extra): self.labels[local] = extra @@ -154,7 +159,7 @@ def _driver(folder, requirements="mss>=9,<11\n", supports=None, services=None, i services = services or [Service(name="driver", command=SERVE)] return ComputerDriverEnv.put( id="@local/~/kit/desk", module="driver.py:Desk", files=files, - declaration={"name": "desk", "supports": supports, "platforms": platforms}, + declaration=DriverDeclaration.model_validate({"name": "desk", "supports": supports, "platforms": platforms}), install={"linux": Install(python="requirements.txt", services=services)} if install else {}, port=18765) @@ -431,10 +436,13 @@ def test_env_build_builds_what_a_put_left_and_prints_each_pair(base, driver, doc assert ComputerArtifact.get(Env.get(env.id).metadata["computer"]["id"]).arch == ["amd64", "arm64"] -def test_env_build_refuses_an_env_that_isnt_a_computer_env(base, driver, docker): +def test_env_build_refuses_an_env_that_isnt_a_computer_env_or_an_app_naming_no_computer(base, driver, docker, + tmp_path): refused = CliRunner().invoke(cli, ["env", "build", "--id", driver.id]) + alone = CliRunner().invoke(cli, ["env", "build", "--id", _app(None, None, tmp_path / "mail").id]) assert refused.exit_code == 1 and "is a computer_driver env; only a computer env is built" in refused.output + assert alone.exit_code == 1 and "names no computer and driver, so it has nothing to build alone" in alone.output @pytest.mark.asyncio @@ -651,7 +659,8 @@ async def terminate(): sandbox.terminate = terminate made.sandboxes.append(sandbox) - return boot.Booted(sandbox, "http://127.0.0.1:20001", DRIVER_CARD) + return boot.Booted(sandbox, "http://127.0.0.1:20001", + {**DRIVER_CARD, "name": kwargs["environment"]["ENVIRONMENT_NAME"]}) async def invoke(base_url, card, uri, params=None, timeout=30, verify=True, *, method=None): made.calls.append((uri, method, params)) @@ -764,8 +773,9 @@ async def terminating(): ({"driver_settings": {"default_space": "polar"}}, "env_params.driver_settings.default_space"), ({"driver_settings": {"colour": "blue"}}, "env_params.driver_settings.colour: Extra inputs are not permitted"), ({"colour": "blue"}, "env_params.colour"), + ({"app_settings": {"mail": {}}}, "env_params.app_settings.mail: @local/~/kit/desktop has no app 'mail'"), ], ids=["form", "build-setting", "resolution-over-display", "setting-type", "driver-setting", "driver-unknown", - "unknown"]) + "unknown", "app"]) async def test_env_params_a_computer_env_cant_take_are_refused_at_save_and_at_deploy(base, driver, docker, boots, env_params, problem): env = _put(base, driver, display={"width": 1600, "height": 900}) @@ -843,3 +853,220 @@ def test_a_deploy_its_sandbox_provider_cant_carry_out_is_refused_at_save(base, d (refusal,) = DeployEnvTaskStep(id="d", version=None, env_id=env.id).preflight() assert problem in refusal + + +@pytest.fixture +def computer(tmp_path, monkeypatch): + (tmp_path / "home").mkdir() + return Computer(tmp_path / "home").install(monkeypatch) + + +def _deployed_env(base, driver, computer, **record): + env = _put(base, driver, compose=False) + env._deployed = computer.deployed(env_id=env.id, **record) + return env + + +def _files(tmp_path, **contents): + for name, data in contents.items(): + (tmp_path / "files" / name).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / "files" / name).write_bytes(data) + return FileArtifactUniverse.put_bundled("@local/~/kit/files", files={ + name: tmp_path / "files" / name for name in contents}) + + +@pytest.mark.asyncio +async def test_a_load_writes_each_file_into_the_computers_downloads_through_its_driver_in_batches( + base, driver, docker, computer, tmp_path, monkeypatch): + monkeypatch.setattr(inside, "BATCH_BYTES", 10) + monkeypatch.setattr(inside, "issues_grants_to", lambda store, sandbox_type: False) + env = _deployed_env(base, driver, computer) + + result = await env.load_file_artifact_universe(_files(tmp_path, **{"a.txt": b"aaaa", "b.txt": b"bbbb", + "sub/big.bin": b"x" * 20})) + + assert result.destination_path == "Downloads" + assert result.files == {"a.txt": "Downloads/a.txt", "b.txt": "Downloads/b.txt", "sub/big.bin": "Downloads/sub/big.bin"} + batches = [sorted(marker["path"] for marker in markers) for kind, markers, _ in computer.calls] + assert sorted(batches) == [["Downloads/a.txt", "Downloads/b.txt"], ["Downloads/sub/big.bin"]] + assert (computer.home / "Downloads/sub/big.bin").read_bytes() == b"x" * 20 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("network, granted", [ + (None, True), ({"mode": "allowlist", "allow_hosts": ["example.com"]}, False), ({"mode": "allowlist"}, False), +], ids=["open", "allowlist", "closed"]) +async def test_a_load_reads_through_a_grant_that_reaches_the_computer_unless_it_has_no_network( + base, driver, docker, computer, tmp_path, monkeypatch, network, granted): + grants = [] + monkeypatch.setattr(inside, "issues_grants_to", lambda store, sandbox_type: sandbox_type == "local") + monkeypatch.setattr(inside, "read_object", lambda store, url: grants.append(url) or SimpleNamespace( + size_bytes=4, model_dump=lambda mode: {"read": {"url": f"https://store.test/{len(grants)}"}})) + env = _deployed_env(base, driver, computer, network=network) + + await env.load_file_artifact_universe(_files(tmp_path, **{"a.txt": b"aaaa"}), destination_path="Documents/in") + + ((_, (marker,), _),) = computer.calls + assert marker["path"] == "Documents/in/a.txt" + assert ("read" in marker, len(grants)) == ((True, 1) if granted else (False, 0)) + assert (computer.home / "Documents/in/a.txt").exists() is not granted + + +@pytest.mark.asyncio +@pytest.mark.parametrize("destination, problem", [ + ("/home/user/Documents", "a computer's destination_path is relative to its session user's home"), + ("Documents/../..", "must not contain '..' segments"), +]) +async def test_a_load_outside_the_home_is_refused_before_anything_is_sent(base, driver, docker, computer, tmp_path, + destination, problem): + env = _deployed_env(base, driver, computer) + + with pytest.raises(ValueError, match=problem): + await env.load_file_artifact_universe(_files(tmp_path, **{"a.txt": b"aaaa"}), destination_path=destination) + + assert computer.calls == [] + + +MAIL_SETTINGS = {"properties": { + "signature": {"default": "", "type": "string"}, + "password": {"default": "", "type": "string", "format": "password", "writeOnly": True, "minLength": 32}, + "token": {"anyOf": [{"type": "string", "format": "password", "writeOnly": True}, {"type": "null"}], "default": None}, + "theme": {"enum": ["light", "dark"], "type": "string", "default": "light"}}, "type": "object"} + + +def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, **changes): + """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements.""" + folder.mkdir(parents=True, exist_ok=True) + (folder / "app.py").write_text("class Mail: ...\n") + (folder / "requirements.txt").write_text("agentenv-framework-protocol\n") + files = FileArtifactUniverse.put_bundled("@local/~/kit/mail__files", files={ + name: folder / name for name in ("app.py", "requirements.txt")}) + declaration = AppDeclaration.model_validate({ + "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], + "permissions": [], "launch": None, "settings": settings, "skill": None, + "data": [], "install_steps": [], "ready_checks": [], + "install": {"linux": install or {"python": "requirements.txt", "services": [ + {"name": "imap", "command": "venv/bin/python imap.py"}]}}}) + return ComputerAppEnv.put(**{"id": "@local/~/kit/mail", "environment_name": "mail", "module": "app.py:Mail", + "files": files, "declaration": declaration, "computer": base, "driver": driver, + "compose": compose, **changes}) + + +def test_an_app_deployed_alone_is_composed_with_its_files_venv_and_programs_under_its_prefix(base, driver, docker, + tmp_path): + app = _app(base, driver, tmp_path / "mail") + + assert sorted(build["platform"] for build in docker.builds) == ["linux/amd64", "linux/arm64"] + build = docker.builds[0] + assert "apps/mail/app.py" in build["files"] and "apps/mail/requirements.txt" in build["files"] + dockerfile = build["dockerfile"] + for line in ("COPY apps/mail/ /opt/agentenv/apps/mail/", + "RUN python3 -m venv /opt/agentenv/apps/mail/venv && /opt/agentenv/apps/mail/venv/bin/pip install " + "--no-cache-dir -r /opt/agentenv/apps/mail/requirements.txt", + "COPY agentenv-app-mail.conf /etc/supervisor/conf.d/agentenv-app-mail.conf"): + assert line in dockerfile + assert dockerfile.index("COPY driver/") < dockerfile.index("COPY apps/mail/") < dockerfile.index("USER user") + assert build["confs"]["agentenv-app-mail.conf"] == ( + "[program:app-mail-imap]\ncommand=/opt/agentenv/apps/mail/venv/bin/python imap.py\n" + "directory=/opt/agentenv/apps/mail\nenvironment=ENVIRONMENT_NAME=\"mail\"\nautorestart=true\n" + "redirect_stderr=true\nstdout_logfile=/var/log/agentenv/app-mail-imap.log\n") + stored = Env.get(app.id) + assert isinstance(stored, ComputerAppEnv) and stored.build_statuses() == {"container/amd64": "unverified", + "container/arm64": "ready"} + + +def test_an_app_is_composed_only_for_the_arches_it_supports(base, driver, docker, tmp_path): + _app(base, driver, tmp_path / "mail", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["arm64"]}]) + + assert [build["platform"] for build in docker.builds] == ["linux/arm64"] + + +@pytest.mark.parametrize("changes, problem", [ + ({"supports": [{"os": {"type": "windows"}, "arch": ["amd64"]}]}, "app: @local/~/kit/mail supports no platform"), + ({"supports": [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["riscv64"]}]}, + "app: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and @local/~/kit/mail all run"), + ({"install": {"services": [{"name": "imap", "command": "a"}, {"name": "imap", "command": "b"}]}}, + "install.linux.services: Value error, names a service twice"), + ({"driver": None}, "a computer app deploys alone on a computer and a driver; name both or neither"), +], ids=["platform", "arch", "install", "no-driver"]) +def test_an_app_that_doesnt_fit_its_computer_and_driver_is_refused_before_anything_is_stored(base, driver, docker, + tmp_path, changes, problem): + with pytest.raises(ValueError, match=problem.replace(".", "[.]")): + _app(base, changes.pop("driver", driver), tmp_path / "mail", **changes) + + with pytest.raises(NotFoundError): + Env.get("@local/~/kit/mail") + assert docker.builds == [] + + +@pytest.mark.asyncio +async def test_an_app_deployed_alone_is_named_after_it_and_handed_its_settings_a_secret_resolved( + base, driver, docker, boots, tmp_path, monkeypatch): + monkeypatch.setenv("MAIL_PASSWORD", "s3cret") + monkeypatch.setenv("MAIL_TOKEN", "t0ken") + boots.display = (1280, 800) + app = _app(base, driver, tmp_path / "mail") + settings = {"mail": {"signature": "-- Ada", "password": "env:MAIL_PASSWORD", "token": "env:MAIL_TOKEN"}} + + deployed = await Env.get(app.id).deploy(env_params={"app_settings": settings}) + + environment = boots.boots[0]["environment"] + assert environment["ENVIRONMENT_NAME"] == "mail" and deployed.environment_card["name"] == "mail" + assert json.loads(environment["AGENTENV_APP_SETTINGS"]) == {"mail": {"signature": "-- Ada", "password": "s3cret", + "token": "t0ken"}} + assert deployed.env_id == app.id and deployed.metadata["computer"]["env_params"] == {"app_settings": settings} + + +def test_a_secrets_reference_meets_no_constraint_on_its_value_and_an_optional_one_may_be_null(base, driver, docker, + tmp_path): + app = _app(base, driver, tmp_path / "mail") + env_params = {"app_settings": {"mail": {"password": "env:MAIL_PASSWORD", "token": None}}} + + assert DeployEnvTaskStep(id="d", version=None, env_id=app.id, env_params=env_params).preflight() == [] + + +def test_a_required_secret_left_out_is_named_and_a_secrets_value_never_reaches_a_message(base, driver, docker, + tmp_path): + settings = {**MAIL_SETTINGS, "required": ["password", "signature"]} + app = _app(base, driver, tmp_path / "mail", settings=settings) + + left_out = DeployEnvTaskStep(id="d", version=None, env_id=app.id, + env_params={"app_settings": {"mail": {"signature": "-- Ada"}}}).preflight() + given = DeployEnvTaskStep(id="d", version=None, env_id=app.id, env_params={"app_settings": {"mail": { + "signature": "-- Ada", "password": "hunter2"}}}).preflight() + + assert len(left_out) == 1 and "env_params.app_settings.mail: 'password' is a required property" in left_out[0] + assert len(given) == 1 and "password is a secret" in given[0] and "hunter2" not in given[0] + + +@pytest.mark.asyncio +async def test_a_load_whose_connection_to_the_driver_breaks_says_so(base, driver, docker, computer, tmp_path, + monkeypatch): + monkeypatch.setattr(inside, "issues_grants_to", lambda store, sandbox_type: False) + computer.add_fails = httpx.ReadError("") + env = _deployed_env(base, driver, computer) + + with pytest.raises(RuntimeError, match=re.escape( + f"env '{env.id}': the connection to its driver broke during data/add, so some of the files it carried " + "may be written, each whole (ReadError)")): + await env.load_file_artifact_universe(_files(tmp_path, **{"a.txt": b"aaaa"})) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("values, problem", [ + ({"colour": "blue"}, "env_params.app_settings.mail.colour: @local/~/kit/mail declares no setting 'colour'"), + ({"signature": 3}, "env_params.app_settings.mail.signature: 3 is not of type 'string'"), + ({"theme": "blue"}, "env_params.app_settings.mail.theme: 'blue' is not one of ['light', 'dark']"), + ({"password": "hunter2"}, "env_params.app_settings.mail.password is a secret, so it takes a secret:KEY"), + ({"token": "hunter2"}, "env_params.app_settings.mail.token is a secret, so it takes a secret:KEY"), +], ids=["unknown", "type", "enum", "secret-value", "optional-secret-value"]) +async def test_app_settings_an_app_doesnt_declare_are_refused_at_save_and_at_deploy(base, driver, docker, boots, + tmp_path, values, problem): + app = _app(base, driver, tmp_path / "mail") + env_params = {"app_settings": {"mail": values}} + + saved = DeployEnvTaskStep(id="d", version=None, env_id=app.id, env_params=env_params).preflight() + with pytest.raises(DeployRefused, match=re.escape(problem)): + await Env.get(app.id).deploy(env_params=env_params) + + assert len(saved) == 1 and problem in saved[0] and boots.boots == [] diff --git a/tst/unit/serialization/golden/env/computer_app.json b/tst/unit/serialization/golden/env/computer_app.json new file mode 100644 index 000000000..109098b92 --- /dev/null +++ b/tst/unit/serialization/golden/env/computer_app.json @@ -0,0 +1,78 @@ +{ + "computer": { + "id": "ubuntu-24.04", + "version": 1 + }, + "declaration": { + "data": [ + "data/add", + "data/get" + ], + "install": { + "linux": { + "files": ".", + "python": "requirements.txt", + "services": [ + { + "command": "venv/bin/python serve.py", + "name": "notes" + } + ] + } + }, + "install_steps": [], + "launch": { + "at": "demand", + "command": [ + "notes" + ] + }, + "permissions": [], + "ready_checks": [], + "settings": { + "properties": { + "signature": { + "default": "", + "title": "Signature", + "type": "string" + } + }, + "title": "Settings", + "type": "object" + }, + "skill": { + "file": "skill.md" + }, + "supports": [ + { + "arch": [ + "amd64" + ], + "os": { + "display": [ + "x11" + ], + "type": "linux" + } + } + ] + }, + "driver": { + "id": "reference-driver", + "version": 4 + }, + "environment_name": "notes", + "files": { + "id": "tutorial-files", + "type": "file_artifact_universe", + "version": 4 + }, + "id": "notes", + "metadata": { + "owner": "example-team" + }, + "module": "app.py:Notes", + "service_name": "notes", + "type": "computer_app", + "version": 3 +} diff --git a/tst/unit/serialization/test_golden_wire_format.py b/tst/unit/serialization/test_golden_wire_format.py index f0cbe123c..802d442a2 100644 --- a/tst/unit/serialization/test_golden_wire_format.py +++ b/tst/unit/serialization/test_golden_wire_format.py @@ -27,6 +27,7 @@ from pathlib import Path import pytest +from agentenv_protocol.computer import AppDeclaration, DriverDeclaration from agent_env.artifact.artifacts.cli import CliArtifact from agent_env.artifact.artifacts.computer import ComputerArtifact @@ -39,8 +40,9 @@ from agent_env.artifact.ref import ArtifactRef from agent_env.artifact.registry import get_artifact_registry from agent_env.env.envs.computer.env import ComputerEnv -from agent_env.env.envs.computer.install import Install, Service -from agent_env.env.envs.computer_driver import ComputerDriverEnv +from agent_env.env.envs.computer.app_env import ComputerAppEnv +from agent_env.env.envs.computer.utils.install import Install, Service +from agent_env.env.envs.computer.driver_env import ComputerDriverEnv from agent_env.env.envs.gateway_server import GatewayEnv from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.env.envs.multi_env import MultiEnv @@ -173,16 +175,35 @@ def _website_env() -> WebsiteEnv: "installed": {"mss": "10.1.0"}, "built_at": "2026-10-28T10:02:11+00:00"}}}, "computer": {"id": "desktop__computer", "version": 1}}, ), + "computer_app": lambda: ComputerAppEnv( + id="notes", + version=3, + environment_name="notes", + module="app.py:Notes", + files=ARTIFACT_FIXTURES["file_artifact_universe"](), + declaration=AppDeclaration.model_validate({ + "supports": [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}], "permissions": [], + "install": {"linux": {"files": ".", "python": "requirements.txt", + "services": [{"name": "notes", "command": "venv/bin/python serve.py"}]}}, + "launch": {"at": "demand", "command": ["notes"]}, + "settings": {"properties": {"signature": {"default": "", "title": "Signature", "type": "string"}}, + "title": "Settings", "type": "object"}, + "skill": {"file": "skill.md"}, "data": ["data/add", "data/get"], "install_steps": [], "ready_checks": []}), + computer=ARTIFACT_FIXTURES["computer"]().model_copy(update={"version": 1}), + driver=ENV_FIXTURES["computer_driver"](), + metadata={"owner": "example-team"}, + ), "computer_driver": lambda: ComputerDriverEnv( id="reference-driver", version=4, module="driver.py:ReferenceDriver", files=ARTIFACT_FIXTURES["file_artifact_universe"](), - declaration={"name": "reference", "supports": [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}], - "platforms": [{"platform": {"os": "linux", "arch": "amd64", "display": "x11"}, - "provides": ["urn:agentenv:computer/v1", "urn:agentenv:computer-ready/v1"], - "tools": ["computer_screenshot", "computer_click"], - "features": {"buttons": ["left"], "max_click_count": 1}}]}, + declaration=DriverDeclaration.model_validate({ + "name": "reference", "supports": [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64"]}], + "platforms": [{"platform": {"os": "linux", "arch": "amd64", "display": "x11"}, + "provides": ["urn:agentenv:computer/v1", "urn:agentenv:computer-ready/v1"], + "tools": ["computer_screenshot", "computer_click"], + "features": {"buttons": ["left"], "max_click_count": 1}}]}), install={"linux": Install(python="requirements.txt", services=[Service( name="driver", command="venv/bin/python -m agentenv_protocol.computer.serve driver.py:ReferenceDriver")])}, port=18765, diff --git a/tst/unit/store/derived_ids_test.py b/tst/unit/store/derived_ids_test.py index 20c6c97e6..9992e0c64 100644 --- a/tst/unit/store/derived_ids_test.py +++ b/tst/unit/store/derived_ids_test.py @@ -238,6 +238,8 @@ def test_an_env_snapshot_step_and_a_capture_series_in_one_run_snapshot_one_env_i class _VerifierSandbox: + type = None + def scoped_name(self, name): return name diff --git a/tst/unit/store/fakes.py b/tst/unit/store/fakes.py index d5fa4250e..a308272e2 100644 --- a/tst/unit/store/fakes.py +++ b/tst/unit/store/fakes.py @@ -248,6 +248,7 @@ class CollectingVm: otherwise.""" mode = "vm" + type = None def scoped_name(self, name): return name diff --git a/tst/unit/task_step/test_load_artifact_service_universe.py b/tst/unit/task_step/test_load_artifact_service_universe.py index 0e6c4d0ca..a4757eed1 100644 --- a/tst/unit/task_step/test_load_artifact_service_universe.py +++ b/tst/unit/task_step/test_load_artifact_service_universe.py @@ -12,6 +12,8 @@ from __future__ import annotations +import asyncio +import re from types import SimpleNamespace import pytest @@ -20,7 +22,9 @@ from agent_env.artifact.artifacts.file import FileArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.artifact.artifacts.environment_universe import EnvironmentUniverseArtifact -from agent_env.env.env import DeployedEnv, DeployedGatewayEnv +from agent_env.env.env import DeployedEnv, DeployedGatewayEnv, Env +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.mcp_server import MCPServerEnv from agent_env.task_step.context import DeployedAgent, DeployedSandbox, TaskStepContext from agent_env.task_step.task_steps import load_artifact as load_artifact_mod from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep @@ -198,3 +202,22 @@ async def _async_none(*a, **kw): class _FakeProvider: async def get_sandbox(self, sandbox_id): return SimpleNamespace(scoped_name=lambda name: name) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("env_class, problems", [ + (ComputerEnv, ["load_artifact 'load': computer env 'desk' loads no environment_universe artifacts, so not " + f"'{UNIVERSE_ID}'"]), + (MCPServerEnv, []), +], ids=["computer", "mcp-server"]) +async def test_an_env_whose_type_loads_no_environment_artifacts_is_refused_at_save_and_at_run(universe, monkeypatch, + env_class, problems): + env = env_class.__new__(env_class) + monkeypatch.setattr(Env, "get", classmethod(lambda cls, id, version=None: env)) + monkeypatch.setattr(env_class, "from_deployed_env", classmethod(lambda cls, deployed: asyncio.sleep(0, result=env))) + step = LoadArtifactTaskStep(id="load", version=None, env_id="desk", artifact_id=UNIVERSE_ID) + + assert step.preflight() == problems + if problems: + with pytest.raises(RuntimeError, match=re.escape(problems[0])): + await step.execute(TaskStepContext(deployed_envs=[_deployed_env("desk")])) diff --git a/tst/unit/task_step/test_run_code.py b/tst/unit/task_step/test_run_code.py index 1c7d30fae..3d48575a8 100644 --- a/tst/unit/task_step/test_run_code.py +++ b/tst/unit/task_step/test_run_code.py @@ -7,6 +7,7 @@ import ast import json +import re import signal import subprocess import sys @@ -15,6 +16,7 @@ from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from agent_env.artifact import FileArtifact @@ -24,6 +26,7 @@ from agent_env.providers.sandbox_providers.sandbox import Sandbox, VmSandbox from agent_env.task_step.context import DeployedAgent, DeployedSandbox, TaskStepContext from agent_env.task_step.task_steps import run_code as _run_code_module +from tst.unit.computer_home import CARD, Computer from tst.unit.event_loop_probe import on_event_loop from agent_env.task_step.task_steps.run_code import ( RunCodeExecutionError, @@ -1289,3 +1292,94 @@ async def test_a_directory_named_after_a_stdlib_module_still_runs(tmp_path): assert ctx.metadata["script_results"]["r"] == {"shadowed": True} + + +@pytest.fixture +def computer(tmp_path, monkeypatch): + (tmp_path / "home").mkdir() + return Computer(tmp_path / "home").install(monkeypatch) + + +async def _run_in(computer, body, deployed=None, **step): + """Execute a run_code step for the computer's env with a script of ``body``.""" + ctx = TaskStepContext(deployed_envs=[deployed or computer.deployed()]) + with patch("agent_env.artifact.Artifact.get", return_value=_file_artifact(body=body)), \ + patch.object(FileArtifact, "load", _load_body): + return await _step(env_id="desk", **step).execute(ctx) + + +@pytest.mark.asyncio +async def test_a_script_for_a_computer_runs_inside_it_and_reaches_its_driver_on_loopback(computer): + body = b"import os\ndef run(inp):\n return {'env': inp['env'], 'cwd': os.getcwd()}\n" + + ctx = await _run_in(computer, body) + + result = ctx.metadata["script_results"]["run-code-1"] + assert result["env"] == {"id": "desk", "url": "http://127.0.0.1:18765", "mcp_url": "http://127.0.0.1:18765/mcp"} + assert Path(result["cwd"]).resolve() == computer.home.resolve() + run_dir = ".agentenv/run_code/run-code-1" + assert [call[1] for call in computer.calls if call[0] == "data/add"] == [ + [{"path": f"{run_dir}/script.py"}], [{"path": f"{run_dir}/input.json"}], [{"path": f"{run_dir}/runner.py"}]] + (script,) = [params for kind, params, *_ in computer.calls if kind == "run" and params["argv"][0] == "python3"] + assert script["argv"] == ["python3", f"{run_dir}/runner.py", run_dir, "run", "script.py", "600"] + assert script["timeout_s"] == 610 + assert ("data/get", [f"{run_dir}/output.json"]) in computer.calls + assert not (computer.home / run_dir).exists() + + +@pytest.mark.asyncio +async def test_a_failed_script_in_a_computer_is_reported_and_leaves_no_run_folder(computer): + with pytest.raises(RunCodeExecutionError, match="(?s)exited 1: .*ValueError: no"): + await _run_in(computer, b"def run(inp):\n raise ValueError('no')\n") + + assert not (computer.home / ".agentenv/run_code/run-code-1").exists() + + +@pytest.mark.asyncio +async def test_a_run_the_computer_stopped_at_its_timeout_is_reported_as_one(computer, monkeypatch): + monkeypatch.setattr(computer, "_run", lambda params: {"exit_code": None, "stdout": "", "stderr": "", + "truncated": False, "timed_out": True} + if params["argv"][0] == "python3" else Computer._run(computer, params)) + + with pytest.raises(RunCodeExecutionError, match="timed out after 30s"): + await _run_in(computer, _DEFAULT_BODY, timeout_seconds=30) + + +@pytest.mark.asyncio +async def test_a_run_whose_call_drops_fails_naming_why_and_the_next_one_runs_afresh(computer, monkeypatch): + invoke, dropped = computer.invoke_extension, [] + + async def drop_once(base_url, card, uri, params=None, timeout=30, verify=True, *, method=None): + if params["argv"][0] == "python3" and not dropped: + dropped.append(params["request_id"]) + raise httpx.RemoteProtocolError("the driver went away") + return await invoke(base_url, card, uri, params, timeout, verify, method=method) + + monkeypatch.setattr("agentenv_protocol.client.invoke_extension", drop_once) + with pytest.raises(RuntimeError, match=re.escape( + "env 'desk': the connection to its driver broke during computer-exec/v1 run (RemoteProtocolError: the " + "driver went away)")): + await _run_in(computer, _DEFAULT_BODY, id="read.v1") + await _run_in(computer, _DEFAULT_BODY, id="read.v1") + + runs = [params for kind, params, *_ in computer.calls if kind == "run"] + staged = [params["argv"] for params in runs if params["argv"][0] == "pkill"] + assert staged == [["pkill", "-f", r"\.agentenv/run_code/read\.v1/runner\.py"]] * 2 + assert not re.search(staged[0][2], "python3 /home/u/.agentenv/run_code/read-v1/runner.py") + scripts = [params["request_id"] for params in runs if params["argv"][0] == "python3"] + assert len(scripts) == 1 and scripts[0] != dropped[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("card, step, problem", [ + ({**CARD, "capabilities": {**CARD["capabilities"], "operations": ["data/get"]}}, {}, + "env 'desk' serves computer-exec/v1 but not data/add, which running code in it needs"), + (CARD, {"timeout_seconds": 3591}, "timeout_seconds 3591 is above the 3590 s a script in env 'desk' may run: it " + "runs a command for at most 3600 s"), +], ids=["no-data-add", "timeout"]) +async def test_a_computer_run_its_driver_cant_carry_out_is_refused_before_anything_is_staged(computer, card, step, + problem): + with pytest.raises(RunCodeValidationError, match=problem): + await _run_in(computer, _DEFAULT_BODY, deployed=computer.deployed(card=card), **step) + + assert computer.calls == [] diff --git a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py index 6a6d362a5..e93918f9d 100644 --- a/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py +++ b/tst/unit/task_step/verifiers/run_container_unit_tests_verifier_test.py @@ -16,6 +16,7 @@ from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import ( RunContainerUnitTestsVerifierTaskStep as Step, ) +from tst.unit.computer_home import Computer from tst.unit.event_loop_probe import on_event_loop @@ -63,6 +64,11 @@ def test_reward_path_roundtrips(): assert restored.reward_path == "/logs/verifier/reward.txt" +def test_a_verifier_writes_its_user_only_when_set_so_an_older_release_reads_root(): + assert "user" not in _step().to_dict() + assert Step.from_dict(_step(user="tester").to_dict()).to_dict()["user"] == "tester" + + def test_default_reward_path_none(): step = Step(id="s", version=None, sandbox_name="h", container_name="c", command="x") assert step.reward_path is None @@ -208,6 +214,8 @@ def test_posix_override_and_stored_env_keys_are_accepted(): class _Sandbox: + type = None + def scoped_name(self, name): return name @@ -390,3 +398,58 @@ def test_only_the_reward_file_is_cleared(local_stores, monkeypatch, reward_path) assert recorded["extracted_files"]["/work/events.tail.json"] == {"seq": 7} assert ("/logs/verifier/reward.txt" in sandbox.files) is (reward_path is None) + + +@pytest.mark.parametrize(("answer", "expected_score"), [("42", 1.0), ("41", 0.0)], ids=["correct", "wrong"]) +def test_a_verifier_for_a_computer_runs_in_it_and_grades_only_the_reward_this_run_wrote(local_stores, monkeypatch, + tmp_path, answer, + expected_score): + computer = Computer(tmp_path).install(monkeypatch) + (tmp_path / "logs").mkdir() + (tmp_path / "logs" / "reward.txt").write_text("1\n") + monkeypatch.setattr(Step, "_upload_text_artifact", staticmethod( + lambda text, artifact_id, description, object_url: SimpleNamespace(id=artifact_id, version=1, + object_url=object_url))) + step = Step(id="scrape", version=None, env_id="desk", setup_commands=["touch setup-ran"], + env_vars={"ANSWER": answer}, reward_path="logs/reward.txt", result_paths=["out.json"], + command='echo \'{"n": 1}\' > out.json; [ "$ANSWER" = 42 ] && echo 1 > logs/reward.txt; true') + ctx = TaskStepContext(deployed_envs=[computer.deployed()]) + + asyncio.run(step.execute(ctx)) + + recorded = ctx.metadata["verifications"]["scrape"] + assert (recorded["score"], recorded["exit_code"]) == (expected_score, 0) + assert recorded["extracted_files"]["out.json"] == {"n": 1} and (tmp_path / "setup-ran").exists() + runs = [call[1] for call in computer.calls if call[0] == "run"] + assert [params["argv"][:2] for params in runs] == [["bash", "-c"], ["rm", "-f"], ["bash", "-c"]] + assert {params["timeout_s"] for params in runs} == {300} and runs[-1]["env"]["ANSWER"] == answer + + +def test_a_verifier_for_a_computer_that_cant_clear_the_reward_file_grades_nothing(local_stores, monkeypatch, + tmp_path): + computer = Computer(tmp_path).install(monkeypatch) + denied = {"exit_code": 1, "stdout": "", "stderr": "rm: cannot remove 'logs/reward.txt': Permission denied", + "truncated": False, "timed_out": False} + monkeypatch.setattr(computer, "_run", lambda params: denied if params["argv"][0] == "rm" else Computer._run( + computer, params)) + step = Step(id="scrape", version=None, env_id="desk", reward_path="logs/reward.txt", command="exit 1") + ctx = TaskStepContext(deployed_envs=[computer.deployed()]) + + with pytest.raises(RuntimeError, match="couldn't clear the reward file 'logs/reward.txt' before the tests, so one " + "left there would be graded: rm: cannot remove"): + asyncio.run(step.execute(ctx)) + + assert "verifications" not in ctx.metadata + assert [call[1]["argv"][:2] for call in computer.calls if call[0] == "run"] == [["rm", "-f"]] + + +@pytest.mark.parametrize("fields, problem", [ + ({"env_id": "desk", "sandbox_name": "h", "container_name": "c"}, "set env_id, or sandbox_name and container_name"), + ({"sandbox_name": "h"}, "set env_id, or sandbox_name and container_name"), + ({"env_id": "desk", "user": "root"}, "commands in env 'desk' run as its computer's session user; leave user unset"), + ({"env_id": "desk", "reward_path": "/logs/reward.txt"}, "relative to its session user's home, not " + "\\['/logs/reward.txt'\\]"), +], ids=["both-targets", "half-a-container", "user", "absolute-path"]) +def test_a_verifier_naming_its_target_wrong_is_refused(fields, problem): + with pytest.raises(ValueError, match=problem): + Step(id="scrape", version=None, command="true", **fields) diff --git a/tst/unit/task_step/verifiers/test_run_context.py b/tst/unit/task_step/verifiers/test_run_context.py index 6403e5900..f6f542fd4 100644 --- a/tst/unit/task_step/verifiers/test_run_context.py +++ b/tst/unit/task_step/verifiers/test_run_context.py @@ -403,6 +403,20 @@ async def test_without_a_prompt_id_the_runs_last_prompt_is_graded(monkeypatch): assert context.metadata["verifications"]["v"]["results"][0]["turns"] == 3 +@pytest.mark.asyncio +async def test_a_verify_asking_for_env_url_is_handed_the_envs_address(monkeypatch): + context, _, _ = recorded_run() + context.deployed_envs.append(SimpleNamespace(env_id="calendar", mcp_url="http://127.0.0.1:20001/mcp", + environment_url="http://127.0.0.1:20001")) + script = (b"async def verify(mcp_url, env_url, run=None):\n" + b" return [{'id': 'c', 'score': 1.0, 'env_url': env_url, 'turns': len(run.turns)}]\n") + + await _outcome(script, monkeypatch, prompt_id="p1").execute(context) + + [row] = context.metadata["verifications"]["v"]["results"] + assert (row["env_url"], row["turns"]) == ("http://127.0.0.1:20001", 3) + + @pytest.mark.asyncio async def test_a_verify_with_an_unrelated_second_parameter_is_called_as_before(monkeypatch): context, _, _ = recorded_run() diff --git a/uv.lock b/uv.lock index f81c157f2..89f990baf 100644 --- a/uv.lock +++ b/uv.lock @@ -55,6 +55,7 @@ dependencies = [ { name = "e2b" }, { name = "fastapi" }, { name = "httpx" }, + { name = "jsonschema" }, { name = "litellm" }, { name = "mcp" }, { name = "modal" }, @@ -178,6 +179,7 @@ requires-dist = [ { name = "griffelib", marker = "extra == 'dev'", specifier = "==2.3.0" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "httpx2", marker = "extra == 'vercel'", specifier = ">=2.13.1,<3" }, + { name = "jsonschema", specifier = ">=4.20" }, { name = "litellm", specifier = ">=1.96.0,<1.97" }, { name = "mcp", specifier = ">=1.25,<2" }, { name = "modal", specifier = ">=1.5.2.dev22" },