From c807a8eb76949d1ae3272724fe99fec66a12e1e6 Mon Sep 17 00:00:00 2001 From: William Loo Date: Thu, 29 Oct 2026 22:41:47 -0700 Subject: [PATCH] feat(website): build a site's server image from source or a registry, and verify its state at /go (FD-4683) agent-env env website put takes the server image from --server-dockerfile, stored with its build context as an a2a agent's image is, from --server-image-ref, or from GitHub, with --health-path; a separate frontend image is optional and warned of. The website_state_verifier step reads each checked session of a website env's sites, or a multi env's, at the site's /go?sid= through /svc//, the verifier endpoint CUA-Gym sites serve, and checks dot paths of its current_state, initial_state or state_diff; with env_version pinned, put and preflight refuse checks of sites that version doesn't serve. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 2 +- src/agent_env/cli/env/website.py | 160 +++++++------ src/agent_env/env/envs/website.py | 2 + src/agent_env/task/episode/rules.py | 2 +- src/agent_env/task_step/registry.py | 2 + .../task_steps/verifiers/__init__.py | 2 + .../verifiers/website_state_verifier.py | 217 ++++++++++++++++++ tst/unit/cli/environment_name_alias_test.py | 1 - tst/unit/cli/website_server_image_cli_test.py | 70 ++++++ .../test_website_state_verifier_step.py | 132 +++++++++++ 10 files changed, 518 insertions(+), 72 deletions(-) create mode 100644 src/agent_env/task_step/task_steps/verifiers/website_state_verifier.py create mode 100644 tst/unit/cli/website_server_image_cli_test.py create mode 100644 tst/unit/task_step/verifiers/test_website_state_verifier_step.py diff --git a/AGENTS.md b/AGENTS.md index df782b4e1..037d582c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -114,7 +114,7 @@ and deserialized through a registry. |---|---| | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, checked at its `health` path; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]`: files, a `python` requirements file for a venv, and the services the base's init runs; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile and build context so any env with the same inputs reuses it; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `website_state_verifier`, which reads each checked session of a deployed website env's sites, or a multi env's, at the site's `/go?sid=` through `/svc//` and checks one dot path of its `current_state`, `initial_state` or `state_diff` each, optionally reduced by `len`, `keys` or `bool`, with `eq`, `ne`, `ge`, `gt`, `le`, `lt`, `in` or `contains`, its sites checked before a run against the `env_version` it pins, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/cli/env/website.py b/src/agent_env/cli/env/website.py index 9045413b7..c2ee06e66 100644 --- a/src/agent_env/cli/env/website.py +++ b/src/agent_env/cli/env/website.py @@ -18,6 +18,7 @@ from agent_env.utils.card_naming import card_name_from_github, card_name_from_source from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, build_image from agent_env.env.envs.website import WebsiteEnv +from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_HEALTH_PATH from agent_env.store.ids import derive_id, image_repository @@ -30,14 +31,24 @@ def website(): @website.command() @click.option("--id", "env_id", required=True, help="Env id") -@click.option("--backend-dockerfile", default=None, type=click.Path(exists=True), help="Path to backend Dockerfile") -@click.option("--backend-docker-context", default=None, type=click.Path(exists=True), help="Backend Docker build context (defaults to Dockerfile's directory)") -@click.option("--backend-dockerfile-github-url", default=None, help="GitHub URL to backend Dockerfile") -@click.option("--backend-docker-context-github-url", default=None, help="GitHub URL to backend build context directory") -@click.option("--frontend-dockerfile", default=None, type=click.Path(exists=True), help="Path to frontend Dockerfile") +@click.option("--server-dockerfile", "--backend-dockerfile", "backend_dockerfile", default=None, type=click.Path(exists=True), + help="Path to the Dockerfile of the server image, which serves the site's pages and API") +@click.option("--server-docker-context", "--backend-docker-context", "backend_docker_context", default=None, + type=click.Path(exists=True), help="The server image's build context (defaults to its Dockerfile's directory)") +@click.option("--server-dockerfile-github-url", "--backend-dockerfile-github-url", "backend_dockerfile_github_url", + default=None, help="GitHub URL to the server image's Dockerfile") +@click.option("--server-docker-context-github-url", "--backend-docker-context-github-url", + "backend_docker_context_github_url", default=None, help="GitHub URL to the server image's build context directory") +@click.option("--server-image-ref", "server_image_ref", default=None, + help="A server image already in a registry, e.g. ghcr.io/org/site:v1: registered as it is, with its tag " + "pinned to the digest the registry serves now (instead of a Dockerfile)") +@click.option("--frontend-dockerfile", default=None, type=click.Path(exists=True), + help="Path to a separate frontend Dockerfile (deprecated: build the pages into the server image)") @click.option("--frontend-docker-context", default=None, type=click.Path(exists=True), help="Frontend Docker build context (defaults to Dockerfile's directory)") -@click.option("--frontend-dockerfile-github-url", default=None, help="GitHub URL to frontend Dockerfile") +@click.option("--frontend-dockerfile-github-url", default=None, help="GitHub URL to a separate frontend Dockerfile (deprecated)") @click.option("--frontend-docker-context-github-url", default=None, help="GitHub URL to frontend build context directory") +@click.option("--health-path", default=AGENT_ENV_WEBSITE_HEALTH_PATH, show_default=True, + help="The path that answers 200 once the site is usable") @environment_name_options @env_provider_type_option("What deploys the env: 'gateway' (a gateway in front of the website), or the type of an installed " "agent_env.env_providers plugin", "website") @@ -50,36 +61,41 @@ def put( backend_docker_context: str | None, backend_dockerfile_github_url: str | None, backend_docker_context_github_url: str | None, + server_image_ref: str | None, frontend_dockerfile: str | None, frontend_docker_context: str | None, frontend_dockerfile_github_url: str | None, frontend_docker_context_github_url: str | None, + health_path: str, environment_name: str | None, env_provider_type: str, metadata_pairs: tuple[str, ...], skip_validation: bool, build_platform: str, ): - """Build and upload a website environment (frontend + backend).""" + """Build and upload a website environment: its server image, which serves the site's pages and API, from a + Dockerfile, GitHub or a registry, and any deprecated separate frontend image.""" environment_name = resolve_environment_name(environment_name, allow_missing=True) - # Validate mutual exclusion - if backend_dockerfile and backend_dockerfile_github_url: - click.echo("Error: --backend-dockerfile and --backend-dockerfile-github-url are mutually exclusive", err=True) - sys.exit(1) - if not backend_dockerfile and not backend_dockerfile_github_url: - click.echo("Error: either --backend-dockerfile or --backend-dockerfile-github-url is required", err=True) + if sum(map(bool, (backend_dockerfile, backend_dockerfile_github_url, server_image_ref))) != 1: + click.echo("Error: exactly one of --server-dockerfile, --server-dockerfile-github-url and --server-image-ref is " + "required", err=True) sys.exit(1) if frontend_dockerfile and frontend_dockerfile_github_url: click.echo("Error: --frontend-dockerfile and --frontend-dockerfile-github-url are mutually exclusive", err=True) sys.exit(1) - if not frontend_dockerfile and not frontend_dockerfile_github_url: - click.echo("Error: either --frontend-dockerfile or --frontend-dockerfile-github-url is required", err=True) - sys.exit(1) use_github = bool(backend_dockerfile_github_url) - if use_github != bool(frontend_dockerfile_github_url): - click.echo("Error: backend and frontend must both use local Dockerfiles or both use GitHub URLs", err=True) + if (frontend_dockerfile and not backend_dockerfile) or (frontend_dockerfile_github_url and not use_github): + click.echo("Error: a frontend image is built the way the server image is, both from local Dockerfiles or both " + "from GitHub URLs", err=True) sys.exit(1) + if server_image_ref and not environment_name: + click.echo("Error: --server-image-ref needs --environment-name, which a registry image's card can't give here", + err=True) + sys.exit(1) + if frontend_dockerfile or frontend_dockerfile_github_url: + click.echo("Warning: a separate frontend image is deprecated; build the site's pages into its server image", + err=True) user_metadata = {} for pair in metadata_pairs: @@ -88,12 +104,13 @@ def put( sys.exit(1) key, value = pair.split("=", 1) user_metadata[key] = value + site = {"health_path": health_path} if use_github: if environment_name is None: environment_name = card_name_from_github(backend_dockerfile_github_url, backend_docker_context_github_url, github_token=os.environ.get("GITHUB_TOKEN")) if not environment_name: - click.echo("Error: could not read @environment_card(name=...) from the backend GitHub source; pass --environment-name.", err=True) + click.echo("Error: could not read @environment_card(name=...) from the server image's GitHub source; pass --environment-name.", err=True) sys.exit(1) click.echo(f"Derived environment_name={environment_name!r} from the environment card.") if build_platform != DEFAULT_BUILD_PLATFORM: @@ -105,7 +122,7 @@ def put( def _redraw(): if lines_printed[0] > 0: click.echo(f"\033[{lines_printed[0]}A", nl=False) - click.echo(f"\033[2K {click.style('Backend:', fg='cyan')} {backend_status[0]}") + click.echo(f"\033[2K {click.style('Server:', fg='cyan')} {backend_status[0]}") click.echo(f"\033[2K {click.style('Frontend:', fg='magenta')} {frontend_status[0]}") lines_printed[0] = 2 @@ -121,7 +138,7 @@ def _frontend_progress(step: str, message: str, percent: int) -> None: id=env_id, backend_dockerfile_github_url=backend_dockerfile_github_url, backend_docker_context_github_url=backend_docker_context_github_url, - frontend_dockerfile_github_url=frontend_dockerfile_github_url, + frontend_dockerfile_github_url=frontend_dockerfile_github_url or "", frontend_docker_context_github_url=frontend_docker_context_github_url, environment_name=environment_name, metadata=user_metadata if user_metadata else None, @@ -129,6 +146,7 @@ def _frontend_progress(step: str, message: str, percent: int) -> None: on_frontend_progress=_frontend_progress, github_token=os.environ.get("GITHUB_TOKEN"), env_provider_type=env_provider_type, + **site, )) click.echo(f"Created WebsiteEnv: id={env.id} version={env.version} environment_name={env.environment_name}") if not skip_validation: @@ -137,64 +155,68 @@ def _frontend_progress(step: str, message: str, percent: int) -> None: click.echo(f"Validation task: {instance_id}") return - # Local Docker build path - backend_dockerfile_path = Path(backend_dockerfile) - backend_ctx = Path(backend_docker_context) if backend_docker_context else backend_dockerfile_path.parent backend_id, frontend_id = derive_id(env_id, "backend_image"), derive_id(env_id, "frontend_image") - refuse_unwritable_ids(env_id, backend_id, frontend_id) - backend_tag = image_repository(backend_id) - if environment_name is None: - environment_name = card_name_from_source(str(backend_dockerfile_path), str(backend_ctx)) - if not environment_name: - click.echo("Error: no @environment_card(name=...) found in the backend source; pass --environment-name.", err=True) - sys.exit(1) - click.echo(f"Derived environment_name={environment_name!r} from the environment card.") - - click.echo("Building website backend Docker image...") - build_image(backend_dockerfile_path, backend_ctx, backend_tag, platform=build_platform) - - click.echo("Creating backend DockerImageArtifact...") - backend_artifact = DockerImageArtifact.put( - id=backend_id, - description="Website backend image created from agent-env CLI", - image_name=backend_tag, - ) - click.echo(f"Created backend artifact: id={backend_artifact.id} version={backend_artifact.version}") - - frontend_dockerfile_path = Path(frontend_dockerfile) - frontend_ctx = Path(frontend_docker_context) if frontend_docker_context else frontend_dockerfile_path.parent - frontend_tag = image_repository(frontend_id) - - click.echo("Building website frontend Docker image...") - build_image(frontend_dockerfile_path, frontend_ctx, frontend_tag, platform=build_platform) - - click.echo("Creating frontend DockerImageArtifact...") - frontend_artifact = DockerImageArtifact.put( - id=frontend_id, - description="Website frontend image created from agent-env CLI", - image_name=frontend_tag, - ) - click.echo(f"Created frontend artifact: id={frontend_artifact.id} version={frontend_artifact.version}") - - metadata = detect_env_metadata(backend_dockerfile_path, backend_ctx) + refuse_unwritable_ids(env_id, backend_id, *([frontend_id] if frontend_dockerfile else [])) + metadata: dict[str, str] = {} + if server_image_ref: + if build_platform != DEFAULT_BUILD_PLATFORM: + click.echo(f"Warning: --platform {build_platform!r} is ignored for --server-image-ref, which builds nothing", err=True) + backend_artifact = DockerImageArtifact.put_ref(backend_id, description=f"Website server image for {env_id}", + image_name=server_image_ref) + click.echo(f"Registered server image: id={backend_artifact.id} version={backend_artifact.version} image={backend_artifact.image_name}") + else: + backend_dockerfile_path = Path(backend_dockerfile) + backend_ctx = Path(backend_docker_context) if backend_docker_context else backend_dockerfile_path.parent + if environment_name is None: + environment_name = card_name_from_source(str(backend_dockerfile_path), str(backend_ctx)) + if not environment_name: + click.echo("Error: no @environment_card(name=...) found in the server image's source; pass --environment-name.", err=True) + sys.exit(1) + click.echo(f"Derived environment_name={environment_name!r} from the environment card.") + backend_artifact = _build(backend_id, backend_dockerfile_path, backend_ctx, "server", build_platform) + metadata = detect_env_metadata(backend_dockerfile_path, backend_ctx) + + frontend_artifact = None + if frontend_dockerfile: + frontend_dockerfile_path = Path(frontend_dockerfile) + frontend_ctx = Path(frontend_docker_context) if frontend_docker_context else frontend_dockerfile_path.parent + frontend_artifact = _build(frontend_id, frontend_dockerfile_path, frontend_ctx, "frontend", build_platform) metadata.update(user_metadata) click.echo("Creating WebsiteEnv...") - env = WebsiteEnv.put( - id=env_id, - backend_docker_image_artifact=backend_artifact, - frontend_docker_image_artifact=frontend_artifact, - environment_name=environment_name, - metadata=metadata if metadata else None, - env_provider_type=env_provider_type, - ) - click.echo(f"Created WebsiteEnv: id={env.id} version={env.version} environment_name={env.environment_name} env_provider_type={env.env_provider_type}") + try: + env = WebsiteEnv.put( + id=env_id, + server_docker_image_artifact=backend_artifact, + frontend_docker_image_artifact=frontend_artifact, + environment_name=environment_name, + metadata=metadata if metadata else None, + env_provider_type=env_provider_type, + **site, + ) + except ValueError as e: + raise click.ClickException(str(e)) from e + click.echo(f"Created WebsiteEnv: id={env.id} version={env.version} environment_name={env.environment_name} " + f"env_provider_type={env.env_provider_type}") if not skip_validation: click.echo("\nValidating environment (use --skip-validation to skip)...") instance_id = asyncio.run(env.validate(on_progress=click.echo)) click.echo(f"Validation task: {instance_id}") +def _build(image_id: str, dockerfile: Path, context: Path, role: str, platform: str) -> DockerImageArtifact: + """Build ``dockerfile`` into the image ``image_id`` and store it with its build context, as an a2a agent's image + is stored.""" + tag = image_repository(image_id) + click.echo(f"Building website {role} Docker image...") + build_image(dockerfile, context, tag, platform=platform) + artifact = DockerImageArtifact.put(id=image_id, description=f"Website {role} image created from agent-env CLI", + image_name=tag, build_context_path=str(context), dockerfile_path=str(dockerfile), + platform=platform) + click.echo(f"Created {role} artifact: id={artifact.id} version={artifact.version}") + return artifact + + @website.command(name="load-environment-artifact") @click.option("--id", "env_id", default=None, help="WebsiteEnv id (optional; cross-checked against the instance)") @click.option("--environment-artifact-id", "environment_artifact_id", diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index b87fc9418..ee26a3508 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -315,6 +315,7 @@ async def put_from_github( on_frontend_progress: ProgressCallback | None = None, github_token: str | None = None, env_provider_type: str = "gateway", + health_path: str = DEFAULT_HEALTH_PATH, ) -> WebsiteEnv: """Build the server image, from ``backend_dockerfile_github_url``, and any deprecated frontend image from GitHub in parallel, and create a WebsiteEnv.""" @@ -363,6 +364,7 @@ async def put_from_github( environment_name=environment_name, metadata=combined_metadata, env_provider_type=env_provider_type, + health_path=health_path, ) return env diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index 1719ac289..f72c473db 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -151,7 +151,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "deploy_sandbox", "end_computer_recording", "start_computer_recording", "env_outcome_verifier", "modify_env_tool_access", "reset_env", "review", "run_container_unit_tests_verifier", "run_docker_container", "snapshot_env", "sync_env_clock", "validation_gate_aggregator", "verify_env_card", "verify_env_core_protocol", "verify_mcp_tool_schema", - "verify_spec_conformance", "verify_universe_load_export_roundtrip", + "verify_spec_conformance", "verify_universe_load_export_roundtrip", "website_state_verifier", # They read only what an outside turn leaves: its reply. "agent_prompt_response_verifier", "verify_mcp_env_assessment", # It skips an agent that isn't deployed. diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index d0c32afca..70332c0a4 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -49,6 +49,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep + from agent_env.task_step.task_steps.verifiers.website_state_verifier import WebsiteStateVerifierTaskStep from agent_env.task_step.task_steps.run_docker_container import RunDockerContainerTaskStep from agent_env.task_step.task_steps.mcp_env_validator.verify_mcp_tool_schema import VerifyMCPToolSchemaTaskStep from agent_env.task_step.task_steps.mcp_env_validator.verify_spec_conformance import VerifySpecConformanceTaskStep @@ -103,6 +104,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(InstallAgentTaskStep): InstallAgentTaskStep, _get_type(RubricsVerifierTaskStep): RubricsVerifierTaskStep, _get_type(RunContainerUnitTestsVerifierTaskStep): RunContainerUnitTestsVerifierTaskStep, + _get_type(WebsiteStateVerifierTaskStep): WebsiteStateVerifierTaskStep, _get_type(RunDockerContainerTaskStep): RunDockerContainerTaskStep, _get_type(VerifyMCPToolSchemaTaskStep): VerifyMCPToolSchemaTaskStep, _get_type(VerifySpecConformanceTaskStep): VerifySpecConformanceTaskStep, diff --git a/src/agent_env/task_step/task_steps/verifiers/__init__.py b/src/agent_env/task_step/task_steps/verifiers/__init__.py index 781c29c39..e3e284145 100644 --- a/src/agent_env/task_step/task_steps/verifiers/__init__.py +++ b/src/agent_env/task_step/task_steps/verifiers/__init__.py @@ -10,6 +10,7 @@ from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep +from agent_env.task_step.task_steps.verifiers.website_state_verifier import WebsiteStateVerifierTaskStep from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep __all__ = [ @@ -26,5 +27,6 @@ "ScoreAggregator", "Turn", "VerifySandboxTaskStep", + "WebsiteStateVerifierTaskStep", "aggregate_score", ] diff --git a/src/agent_env/task_step/task_steps/verifiers/website_state_verifier.py b/src/agent_env/task_step/task_steps/verifiers/website_state_verifier.py new file mode 100644 index 000000000..2546c9141 --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/website_state_verifier.py @@ -0,0 +1,217 @@ +"""Verify the state of deployed websites, a website env or a multi env's: each check reads one value from one session +of one site, as the site's /go?sid= reports it ({initial_state, current_state, state_diff}, the verifier +endpoint CUA-Gym sites serve), reached through the gateway's /svc//, and compares it with an expected +value.""" + +from __future__ import annotations + +import logging +import re +import uuid + +import httpx +from dataclasses import asdict, dataclass +from typing import Any, ClassVar, Optional + +from agent_env.entity.refs import EntityRef +from agent_env.env import legacy_protocol +from agent_env.env.env import Env, gateway_url_of +from agent_env.store.base import NotFoundError +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep, TaskStepDependency, attach_retry_config +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score + +logger = logging.getLogger(__name__) + +SOURCES = ("current_state", "initial_state", "state_diff") +OPS = ("eq", "ne", "ge", "gt", "le", "lt", "in", "contains") +REDUCES = ("len", "keys", "bool") +DEFAULT_SID = "default" +GO_TIMEOUT_S = 30 + + +@dataclass(frozen=True) +class StateCheck: + """One check: ``op`` of the value at ``path`` (dot-separated, a key's ``.`` and ``\\`` escaped with ``\\``, a list + index by number; empty for the whole source) in session ``sid`` of ``site``, a website env's environment_name, + reduced by ``reduce`` first when given, and ``value``. A path that isn't there reads as None.""" + + site: str + path: str + op: str + value: Any = None + sid: str = DEFAULT_SID + source: str = "current_state" + reduce: Optional[str] = None + weight: float = 1.0 + + def problems(self) -> list[str]: + named = f"check of {self.site!r} {self.path!r}" + return ([f"{named}: op {self.op!r} is none of {', '.join(OPS)}"] if self.op not in OPS else []) + ( + [f"{named}: source {self.source!r} is none of {', '.join(SOURCES)}"] if self.source not in SOURCES else []) + ( + [f"{named}: reduce {self.reduce!r} is none of {', '.join(REDUCES)}"] + if self.reduce is not None and self.reduce not in REDUCES else []) + + def result(self, sessions: dict) -> dict: + """The check's result row against ``sessions``, each site's ``/go`` answer by its sid.""" + session = sessions.get(self.site, {}).get(self.sid) + actual = None if session is None else _read(session.get(self.source), self.path, self.source == "state_diff") + if self.reduce is not None: + actual = _reduce(actual, self.reduce) + try: + passed = _compare(actual, self.op, self.value) + except TypeError: + passed = False + return {"criterion": f"{self.site}[{self.sid}].{self.source}{'.' + self.path if self.path else ''}" + f"{' |' + self.reduce if self.reduce else ''} {self.op} {self.value!r}", + "result": passed, "actual": actual, "weight": self.weight, + **({} if session is not None else {"detail": f"{self.site!r} has no session {self.sid!r}"})} + + +def _read(source: Any, path: str, whole_keys: bool) -> Any: + if not path: + return source + if whole_keys: + return source.get(path) if isinstance(source, dict) else None + value = source + for part in _parts(path): + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list) and re.fullmatch(r"-?\d+", part) and -len(value) <= int(part) < len(value): + value = value[int(part)] + else: + return None + return value + + +def _parts(path: str) -> list[str]: + """``path``'s keys, split at each ``.`` that no ``\\`` escapes, as the runtime escapes a key's ``.`` and ``\\``.""" + parts, part, chars = [], [], iter(path) + for char in chars: + if char == "\\": + part.append(next(chars, "")) + elif char == ".": + parts.append("".join(part)) + part = [] + else: + part.append(char) + return [*parts, "".join(part)] + + +def _reduce(value: Any, how: str) -> Any: + if how == "bool": + return bool(value) + if how == "keys": + return sorted(value) if isinstance(value, dict) else [] + return len(value) if isinstance(value, (dict, list, str)) else 0 + + +def _compare(actual: Any, op: str, expected: Any) -> bool: + if op == "eq": + return actual == expected + if op == "ne": + return actual != expected + if op == "in": + return actual in expected + if op == "contains": + return actual is not None and expected in actual + if actual is None or expected is None: + return False + return {"ge": actual >= expected, "gt": actual > expected, "le": actual <= expected, "lt": actual < expected}[op] + + +class WebsiteStateVerifierTaskStep(TaskStep): + """Runs ``checks`` against the state the websites of the deployed env ``env_id``, a website env or a multi env, + report at ``/go?sid=``, one result row each, and scores them with ``score_aggregator``. The sites checks name are + checked before a run against ``env_version``, the version the task deploys, when it's given.""" + + type: ClassVar[str] = "website_state_verifier" + entity_refs = (EntityRef.env("env_id", version_field="env_version"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, checks: list[StateCheck | dict], + score_aggregator: Optional[ScoreAggregator | str] = None, verifier_id: Optional[str] = None, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + env_version: Optional[int] = None): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id = env_id + self.env_version = env_version + self.checks = [check if isinstance(check, StateCheck) else StateCheck(**check) for check in checks] + self.score_aggregator = ScoreAggregator(score_aggregator) if isinstance(score_aggregator, str) else ( + score_aggregator or ScoreAggregator.ALL_PASS) + self.verifier_id = verifier_id or uuid.uuid4().hex + if problems := [problem for check in self.checks for problem in check.problems()]: + raise ValueError(f"website_state_verifier {id!r}: " + "; ".join(problems)) + + def to_dict(self) -> dict: + return {**super().to_dict(), "env_id": self.env_id, "env_version": self.env_version, + "checks": [asdict(check) for check in self.checks], "score_aggregator": self.score_aggregator.value, + "verifier_id": self.verifier_id} + + @classmethod + def from_dict(cls, data: dict) -> WebsiteStateVerifierTaskStep: + return cls(**cls._base_from_dict(data), env_id=data["env_id"], env_version=data.get("env_version"), + checks=data["checks"], score_aggregator=data.get("score_aggregator"), + verifier_id=data.get("verifier_id")) + + def preflight(self) -> list[str]: + """The env ``env_id`` when it holds no website, and the sites checks name that it doesn't serve, read from + ``env_version``; with no ``env_version`` the sites are left to the run, which reads the version it deploys.""" + try: + env = Env.get(self.env_id, self.env_version) + except NotFoundError: + return [] + if not (sites := _sites(env)): + return [f"website_state_verifier {self.id!r}: {self.env_id} is a {env.type} env with no website; name a " + "website env or a multi env of websites"] + if self.env_version is None: + return [] + if unknown := sorted({check.site for check in self.checks} - sites): + return [f"website_state_verifier {self.id!r}: {', '.join(map(repr, unknown))} " + f"{'is not a site' if len(unknown) == 1 else 'are not sites'} of {self.env_id} " + f"v{self.env_version}, whose sites are {sorted(sites)}"] + return [] + + @classmethod + def put(cls, **kwargs: Any) -> WebsiteStateVerifierTaskStep: + """Store the step once its preflight passes. Refused, naming every problem, before anything is stored.""" + from agent_env.task_step.store import get_task_step_store + + kwargs.setdefault("version", None) + retry_config = kwargs.pop("retry_config", None) + step = cls(**kwargs) + if problems := step.preflight(): + raise ValueError("; ".join(problems)) + if retry_config is not None: + attach_retry_config(step, {"retry_config": retry_config}) + return get_task_step_store().put_document(step) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) + if deployed is None: + raise RuntimeError(f"Env '{self.env_id}' not found in context.deployed_envs") + sites = _sites(Env.get(deployed.env_id, deployed.env_version)) + gateway_url = gateway_url_of(deployed) + sessions: dict[str, dict] = {} + async with httpx.AsyncClient() as client: + for site, sid in sorted({(check.site, check.sid) for check in self.checks}): + if site not in sites: + raise RuntimeError(f"env '{self.env_id}' serves no website {site!r}; its sites are {sorted(sites)}") + base_url = legacy_protocol.environment_base_url(gateway_url, site, mcp=False) + response = await client.get(f"{base_url}/go", params={"sid": sid}, timeout=GO_TIMEOUT_S) + response.raise_for_status() + sessions.setdefault(site, {})[sid] = response.json() + results = [check.result(sessions) for check in self.checks] + score = aggregate_score(results, self.score_aggregator) + context.metadata.setdefault("verifications", {})[self.verifier_id] = {"results": results, "score": score} + logger.info(f"Verification '{self.verifier_id}' complete: {len(results)} checks, score={score}") + return context + + +def _sites(env: Env) -> set[str]: + """The environment_names of the website envs ``env`` is or holds.""" + from agent_env.env.envs.multi_env import MultiEnv + from agent_env.env.envs.website import WebsiteEnv + + if isinstance(env, WebsiteEnv): + return {env.environment_name} + return {child.environment_name for child in env.website_envs} if isinstance(env, MultiEnv) else set() diff --git a/tst/unit/cli/environment_name_alias_test.py b/tst/unit/cli/environment_name_alias_test.py index 9a598ac2b..6954b728b 100644 --- a/tst/unit/cli/environment_name_alias_test.py +++ b/tst/unit/cli/environment_name_alias_test.py @@ -28,7 +28,6 @@ [ "env", "website", "put", "--id", "x", "--backend-dockerfile-github-url", "https://github.com/o/r/tree/main/b/Dockerfile", - "--frontend-dockerfile-github-url", "https://github.com/o/r/tree/main/f/Dockerfile", "--skip-validation", ], "agent_env.cli.env.website.WebsiteEnv.put_from_github", diff --git a/tst/unit/cli/website_server_image_cli_test.py b/tst/unit/cli/website_server_image_cli_test.py new file mode 100644 index 000000000..37de89103 --- /dev/null +++ b/tst/unit/cli/website_server_image_cli_test.py @@ -0,0 +1,70 @@ +"""agent-env env website put: a site's server image, from a Dockerfile stored with its build context as an a2a +agent's is, or from a registry, checked at its health path; a separate frontend image is optional and warned of.""" + +from unittest.mock import patch + +from click.testing import CliRunner + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store + +from agent_env.cli.env.website import website +from agent_env.env import Env + + +def _stored(id, **_): + return get_artifact_store().put_document(DockerImageArtifact( + id=id, description="", image_name=f"registry.example/{id}@sha256:" + "a" * 64)) + + +def _put(*args): + return CliRunner().invoke(website, ["put", "--id", "mail-site", *args, "--skip-validation"]) + + +def test_a_site_is_built_from_its_server_dockerfile_alone_with_its_build_context(local_stores, tmp_path): + (tmp_path / "Dockerfile").write_text("FROM scratch\n") + built = [] + with patch("agent_env.cli.env.website.build_image", side_effect=lambda *a, **k: built.append(a[2])), \ + patch("agent_env.artifact.artifacts.docker_image.DockerImageArtifact.put", + side_effect=lambda **k: _stored(k["id"])) as stored: + result = _put("--server-dockerfile", str(tmp_path / "Dockerfile"), "--environment-name", "mail", + "--health-path", "/healthz") + assert result.exit_code == 0, result.output + assert len(built) == 1 and "deprecated" not in result.output + assert stored.call_args.kwargs["build_context_path"] == str(tmp_path) + assert stored.call_args.kwargs["dockerfile_path"] == str(tmp_path / "Dockerfile") + env = Env.get("mail-site") + assert (env.frontend_docker_image_artifact, env.health_path) == (None, "/healthz") + + +def test_a_site_registers_a_server_image_already_in_a_registry(local_stores): + with patch("agent_env.cli.env.website.DockerImageArtifact.put_ref", + side_effect=lambda id, **k: _stored(id)) as put_ref: + result = _put("--server-image-ref", "ghcr.io/acme/mail:3", "--environment-name", "mail") + assert result.exit_code == 0, result.output + assert put_ref.call_args.kwargs["image_name"] == "ghcr.io/acme/mail:3" + assert Env.get("mail-site").server_docker_image_artifact.id == "mail-site__backend_image" + + +def test_put_refuses_no_server_image_two_of_them_an_unnamed_registry_image_and_a_bad_health_path(local_stores, tmp_path): + (tmp_path / "Dockerfile").write_text("FROM scratch\n") + assert "exactly one of --server-dockerfile" in _put("--environment-name", "mail").output + assert "exactly one of" in _put("--server-dockerfile", str(tmp_path / "Dockerfile"), + "--server-image-ref", "ghcr.io/acme/mail:3").output + assert "needs --environment-name" in _put("--server-image-ref", "ghcr.io/acme/mail:3").output + with patch("agent_env.cli.env.website.DockerImageArtifact.put_ref", side_effect=lambda id, **k: _stored(id)): + refused = _put("--server-image-ref", "ghcr.io/acme/mail:3", "--environment-name", "mail", "--health-path", "ok") + assert refused.exit_code == 1 and "health: 'ok' is not a path" in refused.output + + +def test_a_separate_frontend_image_still_builds_and_is_warned_of(local_stores, tmp_path): + (tmp_path / "server.Dockerfile").write_text("FROM scratch\n") + (tmp_path / "frontend.Dockerfile").write_text("FROM scratch\n") + with patch("agent_env.cli.env.website.build_image"), \ + patch("agent_env.artifact.artifacts.docker_image.DockerImageArtifact.put", + side_effect=lambda **k: _stored(k["id"])): + result = _put("--backend-dockerfile", str(tmp_path / "server.Dockerfile"), "--frontend-dockerfile", + str(tmp_path / "frontend.Dockerfile"), "--environment-name", "mail") + assert result.exit_code == 0, result.output + assert "separate frontend image is deprecated" in result.output + assert Env.get("mail-site").frontend_docker_image_artifact.id == "mail-site__frontend_image" diff --git a/tst/unit/task_step/verifiers/test_website_state_verifier_step.py b/tst/unit/task_step/verifiers/test_website_state_verifier_step.py new file mode 100644 index 000000000..68cc4847c --- /dev/null +++ b/tst/unit/task_step/verifiers/test_website_state_verifier_step.py @@ -0,0 +1,132 @@ +"""The website state verifier: each check reads one value of one session of one site, as the site's /go?sid= reports it +through the gateway, and compares it; the step is refused at put for an op it doesn't know, and for a site the env +version it pins doesn't serve.""" + +import asyncio +from types import SimpleNamespace + +import httpx +import pytest + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.env.envs.multi_env import MultiEnv +from agent_env.env.envs.website import WebsiteEnv +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps.verifiers import website_state_verifier +from agent_env.task_step.task_steps.verifiers.website_state_verifier import StateCheck, WebsiteStateVerifierTaskStep + +APPS = {"mail": {"s1": { + "initial_state": {"inbox": [{"id": 1, "read": False}], "user": {"name": "Ada"}}, + "current_state": {"inbox": [{"id": 1, "read": True}, {"id": 2, "read": False}], "user": {"name": "Ada"}, + "drafts": {"d1": "hi"}, "contacts": {"ada@example.com": {"vip": True}}}, + "state_diff": {"inbox": {"initial": [{"id": 1, "read": False}], + "current": [{"id": 1, "read": True}, {"id": 2, "read": False}]}, + "drafts.d1": {"current": "hi"}}}}} + + +@pytest.mark.parametrize(("check", "passed"), [ + (StateCheck(site="mail", sid="s1", path="user.name", op="eq", value="Ada"), True), + (StateCheck(site="mail", sid="s1", path="inbox.0.read", op="eq", value=True), True), + (StateCheck(site="mail", sid="s1", path="inbox.-1.id", op="eq", value=2), True), + (StateCheck(site="mail", sid="s1", path="inbox", op="ge", value=2, reduce="len"), True), + (StateCheck(site="mail", sid="s1", path="inbox", op="eq", value=1, reduce="len", source="initial_state"), True), + (StateCheck(site="mail", sid="s1", path="drafts", op="eq", value=["d1"], reduce="keys"), True), + (StateCheck(site="mail", sid="s1", path="drafts.d1", op="contains", value="h", source="state_diff"), False), + (StateCheck(site="mail", sid="s1", path="drafts.d1", op="eq", value={"current": "hi"}, source="state_diff"), True), + (StateCheck(site="mail", sid="s1", path="", op="eq", value=2, reduce="len", source="state_diff"), True), + (StateCheck(site="mail", sid="s1", path="user.tz", op="eq", value=None), True), + (StateCheck(site="mail", sid="s1", path="user.tz", op="lt", value=3), False), + (StateCheck(site="mail", sid="s1", path="user.name", op="in", value=["Ada", "Grace"]), True), + (StateCheck(site="mail", sid="s1", path="user.name", op="gt", value=1), False), + (StateCheck(site="mail", sid="s1", path="inbox.5", op="ne", value=None), False), + (StateCheck(site="mail", sid="s1", path="inbox.--1", op="eq", value=None), True), + (StateCheck(site="mail", sid="s2", path="", op="eq", value=False, reduce="bool"), True), + (StateCheck(site="mail", sid="s1", path=r"contacts.ada@example\.com.vip", op="eq", value=True), True), + (StateCheck(site="mail", sid="s1", path="contacts.ada@example.com.vip", op="eq", value=None), True), +]) +def test_a_check_reads_its_value_and_compares_it(check, passed): + assert check.result(APPS)["result"] is passed + + +def test_a_check_of_a_session_with_no_state_says_so(): + row = StateCheck(site="mail", sid="s9", path="user.name", op="eq", value="Ada").result(APPS) + assert (row["result"], row["actual"], row["detail"]) == (False, None, "'mail' has no session 's9'") + assert row["criterion"] == "mail[s9].current_state.user.name eq 'Ada'" + + +def test_every_check_that_names_an_unknown_op_source_or_reduce_is_refused_at_once(): + with pytest.raises(ValueError) as refused: + WebsiteStateVerifierTaskStep(id="v", version=None, env_id="office", checks=[ + {"site": "mail", "path": "a", "op": "matches"}, + {"site": "mail", "path": "b", "op": "eq", "source": "final_state", "reduce": "sum"}]) + assert str(refused.value).count("check of 'mail'") == 3 + + +def test_the_step_round_trips_and_is_registered(): + step = WebsiteStateVerifierTaskStep(id="v", version=None, env_id="office", score_aggregator="weighted_average", + checks=[{"site": "mail", "sid": "s1", "path": "inbox", "op": "ge", "value": 2, + "reduce": "len", "weight": 2.0}]) + assert WebsiteStateVerifierTaskStep.from_dict(step.to_dict()).to_dict() == step.to_dict() + assert get_task_step_registry()["website_state_verifier"] is WebsiteStateVerifierTaskStep + + +def _site(name): + image = get_artifact_store().put_document(DockerImageArtifact( + id=f"{name}-server", description="", image_name=f"registry.example/{name}@sha256:" + "d" * 64)) + return WebsiteEnv.put(id=f"{name}-site", server_docker_image_artifact=image, environment_name=name) + + +def test_put_refuses_a_check_of_a_site_the_pinned_env_version_doesnt_serve(local_stores): + mail, calendar = _site("mail"), _site("calendar") + MultiEnv.put(id="office", mcp_server_envs=[], website_envs=[mail, calendar]) + MultiEnv.put(id="office", mcp_server_envs=[], website_envs=[calendar]) + with pytest.raises(ValueError, match=r"'notes' is not a site of office v1, whose sites are \['calendar', 'mail'\]"): + WebsiteStateVerifierTaskStep.put(id="v", env_id="office", env_version=1, checks=[ + {"site": "notes", "path": "", "op": "eq"}, {"site": "mail", "path": "", "op": "eq"}]) + stored = WebsiteStateVerifierTaskStep.put(id="v", env_id="office", env_version=1, retry_config={ + "retry_from_step_id": "deploy"}, checks=[{"site": "mail", "path": "", "op": "eq"}]) + assert stored.version == 1 and stored.retry_config.retry_from_step_id == "deploy" + assert WebsiteStateVerifierTaskStep.from_dict(stored.to_dict()).env_version == 1 + pinned = WebsiteStateVerifierTaskStep(id="w", version=None, env_id="office", env_version=2, + checks=[{"site": "mail", "path": "", "op": "eq"}]) + assert pinned.preflight() == ["website_state_verifier 'w': 'mail' is not a site of office v2, whose sites are " + "['calendar']"] + assert WebsiteStateVerifierTaskStep(id="u", version=None, env_id="office", + checks=[{"site": "mail", "path": "", "op": "eq"}]).preflight() == [] + assert WebsiteStateVerifierTaskStep(id="s", version=None, env_id="mail-site", env_version=1, + checks=[{"site": "mail", "path": "", "op": "eq"}]).preflight() == [] + plain = MultiEnv.put(id="plain", mcp_server_envs=[]) + assert "with no website" in WebsiteStateVerifierTaskStep(id="y", version=None, env_id=plain.id, checks=[]).preflight()[0] + assert WebsiteStateVerifierTaskStep(id="x", version=None, env_id="missing", checks=[]).preflight() == [] + + +def test_execute_reads_each_sites_go_through_the_gateway_and_scores_the_checks(monkeypatch): + by_site = {"mail": APPS["mail"], "calendar": {"s1": {"current_state": {"events": [1, 2]}}}} + asked = [] + + def gateway(request): + site = request.url.path.split("/")[2] + sid = request.url.params["sid"] + asked.append((request.url.path, sid)) + return httpx.Response(200, json=by_site[site][sid]) + + real_client = httpx.AsyncClient + monkeypatch.setattr(website_state_verifier.httpx, "AsyncClient", + lambda **kw: real_client(transport=httpx.MockTransport(gateway), **kw)) + monkeypatch.setattr(website_state_verifier, "gateway_url_of", lambda deployed: "http://gateway") + monkeypatch.setattr(website_state_verifier, "_sites", lambda env: {"mail", "calendar"}) + monkeypatch.setattr(website_state_verifier.Env, "get", lambda id, version=None: None) + step = WebsiteStateVerifierTaskStep(id="v", version=None, env_id="office", verifier_id="office-check", checks=[ + {"site": "mail", "sid": "s1", "path": "inbox.0.read", "op": "eq", "value": True}, + {"site": "mail", "sid": "s1", "path": "drafts", "op": "eq", "value": 0, "reduce": "len"}, + {"site": "calendar", "sid": "s1", "path": "events", "op": "eq", "value": 2, "reduce": "len"}]) + context = TaskStepContext() + context.deployed_envs.append(SimpleNamespace(env_id="office", env_version=1)) + + asyncio.run(step.execute(context)) + + assert asked == [("/svc/calendar/go", "s1"), ("/svc/mail/go", "s1")] + verification = context.metadata["verifications"]["office-check"] + assert [row["result"] for row in verification["results"]] == [True, False, True]