Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/agent_env/artifact/artifacts/docker_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from agent_env.artifact.artifact import Artifact, _write_twin
from agent_env.store.ids import fs_safe, image_repository, is_local_id, local_image_repository
from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore
from agent_env.store.image_store.oci_registry_credentials import is_loopback_host, names_registry, registry_host_from_ref
from agent_env.store.image_store.oci_registry_credentials import image_reference_problem, is_loopback_host, names_registry, registry_host_from_ref
from agent_env.store.image_store.registry_api import NoSuchImageError, pin_digest, pinned_from
from agent_env.utils.docker.context import BuildContext
from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, build_image, normalize_platform
Expand Down Expand Up @@ -193,6 +193,8 @@ def put_tar(
owner = "DockerImageArtifact.put_tar"
if not tar_gz_object_url:
raise ValueError(f"{owner}(): tar_gz_object_url is empty")
if problem := image_reference_problem(image_name):
raise ValueError(f"{owner}(): {problem}")
for url in (tar_gz_object_url, build_context_object_url):
if url:
get_config().check_object_url(id, url)
Expand Down Expand Up @@ -223,6 +225,8 @@ def put_ref(cls, id: str, *, description: str, image_name: str, earlier_pin: str
from agent_env.artifact.store import get_artifact_store
from agent_env.config import get_config

if problem := image_reference_problem(image_name):
raise ValueError(problem)
if not names_registry(image_name):
raise ValueError(f"{image_name!r} doesn't name its registry; spell it out, as in "
f"docker.io/library/{image_name}")
Expand Down
5 changes: 3 additions & 2 deletions src/agent_env/bundle/authoring.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,9 @@
from agent_env.env.env import Env
from agent_env.env.envs._deployment import provider_refusal
from agent_env.providers.env_providers.constants import GATEWAY_SERVICE_NAMES
from agent_env.providers.env_providers.env_gateway_provider import EnvironmentGatewayProvider
from agent_env.providers.env_providers.env_gateway_provider import EnvironmentGatewayProvider, environment_name_problem
from agent_env.providers.env_providers.env_provider import _env_provider_class
from agent_env.utils.card_naming import card_names_in_files

from agent_env.utils.paths import os_reason

from ._fs import relative, show, with_article
Expand Down Expand Up @@ -174,6 +173,8 @@ def accepted_env(self, data: dict, env_class: type[Env], *,
problems.append(self.config_problem(problem))
elif name is not None:
fields["environment_name"] = name
if name_problem := environment_name_problem(name):
problems.append(self.config_problem(name_problem))
if (name in GATEWAY_SERVICE_NAMES and provider is not None
and issubclass(provider, EnvironmentGatewayProvider)):
problems.append(self.config_problem(
Expand Down
8 changes: 3 additions & 5 deletions src/agent_env/env/envs/multi_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import logging
import os
import shlex
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -635,10 +636,7 @@ async def _load_from_snapshot(self, snapshot) -> None:
compose_content = compose_content.replace(
" volumes:\n - ./init-schemas.sql:/docker-entrypoint-initdb.d/init-schemas.sql\n", "",
)
write_script = f'''cat > {DOCKER_COMPOSE_PATH} << 'COMPOSE_EOF'
{compose_content}
COMPOSE_EOF'''
await self._sandbox.exec_script(write_script)
await gw._write_file(self._sandbox, DOCKER_COMPOSE_PATH, compose_content)

# Remove old servicedb container and its anonymous volume, then start fresh from snapshot image
logger.info("Recreating servicedb container with snapshot image...")
Expand Down Expand Up @@ -668,7 +666,7 @@ async def _load_from_snapshot(self, snapshot) -> None:
environment_names = list(dict.fromkeys(
[env.environment_name for env in self.mcp_server_envs] + [env.environment_name for env in self.website_envs]
))
svc_list = " ".join(environment_names + [GATEWAY_SERVICE_NAME, PGWEB_SERVICE_NAME, DB_MCP_SERVICE_NAME])
svc_list = " ".join(map(shlex.quote, environment_names + [GATEWAY_SERVICE_NAME, PGWEB_SERVICE_NAME, DB_MCP_SERVICE_NAME]))
logger.info(f"Recreating services: {svc_list}")
await gw._compose_up(self._sandbox, f"--force-recreate {svc_list}")
await gw._wait_for_gateway(self._sandbox, self._sandbox.host_port(AGENT_ENV_GATEWAY_MCP_PORT))
Expand Down
50 changes: 39 additions & 11 deletions src/agent_env/providers/env_providers/env_gateway_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
Accepts,
SandboxProvider,
)
from agent_env.store.image_store.oci_registry_credentials import image_reference_problem

if TYPE_CHECKING:
from collections.abc import Awaitable, Callable
Expand Down Expand Up @@ -367,6 +368,11 @@ def create_docker_compose(
env-network:
driver: bridge
"""
problems = _compose_name_problems(mcp_servers, website_configs, sidecars, images=(gateway_image,))
if mcp_server_name and ("\n" in mcp_server_name or "\r" in mcp_server_name):
problems.append(f"MCP server name {mcp_server_name!r} must be one line")
if problems:
raise ValueError(f"Can't render the gateway's docker-compose.yml: {'; '.join(problems)}")
lines = ["services:"]

# MCP servers and website backends are both DB clients (one schema each).
Expand Down Expand Up @@ -587,6 +593,8 @@ async def deploy(
from agent_env.providers.env_state import LOCAL_POSTGRES_STATE_TYPE, acquire_state_for_deploy

topology = _gateway_topology(env)
if problems := _compose_name_problems(topology.mcp_servers, topology.website_configs):
raise ValueError(f"Can't deploy {env.id!r} behind a gateway: {'; '.join(problems)}")
state_instance = await acquire_state_for_deploy(
env_state_type=env_state_type, ttl_seconds=ttl_seconds, name_hint=env.id, env_state_instance_id=env_state_instance_id,
)
Expand Down Expand Up @@ -816,17 +824,9 @@ async def _stand_up(_spec):
if self._needs_local_postgres:
init_script = local_store_spec.init_sql
logger.info(f"Generated init-schemas.sql:\n{init_script}")
init_script_path = f"{GATEWAY_APP_DIR}/init-schemas.sql"
write_init_script = f'''cat > {init_script_path} << 'INIT_EOF'
{init_script}
INIT_EOF'''
await sandbox.exec_script(write_init_script, max_retries=2)

# Write docker-compose.yml
write_compose_script = f'''cat > {DOCKER_COMPOSE_PATH} << 'COMPOSE_EOF'
{compose_content}
COMPOSE_EOF'''
await sandbox.exec_script(write_compose_script, max_retries=2)
await self._write_file(sandbox, f"{GATEWAY_APP_DIR}/init-schemas.sql", init_script)

await self._write_file(sandbox, DOCKER_COMPOSE_PATH, compose_content)

# Start MCP services with docker compose
logger.info("Starting MCP services...")
Expand Down Expand Up @@ -1121,6 +1121,12 @@ async def read_trajectory(self, sandbox: VmSandbox) -> list[dict]:
events.append(json.loads(line))
return events

@staticmethod
async def _write_file(sandbox: VmSandbox, path: str, content: str) -> None:
"""Write ``content`` and a newline to ``path`` on the sandbox host, the content one quoted argument so no byte
of it is shell."""
await sandbox.exec_script(f"printf '%s\\n' {shlex.quote(content)} > {shlex.quote(path)}", max_retries=2)

async def _compose_up(self, sandbox: VmSandbox, args: str = "") -> None:
"""``docker compose up -d`` the stack, whose failure names each service that went down and ends with its
output, which compose's own error leaves out."""
Expand Down Expand Up @@ -1501,6 +1507,28 @@ def _gateway_topology(env: Env) -> _GatewayTopology:
)


# Compose's service-name characters, with a letter or digit first so no name reads as a command-line option.
_ENVIRONMENT_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")


def environment_name_problem(name: str) -> str | None:
"""Why ``name`` can't be an environment name, None when it can: a gateway deploy makes it a compose service, a
container hostname and a schema, so it may carry nothing else."""
if isinstance(name, str) and _ENVIRONMENT_NAME.fullmatch(name):
return None
return (f"environment name {name!r} must start with a letter or digit and hold only letters, digits, "
f"'_', '.' and '-'")


def _compose_name_problems(mcp_servers: list[MCPServerConfig], website_configs: list[WebsiteConfig] | None,
sidecars: list[SidecarConfig] | None = None, *, images: tuple[str, ...] = ()) -> list[str]:
"""Why the environment names and images of a gateway deploy can't go into its compose file and shell lines."""
names = [c.environment_name for c in [*mcp_servers, *(website_configs or []), *(sidecars or [])]]
refs = [*images, *(s.image for s in mcp_servers), *(s.image for s in sidecars or []),
*(ref for w in website_configs or [] for ref in (w.backend_image, w.frontend_image))]
return [problem for problem in [*map(environment_name_problem, names), *map(image_reference_problem, refs)] if problem]


# Compose `environment:` keys whose values are secrets and must be masked before the
# rendered compose is logged. Matches the key to the left of the first '=' on an
# ` - KEY=VALUE` env line.
Expand Down
14 changes: 11 additions & 3 deletions src/agent_env/providers/env_state/local_postgres.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
StateContext,
LOCAL_POSTGRES_STATE_TYPE,
)
from agent_env.store.image_store.oci_registry_credentials import image_reference_problem

if TYPE_CHECKING:
from collections.abc import Awaitable, Callable
Expand Down Expand Up @@ -234,7 +235,7 @@ def render_healthcheck_service(
hc = spec.healthcheck
return [
f" {self.env_state_docker_service_name()}:",
f" image: {cfg.db_image}",
f" image: {_checked_image(cfg.db_image)}",
" environment:",
*[f" - {key}={value}" for key, value in spec.env.items()],
" healthcheck:",
Expand Down Expand Up @@ -266,7 +267,7 @@ def render_sidecar_containers(
if cfg.db_web_image:
lines.extend([
f" {PGWEB_SERVICE_NAME}:",
f" image: {cfg.db_web_image}",
f" image: {_checked_image(cfg.db_web_image)}",
" restart: unless-stopped",
" depends_on:",
f" {DATABASE_SERVICE_NAME}:",
Expand All @@ -286,7 +287,7 @@ def render_sidecar_containers(
if cfg.db_mcp_image:
lines.extend([
f" {DB_MCP_SERVICE_NAME}:",
f" image: {cfg.db_mcp_image}",
f" image: {_checked_image(cfg.db_mcp_image)}",
" restart: unless-stopped",
" depends_on:",
f" {DATABASE_SERVICE_NAME}:",
Expand Down Expand Up @@ -394,3 +395,10 @@ async def install_changelog_triggers(
exit_code, _out, err = await store_exec(argv)
if exit_code != 0:
raise RuntimeError(f"Changelog trigger install failed for '{environment_name}': {err[-500:]}")


def _checked_image(ref: str) -> str:
"""``ref``, refused when it isn't an image reference: the compose file it goes into reads it as YAML."""
if problem := image_reference_problem(ref):
raise ValueError(f"Can't render the state store's compose services: {problem}")
return ref
21 changes: 21 additions & 0 deletions src/agent_env/store/image_store/oci_registry_credentials.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import binascii
import json
import logging
import re
from abc import ABC, abstractmethod
from collections.abc import Mapping
from dataclasses import dataclass, field
Expand Down Expand Up @@ -90,6 +91,26 @@ def registry_host_from_ref(ref: str) -> str | None:
return None


# Docker's reference grammar (distribution/reference): [registry/]path[:tag][@digest], without its bracketed IPv6
# registries, which an unquoted compose value reads as the start of a YAML list.
_DOMAIN_LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?"
_DOMAIN = rf"{_DOMAIN_LABEL}(?:\.{_DOMAIN_LABEL})*(?::[0-9]+)?"
_PATH_COMPONENT = r"[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*"
_TAG = r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}"
_DIGEST = r"[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*:[0-9A-Fa-f]{32,}"
_IMAGE_REFERENCE = re.compile(
rf"(?:{_DOMAIN}/)?{_PATH_COMPONENT}(?:/{_PATH_COMPONENT})*(?::{_TAG})?(?:@{_DIGEST})?"
)


def image_reference_problem(ref: str) -> str | None:
"""Why ``ref`` isn't an image reference Docker takes, None when it is: a compose ``image:`` line holds it
unquoted, so it may carry nothing else."""
if isinstance(ref, str) and _IMAGE_REFERENCE.fullmatch(ref):
return None
return f"image name {ref!r} isn't an image reference ([registry/]repository[:tag][@digest])"


def is_loopback_host(host: str | None) -> bool:
"""Whether a registry ``host``, with or without its port, names this machine: a loopback address, the
unspecified ``0.0.0.0``, or ``host.docker.internal``, which only this machine's containers resolve."""
Expand Down
6 changes: 6 additions & 0 deletions tst/unit/artifact/test_docker_image_pulled.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,3 +85,9 @@ def test_one_with_no_tar_gz_has_no_bytes_to_load():
def test_put_tar_refuses_an_empty_tar_gz():
with pytest.raises(ValueError, match="tar_gz_object_url is empty"):
DockerImageArtifact.put_tar("img", description="d", image_name="img:1", tar_gz_object_url="")


@pytest.mark.parametrize("name", ["img:1\n privileged: true", "img 1", ""])
def test_put_tar_refuses_a_name_that_isnt_an_image_reference(name):
with pytest.raises(ValueError, match="isn't an image reference"):
DockerImageArtifact.put_tar("img", description="d", image_name=name, tar_gz_object_url="s3://bucket/img.tar.gz")
10 changes: 10 additions & 0 deletions tst/unit/artifact/test_docker_image_put_ref.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,16 @@ def test_a_name_without_its_registry_is_refused_before_any_request(local_stores,
DockerImageArtifact.get("tool-image")


@pytest.mark.parametrize("ref", ["ghcr.io/org/tool:v1\n privileged: true", "ghcr.io/org/to ol:v1", "ghcr.io/org/Tool:v1"])
def test_a_name_that_isnt_an_image_reference_is_refused_before_any_request(local_stores, resolved, ref):
with pytest.raises(ValueError, match="isn't an image reference"):
DockerImageArtifact.put_ref("tool-image", description="d", image_name=ref)

assert resolved == []
with pytest.raises(NotFoundError):
DockerImageArtifact.get("tool-image")


@pytest.mark.parametrize("ref", ["localhost:5000/team/img:v1", "127.0.0.1:5000/img", "host.docker.internal:5000/img",
"0.0.0.0:5000/img"])
def test_a_registry_on_this_machine_is_refused_for_a_shared_store(local_stores, resolved, ref):
Expand Down
9 changes: 8 additions & 1 deletion tst/unit/bundle/env_toml_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,12 @@ def _website(name, toml=""):
"envs/x: env.toml: unknown key 'metadata'; a mcp_server env takes image, environment_name, env_provider_type, "
"type and id"),
({**_mcp("x"), "envs/x/env.toml": 'environment_name = ""\n'}, "envs/x: env.toml: environment_name can't be empty"),
({**_mcp("x"), "envs/x/env.toml": 'environment_name = "slack\\nCOMPOSE_EOF\\ntouch pwned"\n'},
"envs/x: env.toml: environment name 'slack\\nCOMPOSE_EOF\\ntouch pwned' must start with a letter or digit and "
"hold only letters, digits, '_', '.' and '-'"),
({**_mcp("x"), "envs/x/env.toml": 'environment_name = "a b"\n'},
"envs/x: env.toml: environment name 'a b' must start with a letter or digit and hold only letters, digits, "
"'_', '.' and '-'"),
({**_mcp("x"), "envs/x/env.toml": 'environment_name = "gateway"\n'},
"envs/x: env.toml: environment_name 'gateway' is one a gateway deploy names its own containers (db-mcp, gateway, "
"pgweb, servicedb, website-browser); choose another"),
Expand All @@ -213,7 +219,8 @@ def _website(name, toml=""):
({"envs/x/env.toml": 'type = "service_db"\n'},
"envs/x: a service_db env isn't written from a bundle: config names the one every deploy uses "
"(default_service_db_env_id), and a run builds it when it's missing"),
], ids=["no-card", "several-cards", "store-image-unnamed", "metadata", "empty-name", "reserved-name",
], ids=["no-card", "several-cards", "store-image-unnamed", "metadata", "empty-name", "name-escapes-its-line",
"name-with-a-space", "reserved-name",
"website-on-server", "build-args", "multi-without-envs", "multi-name", "multi-wrong-child", "multi-same-names",
"multi-same-container",
"gateway", "service-db"])
Expand Down
2 changes: 1 addition & 1 deletion tst/unit/env/envs/test_artifact_staging.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from agent_env.env.envs.mcp_server import MCPServerEnv
from agent_env.env.envs.website import WebsiteEnv
from agent_env.providers.sandbox_providers.local_sandbox import LocalSandbox
from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox

_SHELL_NAMES = [
("email", "x;touch pwned;#.json"),
Expand Down
2 changes: 1 addition & 1 deletion tst/unit/env/envs/test_mcp_server_container_mode.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ async def server_path(env, sandbox_provider, **kwargs):


def _env(env_provider_type: str = "gateway") -> MCPServerEnv:
return MCPServerEnv(id="mcp-email", version=1, docker_image_artifact=MagicMock(), environment_name="email",
return MCPServerEnv(id="mcp-email", version=1, docker_image_artifact=MagicMock(image_name="mcp-email"), environment_name="email",
env_provider_type=env_provider_type)


Expand Down
27 changes: 27 additions & 0 deletions tst/unit/env/envs/test_multi_env_snapshot_reload_name.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

import shlex
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
Expand All @@ -10,6 +11,7 @@
from agent_env.env.envs.multi_env import MultiEnv
from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT
from agent_env.providers.env_providers import EnvironmentGatewayProvider
from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH


class _Stop(Exception):
Expand Down Expand Up @@ -57,3 +59,28 @@ async def test_snapshot_reload_waits_for_the_gateway_where_it_is_published():
await env._load_from_snapshot(snapshot)

assert wait.call_args.args[1] == AGENT_ENV_GATEWAY_MCP_PORT + 1000


@pytest.mark.asyncio
async def test_snapshot_reload_writes_its_compose_as_one_argument_and_names_each_service_as_one():
env = MultiEnv(id="crm-suite", version=1, mcp_server_envs=[], name="crm")
env.mcp_server_envs = [MagicMock(environment_name="a b")]
env._sandbox = MagicMock(mode="vm", load_docker_images=AsyncMock(), exec_script=AsyncMock(return_value=""),
host_ips=("127.0.0.1",), host_port=lambda port: port)
db_image = MagicMock(spec=DockerImageArtifact, image_name="snap:1")
snapshot = MagicMock(db_image_artifact_id="snap", db_image_artifact_version=1)
compose = "services:\nCOMPOSE_EOF\ntouch pwned\n"

with patch("agent_env.artifact.Artifact.get", return_value=db_image), \
patch("agent_env.env.env.Env.get", return_value=MagicMock()), \
patch("agent_env.config.get_config", return_value=MagicMock(default_gateway_env_id="gw-id", default_service_db_env_id="db-id")), \
patch("agent_env.providers.env_state.LocalPostgresStateProvider"), \
patch.object(EnvironmentGatewayProvider, "create_docker_compose", return_value=compose), \
patch.object(EnvironmentGatewayProvider, "_wait_for_gateway", side_effect=_Stop):
with pytest.raises(_Stop):
await env._load_from_snapshot(snapshot)

scripts = [call.args[0] for call in env._sandbox.exec_script.await_args_list]
assert ["printf", "%s\\n", compose, ">", DOCKER_COMPOSE_PATH] in map(shlex.split, scripts)
assert not any("<<" in script for script in scripts)
assert any(script.endswith("--force-recreate 'a b' gateway pgweb db-mcp") for script in scripts)
7 changes: 5 additions & 2 deletions tst/unit/env/envs/test_multi_env_vm_sizing.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,11 @@


def _env(n_mcp: int = 0, n_web: int = 0) -> MultiEnv:
env = MultiEnv(id="e", version=1, mcp_server_envs=[MagicMock() for _ in range(n_mcp)])
env.website_envs = [MagicMock() for _ in range(n_web)]
env = MultiEnv(id="e", version=1, mcp_server_envs=[
MagicMock(environment_name=f"mcp{i}", **{"docker_image_artifact.image_name": f"mcp{i}"}) for i in range(n_mcp)])
env.website_envs = [MagicMock(environment_name=f"web{i}", **{"backend_docker_image_artifact.image_name": f"web{i}-be",
"frontend_docker_image_artifact.image_name": f"web{i}-fe"})
for i in range(n_web)]
return env


Expand Down
Loading
Loading