Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,8 @@ def __init__(self, environment_card: EnvironmentCard, handler: Any) -> None:
self._app: Any = None
ext_handlers = _served_extensions(handler, _discover_extensions(handler))
environment_card = _merge_extensions(environment_card, ext_handlers + _declared_extensions(handler, ext_handlers))
tool_handlers = _served_tools(handler, _with_forwarded(_discover_tools(handler, environment_card.name), handler))
tool_handlers = _named_tools(handler, _served_tools(
handler, _with_forwarded(_discover_tools(handler, environment_card.name), handler)))
if tool_handlers and not self.supports_tools:
raise ValueError(f"@tool methods require a FastMCP-backed application; {type(self).__name__} cannot serve MCP tools")
self._resources = _discover_resources(handler)
Expand Down Expand Up @@ -1107,11 +1108,15 @@ def read() -> Any:


def _instructions(handler: Any) -> str | None:
"""What the handler's @server_instructions method returns; None without one."""
"""What the handler's @server_instructions method returns, as its ``served_instructions`` serves it; None without
one."""
marked = [method for method, _marker in _marked_members(handler, _INSTRUCTIONS_ATTR)]
if len(marked) > 1:
raise ValueError(f"an environment has one @server_instructions method, not {sorted(m.__name__ for m in marked)}")
return marked[0]() if marked else None
if not marked:
return None
hook = getattr(handler, "served_instructions", None)
return hook(marked[0]()) if hook is not None else marked[0]()


def _discover_methods(handler: Any) -> dict:
Expand Down Expand Up @@ -1320,6 +1325,20 @@ def _served_tools(handler: Any, tool_handlers: list) -> list:
return [(descriptor, by_name[descriptor.name]) for descriptor in chosen]


def _named_tools(handler: Any, tool_handlers: list) -> list:
"""The tools as the handler's ``serve_tools_as`` names and describes them, in their order; as they are without one."""
hook = getattr(handler, "serve_tools_as", None)
if hook is None:
return tool_handlers
named = list(hook([descriptor for descriptor, _fn in tool_handlers]))
if len(named) != len(tool_handlers):
raise ValueError(f"serve_tools_as returned {len(named)} tools for {len(tool_handlers)}")
names = [descriptor.name for descriptor in named]
if len(set(names)) != len(names):
raise ValueError(f"serve_tools_as named two tools alike: {sorted({n for n in names if names.count(n) > 1})}")
return [(descriptor, fn) for descriptor, (_old, fn) in zip(named, tool_handlers)]


def _merge_tools(card: EnvironmentCard, tool_handlers: list) -> EnvironmentCard:
"""Advertise decorator-declared tools under card.capabilities; card-declared names win."""
if not tool_handlers:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
from .keys import PRINTABLE_KEYS
from .manifest import ManifestSegment, RecordingManifest
from .platform import Platform
from .settings import SETTINGS_ENV, ComputerSettings
from .settings import SETTINGS_ENV, TOOL_PREFIX_ENV, ComputerSettings, tool_prefix_problem

if TYPE_CHECKING:
from .card import (
Expand Down Expand Up @@ -127,7 +127,7 @@ def __getattr__(name: str) -> Any:
"Launch", "Ready", "Packages", "AptSource", "Download", "RunScript",
"AppDeclaration", "AppOs",
"APP_SETTINGS_ENV", "part_bytes",
"Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV",
"Display", "Image", "Effect", "ExecResult", "ReadyCheck", "Platform", "ComputerSettings", "SETTINGS_ENV", "TOOL_PREFIX_ENV", "tool_prefix_problem",
"Tab", "PageState", "PageDialog", "FileChooser",
"PRINTABLE_KEYS",
"display", "set_display", "capture", "click", "drag", "scroll", "type_text", "press",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import asyncio
import logging
import os
from pathlib import Path
from typing import Any, Callable, ClassVar, Literal, Optional

Expand All @@ -13,9 +14,9 @@
from starlette.responses import Response
from starlette.websockets import WebSocket

from ..agent_env_environment import (_CARD_CONFIG_ATTR, AgentEnvEnvironment, AgentEnvFastMCPApplication,
AgentEnvError, _error_response, add_websocket_route, control_check,
environment_card)
from ..agent_env_environment import (_CARD_CONFIG_ATTR, _TOOL_ATTR, AgentEnvEnvironment, AgentEnvFastMCPApplication,
AgentEnvError, _error_response, _marked_members, add_websocket_route,
control_check, environment_card)
from ..tool_context import ToolContext
from ..types import EnvironmentCard, EnvironmentExtension, EnvironmentTool
from . import _linux, apps, card, svc
Expand All @@ -27,8 +28,9 @@
from .platform import Platform, detect
from .recording import Recording
from .runtime import ComputerRuntime
from .settings import ComputerSettings, load_settings
from .tools import Tools
from .prefix import ToolPrefix
from .settings import TOOL_PREFIX_ENV, ComputerSettings, load_settings, tool_prefix_problem
from .tools import COMPUTER_TOOLS, Tools

#: The caller every call comes from until agents carry an identity of their own.
DEFAULT_IDENTITY = "default"
Expand Down Expand Up @@ -99,9 +101,15 @@ def __init_subclass__(cls, **kwargs: Any) -> None:
check_class(cls)
check_features(cls)

def __init__(self, settings: ComputerSettings | None = None, platform: Platform | None = None) -> None:
def __init__(self, settings: ComputerSettings | None = None, platform: Platform | None = None,
tool_prefix: str | None = None) -> None:
self.settings = settings if settings is not None else load_settings(model=type(self).Settings)
self.platform = platform if platform is not None else detect()
prefix = os.environ.get(TOOL_PREFIX_ENV, "") if tool_prefix is None else tool_prefix
if problem := tool_prefix_problem(prefix):
raise ValueError(problem)
#: The start of every name it serves a tool under: the deploy's ``AGENTENV_TOOL_PREFIX``, none by default.
self._computer_prefix = ToolPrefix(prefix)
self._computer = ComputerRuntime(self)
#: The routes each app's card opens to agents, by app, read once it answers.
self._computer_agent_routes: dict[str, list[str]] = {}
Expand All @@ -113,7 +121,7 @@ def declaration(cls, platform: Platform) -> EnvironmentCard:
running its ``__init__``: what a ``computer_driver`` env records for each platform it supports. The card
hooks may read the class and the base class's state, not what a subclass's ``__init__`` sets."""
driver = cls.__new__(cls)
ComputerDriverEnvironment.__init__(driver, settings=cls.Settings(), platform=platform)
ComputerDriverEnvironment.__init__(driver, settings=cls.Settings(), platform=platform, tool_prefix="")
config = dict(getattr(cls, _CARD_CONFIG_ATTR, None) or {})
card = EnvironmentCard(**{**config, "name": config.get("name") or cls.__name__})
return AgentEnvFastMCPApplication(card, driver).environment_card
Expand Down Expand Up @@ -203,6 +211,17 @@ async def _computer_relay(self, websocket: WebSocket) -> None:
def served_tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]:
return card.served_tools(self, tools)

def serve_tools_as(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]:
return self._computer_prefix.tools(tools)

def served_instructions(self, text: str) -> str:
return self._computer_prefix.instructions(text, self._computer_other_tools())

def _computer_other_tools(self) -> list[str]:
"""Its driver's own and forwarded tools, whose text stays as written, by their own names."""
return [marker.name for _member, marker in _marked_members(self, _TOOL_ATTR) if marker.name not in COMPUTER_TOOLS] + [
forwarded.tool.name for forwarded in self.forwarded_tools()]

def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]:
return card.served_extensions(self, extensions)

Expand All @@ -213,7 +232,8 @@ def extension_params(self, uri: str) -> dict:
return card.extension_params(self, uri)

def declared_extensions(self) -> list[EnvironmentExtension]:
return card.declared_extensions(self)
return [extension.model_copy(update={"description": self._computer_prefix.text(extension.description)})
for extension in card.declared_extensions(self)]

def _computer_require_init(self) -> None:
if self._computer is None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@ async def _computer_spaces_set(self, default_space: str, default_norm_scale: Opt
self.settings.default_norm_scale)
self.settings = self.settings.model_copy(update={"default_space": space, "default_norm_scale": scale})
for tool in self.mounted_card().capabilities.tools or []:
if tool.name in card.POINT_TOOLS:
await self.update_tool(tool.name, description=card.point_description(self, tool.name))
if (own := self._computer_prefix.own(tool.name)) in card.POINT_TOOLS:
await self.update_tool(tool.name, description=self._computer_prefix.text(card.point_description(self, own)))
return await self._computer_spaces_get()

@extension(COMPUTER_KEYBOARD, op="get", method="GET",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
"""A computer's tool prefix: the names it serves its tools under, so several computers serve theirs side by side on
one gateway. The SDK's ``computer_*`` names in its text are rewritten to them; its driver's own and forwarded tools
keep their text as written, and a sentence tells an agent how those are named."""
from __future__ import annotations

import re
from collections.abc import Iterable

from ..types import EnvironmentTool
from .tools import COMPUTER_TOOLS

#: The SDK's tool names, and a ``computer_*`` family of them, each as a whole word.
_SDK_NAMES = re.compile(r"(?<![\w-])(?:" + "|".join([*map(re.escape, sorted(COMPUTER_TOOLS, key=len, reverse=True)),
r"computer_(?:[a-z]+_)*\*"]) + r")(?![\w-])")


class ToolPrefix:
"""``prefix``, the start of every name a computer serves a tool under; none leaves every name as it is."""

def __init__(self, prefix: str) -> None:
self.prefix = prefix

def served(self, name: str) -> str:
"""The name the tool ``name`` is served under."""
return f"{self.prefix}{name}"

def own(self, served: str) -> str:
"""A served tool's own name."""
return served.removeprefix(self.prefix)

def text(self, text: str | None) -> str | None:
"""``text`` naming the SDK's tools as they are served; any other name stays as written."""
return _SDK_NAMES.sub(lambda m: self.prefix + m.group(0), text) if self.prefix and text else text

def tools(self, tools: list[EnvironmentTool]) -> list[EnvironmentTool]:
"""``tools`` under their served names, the SDK's own with their descriptions' names served too."""
return [tool.model_copy(update={"name": self.served(tool.name), **(
{"description": self.text(tool.description)} if tool.name in COMPUTER_TOOLS else {})}) for tool in tools]

def rule(self, others: Iterable[str]) -> str | None:
"""The sentence telling an agent how ``others``, the tools whose text stays as written, are served; None
without a prefix or any of them."""
example = next(iter(others), None)
if not self.prefix or example is None:
return None
return (f"This computer serves its tools under `{self.prefix}`: a tool this text names without `{self.prefix}` is "
f"served with it, so `{example}` is `{self.prefix}{example}`. Names inside a tool's arguments stay as written.")

def instructions(self, text: str, others: Iterable[str]) -> str:
"""An MCP server's instructions as the computer serves them: after the rule, if there is one, its names served."""
return "\n\n".join(part for part in (self.rule(others), self.text(text)) if part)
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ async def act(self, ctx: ToolContext, operation: str, args: tuple, *, frame: Fra
async with self.queue.turn(self.driver.settings.queue_wait_s):
if operation in DIALOG_BLOCKS and self.has("page") and (dialog := (await self.page_now()).dialog):
raise DialogOpen(f'a {dialog.type} dialog is open on the page: "{dialog.message}"; answer it with '
"computer_dialog first")
f"{self.driver._computer_prefix.served('computer_dialog')} first")
if mapped and frame is not None:
self.frames.check_current(frame, await self.display_now())
if at_pointer:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,29 @@

import json
import os
import re
from typing import Literal

from pydantic import BaseModel, ConfigDict, Field, ValidationError


SETTINGS_ENV = "AGENTENV_COMPUTER_SETTINGS"
#: The prefix a deploy gives the names of the computer's tools, so several computers serve them side by side.
TOOL_PREFIX_ENV = "AGENTENV_TOOL_PREFIX"
_TOOL_PREFIX = re.compile(r"[A-Za-z0-9_-]*")
#: The longest a tool prefix may be, its separator included, so a prefixed name stays well within the 64 characters
#: model APIs take for a tool's name.
MAX_TOOL_PREFIX = 16


def tool_prefix_problem(prefix: str) -> str | None:
"""What's wrong with ``prefix`` as a tool prefix, or None: only what a tool's name may hold, at most
``MAX_TOOL_PREFIX`` characters."""
if not _TOOL_PREFIX.fullmatch(prefix):
return f"tool prefix {prefix!r} holds characters a tool's name can't"
if len(prefix) > MAX_TOOL_PREFIX:
return f"tool prefix {prefix!r} is {len(prefix)} characters, and a tool prefix is at most {MAX_TOOL_PREFIX}"
return None

Space = Literal["frame", "norm"]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,14 @@ async def skill(driver: ComputerDriverEnvironment) -> tuple[str, str, dict[str,
if notes := driver.notes.strip():
sections += ["## Notes", notes]
body = "\n\n".join(sections) + "\n"
prefix = driver._computer_prefix
lead = f"{rule}\n\n" if (rule := prefix.rule(driver._computer_other_tools())) else ""
if bundled is None:
return description, body, {}
return prefix.text(description), prefix.text(lead + body), {}
own_description, own_body, files = bundled
return (f"{own_description} This computer: {system}, {display.width}x{display.height}, and its computer_* tools.",
own_body.rstrip() + "\n\n" + body, files)
return (prefix.text(f"{own_description} This computer: {system}, {display.width}x{display.height}, and its "
"computer_* tools."),
prefix.text(lead + own_body.rstrip() + "\n\n" + body), files)


def _input(driver: ComputerDriverEnvironment) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from mcp.types import CallToolResult
from pydantic import Field

from ..agent_env_environment import tool
from ..agent_env_environment import _TOOL_ATTR, tool
from ..tool_context import ToolContext
from .errors import InvalidArgument, Unsupported
from .frames import Frame
Expand Down Expand Up @@ -201,7 +201,8 @@ async def computer_type(
raise InvalidArgument(f"text is at most {self.max_text_length} characters on this computer, got {len(text)}")
control = sorted({f"U+{ord(c):04X}" for c in text if c not in "\n\t" and unicodedata.category(c) == "Cc"})
if control:
raise InvalidArgument(f"text has control characters {control}; press keys with computer_key")
raise InvalidArgument(f"text has control characters {control}; press keys with "
f"{self._computer_prefix.served('computer_key')}")
return await computer.act(ctx, "type_text", (text,), frame=computer.frames.latest(self.caller_identity(ctx)),
screenshot=screenshot, summary=f"Typed {len(text)} characters")

Expand Down Expand Up @@ -509,3 +510,7 @@ def _check_range(name: str, value: int, low: int, high: int) -> None:
def _display(frame: Frame) -> str:
d = frame.display
return f"{d.width}×{d.height}" + (f" at scale {d.scale:g}" if d.scale != 1 else "")


#: The tools the SDK serves on every computer, by their own names.
COMPUTER_TOOLS: tuple[str, ...] = tuple(sorted(name for name, member in vars(Tools).items() if hasattr(member, _TOOL_ATTR)))
Loading
Loading