diff --git a/src/agent_env/env/gateway/clock.py b/src/agent_env/env/gateway/clock.py index cceb14364..359558a84 100644 --- a/src/agent_env/env/gateway/clock.py +++ b/src/agent_env/env/gateway/clock.py @@ -186,6 +186,21 @@ def advance(self, virtual_time: str) -> dict: self._lived += jump return self.state() + def set_rate(self, rate) -> dict: + """Change the rate on the current arm from now on, keeping the virtual time, t0 and generation, so time + triggers keep their marks (a re-arm with set_time would re-anchor them); 0 freezes the clock.""" + r = _parse_rate(rate) + with self._lock: + if not self._armed or self._t0 is None: + raise ClockError("clock is not armed") + current = _advance(self._t0, (self._now() - self._anchor) * self._rate + self._ahead) + if current > _MAX_T0 - timedelta(seconds=_OVERFLOW_HORIZON_S * r): + raise ClockError(f"rate {r:g} would overflow datetime near the max from {_iso(current)}") + self._bank() + self._ahead = (current - self._t0).total_seconds() + self._rate = r + return self.state() + def clear(self) -> dict: """Disarm (idempotent). Afterwards reads 404.""" with self._lock: diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index a04bd51f2..37c8326d1 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -79,6 +79,7 @@ def data_plane_load_timeout_s(size_bytes: int | None) -> int: EXT_STATE_URI = "urn:agentenv:state/v1" EXT_CHANGES_URI = "urn:agentenv:changes/v1" EXT_NOTIFICATIONS_URI = "urn:agentenv:notifications/v1" +EXT_USER_CHAT_URI = "urn:agentenv:user-chat/v1" EXT_WEBSITE_URI = "urn:agentenv:website/v1" EXT_BROWSER_URI = "urn:agentenv:browser/v1" EXT_COMPUTER_APP_URI = "urn:agentenv:computer-app/v1" @@ -130,9 +131,11 @@ def sources(names: list[str]) -> str: TRIGGER_NEXT_EVENT_AT = "next_event_at" TRIGGER_NOTIFY = "notify" TRIGGER_WEBSITE_OP = "website_op" +# state reports each fire in flight (running, waits) and each executor call in progress (executor_calls). +TRIGGER_FIRES_IN_STATE = "fires_in_state" TRIGGER_FEATURES = (TRIGGER_NAMED_EXECUTORS, TRIGGER_PROVOKING_ROLES, TRIGGER_ACTION_DELAY, TRIGGER_EXECUTOR_HOURS, TRIGGER_UNLESS, TRIGGER_WEBSITE, TRIGGER_EXECUTOR_HOURS_TIME_ZONE, TRIGGER_NEXT_EVENT_AT, - TRIGGER_NOTIFY, TRIGGER_WEBSITE_OP) + TRIGGER_NOTIFY, TRIGGER_WEBSITE_OP, TRIGGER_FIRES_IN_STATE) # The one `source` a /step call may name instead of step_api, with the id of the hook artifact that made it. STEP_SOURCE_POST_TURN_HOOK = "post_turn_hook" @@ -207,9 +210,10 @@ def random_mcp_server_name() -> str: }, { "uri": EXT_TRIGGERS_URI, - "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path).", + "description": "Register (additive add-by-id), remove, clear, or read (state) typed Action/State triggers + the firing log (harness control path). config sets the engine's opt-in settings: concurrent_fires (a repeat trigger whose actions are all contact replies (nl) or notifications (notify) runs its fires side by side instead of queueing) and executor_timeout_seconds (the time limit of an executor that sets none).", "params": {"endpoint": "/triggers/register", "features": list(TRIGGER_FEATURES), "methods": { "register": {"method": "POST", "endpoint": "/triggers/register", "request": {"type": "object", "properties": {"watch_roles": {"type": "array", "items": {"type": "string"}}, "executor": {"type": "object"}, "executors": {"type": "object"}, "triggers": {"type": "array"}}, "required": ["triggers"]}}, + "config": {"method": "POST", "endpoint": "/triggers/config", "request": {"type": "object", "properties": {"concurrent_fires": {"type": "boolean"}, "executor_timeout_seconds": {"type": "number", "exclusiveMinimum": 0}}}}, "remove": {"method": "POST", "endpoint": "/triggers/remove", "request": {"type": "object", "properties": {"ids": {"type": "array", "items": {"type": "string"}}}, "required": ["ids"]}}, "clear": {"method": "POST", "endpoint": "/triggers/clear", "request": {"type": "object", "properties": {}}}, "state": {"method": "GET", "endpoint": "/triggers/state", "request": {"type": "object", "properties": {}}}}}, @@ -217,11 +221,12 @@ def random_mcp_server_name() -> str: }, { "uri": EXT_CLOCK_URI, - "description": "Virtual clock (urn:agentenv:clock/v1): read the gateway's virtual time (wall-clock-derived, rate-scaled) via get_time; set-time/advance/clear are the harness control path.", + "description": "Virtual clock (urn:agentenv:clock/v1): read the gateway's virtual time (wall-clock-derived, rate-scaled) via get_time; set-time/advance/set_rate/clear are the harness control path (set_rate changes the rate on the current arm, keeping time triggers' marks; 0 freezes the clock).", "params": {"endpoint": "/clock/time", "methods": { "get_time": {"method": "GET", "endpoint": "/clock/time"}, "set_time": {"method": "PUT", "endpoint": "/clock/set-time", "request": {"type": "object", "properties": {"virtual_time": {"type": "string"}, "virtual_seconds_per_real_second": {"type": "number"}}, "required": ["virtual_time"]}}, "advance": {"method": "PUT", "endpoint": "/clock/advance", "request": {"type": "object", "properties": {"virtual_time": {"type": "string"}}, "required": ["virtual_time"]}}, + "set_rate": {"method": "PUT", "endpoint": "/clock/rate", "request": {"type": "object", "properties": {"virtual_seconds_per_real_second": {"type": "number", "minimum": 0}}, "required": ["virtual_seconds_per_real_second"]}}, "clear": {"method": "POST", "endpoint": "/clock/clear", "request": {"type": "object", "properties": {}}}, "state": {"method": "GET", "endpoint": "/clock/state", "request": {"type": "object", "properties": {}}}}}, "required": False, @@ -270,4 +275,17 @@ def random_mcp_server_name() -> str: "clear": {"method": "POST", "endpoint": "/notifications/clear", "request": {"type": "object", "properties": {}}}}}, "required": False, }, + { + "uri": EXT_USER_CHAT_URI, + "description": "User chat (urn:agentenv:user-chat/v1): the message_user tool, for the roles config names, is how an always-on agent reaches its user; each message is queued with its virtual time until the harness takes it. With stay_quiet the roles also get stay_quiet, which sends nothing and is recorded for the next take. Built in: role rules don't hide them from a configured role.", + "params": {"endpoint": "/user_chat/state", "methods": { + "config": {"method": "POST", "endpoint": "/user_chat/config", "request": {"type": "object", "properties": { + "roles": {"type": "array", "items": {"type": "string"}}, "user_name": {"type": "string"}, + "stay_quiet": {"type": "boolean"}}}}, + "state": {"method": "GET", "endpoint": "/user_chat/state", "request": {"type": "object", "properties": {}}}, + "take": {"method": "POST", "endpoint": "/user_chat/take", "request": {"type": "object", "properties": { + "take_id": {"type": "string"}, "from": {"type": "string"}}}}, + "clear": {"method": "POST", "endpoint": "/user_chat/clear", "request": {"type": "object", "properties": {}}}}}, + "required": False, + }, ] diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index da71d495c..25a0aa536 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -90,6 +90,8 @@ ) from .triggers import TriggerEngine, TriggerError from .notifications import CHECK_TOOL_NAMES, NotificationError, Notifications, build_check_tools +from .user_chat import MESSAGE_USER_TOOL_NAME, UserChat, UserChatError, build_user_chat_tool +from .user_chat import STAY_QUIET_TOOL_NAME, build_stay_quiet_tool logger = logging.getLogger(__name__) @@ -201,6 +203,7 @@ async def list_website_urls() -> list[str]: ) self._clock = Clock() self._notifications = Notifications(lambda: self._clock) + self._user_chat = UserChat(lambda: self._clock) self._env_get_time_url_cache: str | None = None self._install_role_filters() @@ -295,6 +298,10 @@ async def triggers_register(request: Request) -> Response: async def triggers_remove(request: Request) -> Response: return await _trigger_write(request, self._trigger_engine.remove) + @self._mcp.custom_route("/triggers/config", methods=["POST"]) + async def triggers_config(request: Request) -> Response: + return await _trigger_write(request, self._trigger_engine.config) + @self._mcp.custom_route("/triggers/clear", methods=["POST"]) async def triggers_clear(request: Request) -> Response: return self._json_response(self._trigger_engine.clear(), 200) @@ -316,6 +323,7 @@ async def clock_set_time(request: Request) -> Response: state = self._clock.set_time(body.get("virtual_time"), body.get("virtual_seconds_per_real_second")) except ClockError as e: return self._json_response({"ok": False, "error": str(e)}, 400) + self._trigger_engine.clock_changed() self._register_get_time_tool() return self._json_response(state, 200) @@ -333,10 +341,26 @@ async def clock_advance(request: Request) -> Response: return self._json_response({"ok": False, "error": str(e)}, 400) return self._json_response({**state, "due": self._trigger_engine.clock_advanced()}, 200) + @self._mcp.custom_route("/clock/rate", methods=["PUT"]) + async def clock_rate(request: Request) -> Response: + try: + body = await request.json() + except Exception: + return self._json_response({"ok": False, "error": "Invalid JSON body"}, 400) + if not isinstance(body, dict): + return self._json_response({"ok": False, "error": "body must be a JSON object"}, 400) + try: + state = self._clock.set_rate(body.get("virtual_seconds_per_real_second")) + except ClockError as e: + return self._json_response({"ok": False, "error": str(e)}, 400) + self._trigger_engine.clock_changed() + return self._json_response(state, 200) + @self._mcp.custom_route("/clock/clear", methods=["POST"]) async def clock_clear(request: Request) -> Response: self._unregister_get_time_tool() state = self._clock.clear() + self._trigger_engine.clock_changed() return self._json_response(state, 200) @self._mcp.custom_route("/clock/time", methods=["GET"]) @@ -408,6 +432,51 @@ async def notifications_clear(request: Request) -> Response: self._unregister_check_tools() return self._json_response(self._notifications.clear(), 200) + @self._mcp.custom_route("/user_chat/config", methods=["POST"]) + async def user_chat_config(request: Request) -> Response: + try: + body = await request.json() + except Exception: + return self._json_response({"ok": False, "error": "Invalid JSON body"}, 400) + if isinstance(body, dict) and body.get("roles"): + try: + await self._ensure_tools_discovered() + except Exception as e: + return self._json_response( + {"ok": False, "error": f"tool discovery has not completed ({type(e).__name__}); retry"}, 503) + if MESSAGE_USER_TOOL_NAME in self._tool_server_urls: + return self._json_response( + {"ok": False, "error": f"'{MESSAGE_USER_TOOL_NAME}' already registered by a backing server"}, 409) + try: + state = self._user_chat.configure(body) + except UserChatError as e: + return self._json_response({"ok": False, "error": str(e)}, 400) + if self._user_chat.enabled: + self._register_user_chat_tool() + else: + self._unregister_user_chat_tool() + return self._json_response(state, 200) + + @self._mcp.custom_route("/user_chat/state", methods=["GET"]) + async def user_chat_state(request: Request) -> Response: + return self._json_response(self._user_chat.state(), 200) + + @self._mcp.custom_route("/user_chat/take", methods=["POST"]) + async def user_chat_take(request: Request) -> Response: + try: + body = await request.json() + except Exception: + body = {} + try: + return self._json_response(self._user_chat.take(body), 200) + except UserChatError as e: + return self._json_response({"ok": False, "error": str(e)}, 400) + + @self._mcp.custom_route("/user_chat/clear", methods=["POST"]) + async def user_chat_clear(request: Request) -> Response: + self._unregister_user_chat_tool() + return self._json_response(self._user_chat.clear(), 200) + @self._mcp.custom_route("/trajectory", methods=["GET"]) async def env_trajectory(request: Request) -> Response: return self._serve_trajectory() @@ -524,9 +593,18 @@ def _listen_socket(self, family: int, host: str) -> socket.socket: def _is_disabled(self, role: str, name: str) -> bool: """Walk the rule chain (role first, then WILDCARD) returning the first concrete True/False; default False. A tool listed hidden by default skips "*"-tool rules: only rules naming it apply, and with none it is hidden. - The scheduled-check tools exist only for the roles the notifications config names, whatever the rules say.""" + The scheduled-check tools exist only for the roles the notifications config names, whatever the rules say. + A role the user-chat config names always has message_user, get_time and the scheduled-check tools the + notifications config gives it, whatever the rules say.""" if name in CHECK_TOOL_NAMES and name not in self._tool_server_urls and role not in self._notifications.roles: return True + if name == MESSAGE_USER_TOOL_NAME and name not in self._tool_server_urls: + return role not in self._user_chat.roles # built in: role rules don't hide it from a configured role + if (name == GET_TIME_TOOL_NAME or name in CHECK_TOOL_NAMES) and name not in self._tool_server_urls \ + and role in self._user_chat.roles: + return False + if name == STAY_QUIET_TOOL_NAME and name not in self._tool_server_urls: + return not (self._user_chat.stay_quiet and role in self._user_chat.roles) # opt-in, like message_user hidden = name in self._hidden_by_default for r in (role, WILDCARD): rules = self._role_rules.get(r, {}) @@ -937,7 +1015,8 @@ async def _discover_and_register_tools(self, sessions: dict[str, ClientSession]) # clock and the checks refuse to turn on over it sources: dict[str, list[str]] = { tool.name: ["gateway"] for tool in self._server_tools.get("gateway", []) - if tool.name != GET_TIME_TOOL_NAME and tool.name not in CHECK_TOOL_NAMES} + if tool.name != GET_TIME_TOOL_NAME and tool.name not in CHECK_TOOL_NAMES + and tool.name != MESSAGE_USER_TOOL_NAME and tool.name != STAY_QUIET_TOOL_NAME} for server, tools in listed: for tool in tools: sources.setdefault(tool.name, []).append(server.name) @@ -1231,6 +1310,54 @@ def _unregister_check_tools(self) -> None: self._server_tools.pop("gateway", None) self._trigger_engine.invalidate_readonly_cache() + def _register_user_chat_tool(self) -> None: + """Expose message_user to the roles the user-chat config names (rebuilt so it names the user).""" + tools = self._mcp._tool_manager._tools + if MESSAGE_USER_TOOL_NAME in self._tool_server_urls: + return + tool = build_user_chat_tool(self._user_chat, self._create_arg_model_from_schema) + tools[MESSAGE_USER_TOOL_NAME] = tool + mirrored = [t for t in self._server_tools.get("gateway", []) if t.name != MESSAGE_USER_TOOL_NAME] + mirrored.append(MCPTool(name=tool.name, description=tool.description, inputSchema=tool.parameters, + annotations=tool.annotations)) + self._server_tools["gateway"] = mirrored + self._sync_stay_quiet_tool() + self._trigger_engine.invalidate_readonly_cache() + + def _sync_stay_quiet_tool(self) -> None: + """stay_quiet exists only while the user-chat config turns it on (opt-in); otherwise it is absent.""" + tools = self._mcp._tool_manager._tools + if STAY_QUIET_TOOL_NAME in self._tool_server_urls: + return + mirrored = [t for t in self._server_tools.get("gateway", []) if t.name != STAY_QUIET_TOOL_NAME] + if self._user_chat.stay_quiet: + tool = build_stay_quiet_tool(self._user_chat, self._create_arg_model_from_schema) + tools[STAY_QUIET_TOOL_NAME] = tool + mirrored.append(MCPTool(name=tool.name, description=tool.description, inputSchema=tool.parameters, + annotations=tool.annotations)) + else: + tools.pop(STAY_QUIET_TOOL_NAME, None) + if mirrored: + self._server_tools["gateway"] = mirrored + else: + self._server_tools.pop("gateway", None) + + def _unregister_user_chat_tool(self) -> None: + """Remove message_user and stay_quiet when the chat is turned off or cleared; a backing server's are left.""" + tools = self._mcp._tool_manager._tools + if MESSAGE_USER_TOOL_NAME in self._tool_server_urls: + return + tools.pop(MESSAGE_USER_TOOL_NAME, None) + if STAY_QUIET_TOOL_NAME not in self._tool_server_urls: + tools.pop(STAY_QUIET_TOOL_NAME, None) + remaining = [t for t in self._server_tools.get("gateway", []) + if t.name not in (MESSAGE_USER_TOOL_NAME, STAY_QUIET_TOOL_NAME)] + if remaining: + self._server_tools["gateway"] = remaining + else: + self._server_tools.pop("gateway", None) + self._trigger_engine.invalidate_readonly_cache() + def _create_and_register_proxy_tool(self, server_url: str, tool: MCPTool) -> None: self._mcp._tool_manager._tools[tool.name] = self._proxy_tool(server_url, tool) diff --git a/src/agent_env/env/gateway/triggers.py b/src/agent_env/env/gateway/triggers.py index 415ff99fc..223780698 100644 --- a/src/agent_env/env/gateway/triggers.py +++ b/src/agent_env/env/gateway/triggers.py @@ -16,6 +16,7 @@ import uuid from collections import deque from datetime import datetime, time, timedelta, timezone, tzinfo +from time import monotonic as _monotonic from typing import Any from zoneinfo import ZoneInfo @@ -51,8 +52,13 @@ _BARRIER_AT = ("provoking_call",) _BARRIER_KEYS = ("at", "timeout_seconds") _DROP = object() # an unresolved `${x?}` used as a whole value: drop the key / element -# Real seconds between re-reads of the clock while waiting out virtual time, so a rate change, re-arm or clear is followed. +# Real seconds between re-reads of the clock while waiting out virtual time, so a change made other than through the +# gateway's clock routes (which wake the waits: clock_changed) is followed. _VIRTUAL_RECHECK_S = 5.0 +# The clock's resolution: virtual times are microseconds. +_VIRTUAL_RESOLUTION_S = 1e-6 +# Seconds an executor call waits to connect, and for a task poll's answer. +_EXECUTOR_HTTP_TIMEOUT_S = 30.0 # An allowlist, not a size bound: identifier/metadata keys only, since bodies and addresses are PII. _ECHO_KEYS = frozenset({ "action", "createdTime", "fileName", "kind", "mimeType", "modifiedTime", "name", "role", @@ -440,7 +446,14 @@ def __init__(self, gateway): self._driver_task: asyncio.Task | None = None self._driver_interval = 1.0 # real seconds between autonomous time-trigger polls self._time_trigger_gate = asyncio.Event() # parks the driver while no time-trigger is armed - self._clock_moved = asyncio.Event() # set, and replaced, when an advance moves the clock under waiting actions + self._clock_moved = asyncio.Event() # set, and replaced, when the clock changes under waiting actions + # Opt-in settings (`config`), off by default so a registration behaves as it always has. + self._concurrent_fires = False # a contact-reply trigger's repeat fires run side by side instead of queueing + self._executor_timeout_default: float | None = None # the time limit of an executor that sets none + self._unset_timeouts: set[str | None] = set() # executors registered without a timeout_seconds of their own + # One turn at a time per executor conversation (its URL and context id): fires of one trigger running side by + # side would otherwise resume the same session at once. + self._executor_locks: dict[tuple[str, str], asyncio.Lock] = {} def register(self, body: Any) -> dict: """Additively add each trigger by id (idempotent upsert; conflicting id or bad shape -> 400); watch_roles and @@ -448,7 +461,7 @@ def register(self, body: Any) -> dict: if not isinstance(body, dict): raise TriggerError("body must be a JSON object") watch_roles = self._resolve_watch_roles(body) - executors = self._resolve_executors(body, watch_roles) + executors, unset_timeouts = self._resolve_executors(body, watch_roles) triggers = body.get("triggers") if not isinstance(triggers, list): raise TriggerError("triggers must be a list") @@ -480,15 +493,19 @@ def register(self, body: Any) -> dict: self._readonly_tools = None self._watch_roles = watch_roles self._executors = executors + self._unset_timeouts |= unset_timeouts self._config_set = True for tid, spec in to_add: + # `running` counts its fires in flight, `alongside` holds those run beside the first (concurrent_fires), + # `waits` each fire waiting out a delay or its executor's hours, `calls` each executor call in progress. self._triggers[tid] = {"spec": spec, "status": "armed", "detected_at": None, "fired_at": None, "evaluating": False, "next_mark": None, "resolved": False, "fire_count": 0, "failure_count": 0, "last_failure_at": None, "rng": _make_rng(spec), "mark_gen": None, "pending": deque(), "task": None, "delay_rngs": _delay_rngs(spec), - "attempts": 0, "waiting": False, "resumes": None, "hours": None} + "attempts": 0, "waiting": False, "running": 0, "alongside": set(), + "waits": {}, "calls": []} self._emit("added", tid) self._recompute_time_flag() return {"ok": True, "added": batch_order, "all": list(self._triggers)} @@ -535,8 +552,9 @@ def remove(self, body: Any) -> dict: for tid in ids: trig = self._triggers.pop(tid, None) if trig is not None: - if trig["task"] is not None: - trig["task"].cancel() # a fire waiting out a delay or hours would otherwise still act + for task in [trig["task"], *trig["alongside"]]: + if task is not None: + task.cancel() # a fire waiting out a delay or hours would otherwise still act removed.append(tid) self._emit("removed", tid) self._recompute_time_flag() @@ -556,9 +574,49 @@ def clear(self) -> dict: self._in_flight = {} self._config_set = False self._readonly_tools = None + self._concurrent_fires = False + self._executor_timeout_default = None + self._unset_timeouts = set() + self._executor_locks = {} self._recompute_time_flag() return {"ok": True} + def config(self, body: Any) -> dict: + """Set the engine's opt-in settings, each off unless the body sets it (an omitted one resets): with + ``concurrent_fires``, each further call that provokes a repeat trigger already firing starts its own fire at + once, side by side, instead of queueing behind the one running, when every action of the trigger is a contact's + reply (``nl``) or a notification (``notify``); any other trigger still queues, since its tool actions may read + and then write the same rows. A queued fire draws its delay only when its turn comes, and state shows no due + time for it, so a quick reply queued behind a slow one would wait out the slow one's delay too. + Fires of one trigger side by side still take turns on its executor conversation (``_executor_turn``). + ``executor_timeout_seconds`` is the time limit of each executor whose registration set no + ``timeout_seconds`` (else 120). ``clear`` resets both.""" + if not isinstance(body, dict): + raise TriggerError("body must be a JSON object") + unknown = sorted(set(body) - {"concurrent_fires", "executor_timeout_seconds"}) + if unknown: + raise TriggerError(f"unknown setting(s) {unknown}; expected concurrent_fires, executor_timeout_seconds") + concurrent = body.get("concurrent_fires", False) + if not isinstance(concurrent, bool): + raise TriggerError(f"concurrent_fires must be true or false (got {concurrent!r})") + timeout = body.get("executor_timeout_seconds") + if timeout is not None and (isinstance(timeout, bool) or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) or timeout <= 0): + raise TriggerError(f"executor_timeout_seconds must be a positive, finite number (got {timeout!r})") + self._concurrent_fires = concurrent + self._executor_timeout_default = None if timeout is None else float(timeout) + return {"ok": True, **self._settings()} + + def _settings(self) -> dict: + return {"concurrent_fires": self._concurrent_fires, + "executor_timeout_seconds": self._executor_timeout_default} + + def _executor_timeout(self, name: str | None, executor: dict) -> float: + """The executor's time limit: its own, else the configured default, else 120.""" + if self._executor_timeout_default is not None and name in self._unset_timeouts: + return self._executor_timeout_default + return executor["timeout_seconds"] + def _recompute_time_flag(self) -> None: # "firing" counts: a recurrence mid-fire returns to "armed" and still needs the driver. self._has_time_triggers = any( @@ -616,14 +674,15 @@ def _resolve_watch_roles(self, body: dict) -> set[str]: raise TriggerError(f"watch_roles {sorted(new)} conflicts with the already-registered {sorted(self._watch_roles)} — clear to change it") return new - def _resolve_executors(self, body: dict, watch_roles: set[str]) -> dict[str | None, dict]: + def _resolve_executors(self, body: dict, watch_roles: set[str]) -> tuple[dict[str | None, dict], set[str | None]]: """The executors once this register lands: those before, plus the body's `executor` (under None) and - `executors`, each set once.""" + `executors`, each set once; and those of them new here that set no `timeout_seconds` of their own.""" named = body.get("executors") if named is not None and (not isinstance(named, dict) or not all(isinstance(n, str) and n for n in named)): raise TriggerError("executors must be an object keyed by non-empty executor names") incoming = ({None: body["executor"]} if body.get("executor") is not None else {}) | (named or {}) merged = dict(self._executors) + unset: set[str | None] = set() for name, raw in incoming.items(): normalized = self._normalize_executor(raw, "executor" if name is None else f"executors[{name!r}]", watch_roles) @@ -632,8 +691,10 @@ def _resolve_executors(self, body: dict, watch_roles: set[str]) -> dict[str | No if name is None else f"executor {name!r} conflicts with the already-registered executor of that name — " "clear to change it") + if name not in merged and raw.get("timeout_seconds") is None: + unset.add(name) merged[name] = normalized - return merged + return merged, unset @staticmethod def _normalize_executor(raw: Any, where: str, watch_roles: set[str]) -> dict: @@ -896,10 +957,15 @@ def state(self) -> dict: resumes = {tid: self._resumes_at(t) for tid, t in self._triggers.items()} upcoming = [self._upcoming_mark(t) for t in self._triggers.values()] + list(resumes.values()) next_event = min((at for at in upcoming if at is not None), default=None) + now = _monotonic() return { "config": {"watch_roles": sorted(self._watch_roles), "executor_configured": None in self._executors, "executors": sorted(n for n in self._executors if n is not None)}, + "settings": self._settings(), # what `config` set + # `running` counts a trigger's fires in flight, `waits` says when each of them waiting out a delay or its + # executor's hours goes on, and `executor_calls` lists each call to an executor in progress, with the + # seconds left of its time limit; `waiting` and `resumes_at` sum `waits` up. "triggers": [ {"id": tid, "type": t["spec"]["when"]["type"], "when": {k: t["spec"]["when"][k] for k in _WHEN_SUMMARY_KEYS if k in t["spec"]["when"]}, @@ -907,9 +973,14 @@ def state(self) -> dict: "fired_at": t["fired_at"], "fire_count": t["fire_count"], "failure_count": t["failure_count"], "last_failure_at": t["last_failure_at"], "barrier": t["spec"].get("barrier"), - "pending": len(t["pending"]), "waiting": t["waiting"], "evaluating": t["evaluating"], + "pending": len(t["pending"]), "waiting": bool(t["waits"]), "evaluating": t["evaluating"], "resumes_at": _iso(resumes[tid]) if resumes[tid] else None, - "next_mark": _iso(t["next_mark"]) if t["next_mark"] else None} + "next_mark": _iso(t["next_mark"]) if t["next_mark"] else None, + "running": t["running"], + "waits": sorted(({"resumes_at": _iso(at) if (at := self._wait_resumes(w)) else None} + for w in t["waits"].values()), key=lambda w: w["resumes_at"] or ""), + "executor_calls": [{"executor": c["executor"], "action_index": c["action_index"], + "seconds_left": round(max(0.0, c["deadline"] - now), 1)} for c in t["calls"]]} for tid, t in self._triggers.items() ], "next_event_at": _iso(next_event) if next_event else None, @@ -1035,8 +1106,17 @@ def _executor_roles(self) -> set[str]: def _mark_detected(self, tid: str, trig: dict, provoking: dict, ctx: dict | None = None) -> asyncio.Task | None: ctx = ctx if ctx is not None else {} since = self._gw._clock.elapsed() # a delay runs from the provoking call, not from when its turn comes + if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES and self._concurrent_fires and _replies_only(trig["spec"]): + # Mid-fire, side by side: this call's fire starts now, so a quick reply doesn't wait behind a slow one. + self._emit("detected", tid, provoking=provoking, alongside=trig["running"]) + task = self._start_fire(trig, lambda ticket: self._fire_alongside(tid, trig, ctx, since, ticket)) + task.set_name(tid) + trig["alongside"].add(task) + task.add_done_callback(trig["alongside"].discard) + return task if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES: - # Mid-fire: queue it; the running fire drains the queue before it settles. + # Mid-fire: queue it; the running fire drains the queue before it settles, or, when only fires run side by + # side are left (concurrent_fires turned off meanwhile), the last of them to end starts one that does. trig["pending"].append((ctx, since)) trig["status"] = "queued" self._emit("detected", tid, provoking=provoking, queued=len(trig["pending"])) @@ -1044,7 +1124,7 @@ def _mark_detected(self, tid: str, trig: dict, provoking: dict, ctx: dict | None trig["status"] = "firing" trig["detected_at"] = _now() self._emit("detected", tid, provoking=provoking) - trig["task"] = self._schedule(self._fire(tid, trig, ctx, since)) + trig["task"] = self._start_fire(trig, lambda ticket: self._fire(tid, trig, ctx, since, ticket)) trig["task"].set_name(tid) return trig["task"] @@ -1092,11 +1172,18 @@ def clock_advanced(self) -> list[str]: due = self._eval_time_triggers() now = self._gw._clock.now() ended = [tid for tid, t in self._triggers.items() - if now is not None and (at := self._resumes_at(t)) is not None and at <= now - and (t["hours"] is None or _next_opening(now, t["hours"]) is None)] + if now is not None and any((at := self._wait_resumes(w)) is not None and at <= now + and (w["hours"] is None or _next_opening(now, w["hours"]) is None) + for w in t["waits"].values())] + self.clock_changed() + return list(dict.fromkeys(due + ended)) + + def clock_changed(self) -> None: + """Wake the actions waiting out virtual time to read the clock again, after it jumps, is re-armed, changes rate + or is cleared: one napping on a frozen clock would otherwise see it run again only at its next recheck, up to + _VIRTUAL_RECHECK_S real seconds (minutes of virtual time) late.""" self._clock_moved.set() self._clock_moved = asyncio.Event() - return list(dict.fromkeys(due + ended)) def _resolve_first_mark(self, trig: dict, now: datetime) -> None: """Stamp the first mark for a non-anchored time-trigger; event-anchored ones wait for _resolve_dependents.""" @@ -1135,8 +1222,14 @@ def _upcoming_mark(self, trig: dict) -> datetime | None: return None if mark is None or (until is not None and mark > until) else mark def _resumes_at(self, trig: dict) -> datetime | None: - """When a fire waiting out a delay or its executor's hours goes on, in virtual time; None when none waits.""" - resumes = trig["resumes"] + """When the first of its fires waiting out a delay or its executor's hours goes on, in virtual time; None when + none waits.""" + return min((at for w in trig["waits"].values() if (at := self._wait_resumes(w)) is not None), default=None) + + def _wait_resumes(self, wait: dict) -> datetime | None: + """When one waiting fire goes on, in virtual time: its delay's end (a Clock.elapsed() reading) or its + executor's next opening.""" + resumes = wait["resumes"] if resumes is None or isinstance(resumes, datetime): return resumes return self._gw._clock.at_elapsed(resumes) @@ -1214,9 +1307,11 @@ def _mark_time_detected(self, tid: str, trig: dict, marks: list[datetime], cappe # first/last only: a burst can carry _MAX_CATCHUP marks, each with its own `fired` event. self._emit("detected", tid, provoking=provoking, due=len(marks), capped=capped, first_mark=_iso(marks[0]), last_mark=_iso(marks[-1])) - trig["task"] = self._schedule(self._fire_time(tid, trig, marks, capped, terminated)) + trig["task"] = self._start_fire(trig, lambda ticket: self._fire_time(tid, trig, marks, capped, terminated, + ticket)) - async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: bool, terminated: bool) -> None: + async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: bool, terminated: bool, + ticket: dict) -> None: """Run the actions once per due mark, then re-arm (recurring) or retire (one-shot / terminated).""" try: await self._gw._log_event({"event_type": "trigger_fired", "source": "trigger_engine", "trigger_id": tid}) @@ -1227,8 +1322,9 @@ async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: sinces = [elapsed - (max(0.0, (now - mark).total_seconds()) if now is not None else 0.0) for mark in marks] for mark, since in zip(marks, sinces): trig["attempts"] += 1 + fire = trig["attempts"] for index, action in enumerate(trig["spec"]["actions"]): - if not await self._run_action(tid, trig, index, action, {}, since): + if not await self._run_action(tid, trig, index, action, {}, since, fire=fire): trig["status"] = "failed" trig["next_mark"] = None self._emit("failed", tid, action_index=index) @@ -1244,6 +1340,7 @@ async def _fire_time(self, tid: str, trig: dict, marks: list[datetime], capped: self._emit("failed", tid, detail=str(e)[:300]) return finally: + _uncount(trig, ticket) self._recompute_time_flag() # a terminated trigger lets the driver park again self._resolve_dependents(tid) # success-only, outside the try (see _fire) @@ -1264,8 +1361,23 @@ def _is_readonly(self, name: str) -> bool: self._readonly_tools = readonly return name in self._readonly_tools + def _start_fire(self, trig: dict, run) -> asyncio.Task: + """Schedule ``run(ticket)``, one fire of ``trig``, counted in its ``running`` from now, so state shows it at work + before its task first runs. The fire's ``finally`` uncounts it (``_uncount``); one cancelled before it began + is uncounted when its task ends.""" + ticket = {"counted": False} + task = self._schedule(run(ticket)) # counted only once it exists, so a schedule that raises leaves no count + trig["running"] += 1 + ticket["counted"] = True + task.add_done_callback(lambda _: _uncount(trig, ticket)) + return task + def _schedule(self, coro) -> asyncio.Task: - task = asyncio.get_running_loop().create_task(coro) + try: + task = asyncio.get_running_loop().create_task(coro) + except BaseException: + coro.close() # never awaited: closed, so it doesn't warn + raise self._tasks.add(task) task.add_done_callback(self._tasks.discard) return task @@ -1340,52 +1452,89 @@ async def _internal_call(self, tool_name: str, arguments: dict, role: str | None caller = Caller(role=role) if role is not None else None return await gw._call_child_tool(server_url, tool_name, arguments, caller=caller) - async def _fire(self, tid: str, trig: dict, ctx: dict, since: float) -> None: + async def _fire(self, tid: str, trig: dict, ctx: dict, since: float, ticket: dict) -> None: """Run the actions for one provoking call, then drain any calls queued while firing (repeat triggers). A failure always re-arms: a mirror states a standing invariant, so one transient error must not silence it for the rest of the run. Ends armed or fired (one-shot success).""" - spec = trig["spec"] - repeat = bool(spec["when"].get("repeat")) + repeat = bool(trig["spec"]["when"].get("repeat")) + outcome: str | None = None # the status a normal end leaves try: while True: - ok = True - trig["attempts"] += 1 - try: - await self._gw._log_event({"event_type": "trigger_fired", "source": "trigger_engine", "trigger_id": tid, - "provoking_tool": (ctx.get("args") is not None and spec["when"].get("tool")) or None, - **({"provoking_request": ctx["request"]} if "request" in ctx else {})}) - for index, action in enumerate(spec["actions"]): - if not await self._run_action(tid, trig, index, action, ctx, since): - ok = False - self._emit("failed", tid, action_index=index) - break - if ok: - trig["fire_count"] += 1 - trig["fired_at"] = _now() - self._emit("fired", tid, fire_count=trig["fire_count"]) - except Exception as e: - logger.exception(f"trigger '{tid}' firing failed") - ok = False - self._emit("failed", tid, detail=str(e)[:300]) - if ok: - self._resolve_dependents(tid) # success-only, outside the try so it can't flip a fired trigger to failed + ok = await self._fire_once(tid, trig, ctx, since) if trig["pending"]: ctx, since = trig["pending"].popleft() trig["status"] = "firing" trig["detected_at"] = _now() continue - trig["status"] = "armed" if (repeat or not ok) else "fired" + outcome = "armed" if (repeat or not ok) else "fired" return finally: - # An abnormal exit must not wedge the trigger in firing/queued, nor drop its queue silently; a trigger removed - # or cleared meanwhile is gone, and has nothing left to report. - if trig["status"] in TRIGGER_IN_FLIGHT_STATUSES and self._triggers.get(tid) is trig: - self._emit("failed", tid, detail="firing did not settle", dropped=len(trig["pending"])) - trig["status"] = "armed" - trig["pending"].clear() + _uncount(trig, ticket) trig["task"] = None + self._settle_status(tid, trig, outcome) - async def _run_action(self, tid: str, trig: dict, index: int, action: dict, ctx: dict, since: float) -> bool: + async def _fire_alongside(self, tid: str, trig: dict, ctx: dict, since: float, ticket: dict) -> None: + """One fire of a repeat trigger run beside its others (``concurrent_fires``); the last of them to end re-arms + it, as a repeat trigger always ends armed.""" + try: + await self._fire_once(tid, trig, ctx, since) + finally: + _uncount(trig, ticket) + self._settle_status(tid, trig, "armed") + + def _settle_status(self, tid: str, trig: dict, outcome: str | None) -> None: + """A fire of ``tid`` ended, normally (leaving ``outcome``) or not (None); the last of its fires to end sets its + status. An abnormal exit must not wedge the trigger in firing/queued, nor drop its queue silently; a trigger + removed or cleared meanwhile is gone, and has nothing left to report.""" + if trig["running"] or trig["status"] not in TRIGGER_IN_FLIGHT_STATUSES: + return + if outcome is not None and trig["pending"] and trig["task"] is None and self._triggers.get(tid) is trig: + # Calls queued while only fires side by side ran (concurrent_fires turned off meanwhile): none of those + # drains a queue, so a fire that does starts now rather than leave them stranded. + ctx, since = trig["pending"].popleft() + trig["status"] = "firing" + trig["detected_at"] = _now() + trig["task"] = self._start_fire(trig, lambda ticket: self._fire(tid, trig, ctx, since, ticket)) + trig["task"].set_name(tid) + return + if outcome is not None: + trig["status"] = outcome + elif self._triggers.get(tid) is trig: + self._emit("failed", tid, detail="firing did not settle", dropped=len(trig["pending"])) + trig["status"] = "armed" + trig["pending"].clear() + + async def _fire_once(self, tid: str, trig: dict, ctx: dict, since: float) -> bool: + """Run the actions once for one provoking call; whether they all went through.""" + spec = trig["spec"] + ok = True + trig["attempts"] += 1 + fire = trig["attempts"] # this fire's number, read now: fires side by side move `attempts` on meanwhile + try: + await self._gw._log_event({"event_type": "trigger_fired", "source": "trigger_engine", "trigger_id": tid, + "provoking_tool": (ctx.get("args") is not None and spec["when"].get("tool")) or None, + **({"provoking_request": ctx["request"]} if "request" in ctx else {})}) + for index, action in enumerate(spec["actions"]): + if not await self._run_action(tid, trig, index, action, ctx, since, fire=fire): + ok = False + self._emit("failed", tid, action_index=index) + break + if ok: + trig["fire_count"] += 1 + trig["fired_at"] = _now() + self._emit("fired", tid, fire_count=trig["fire_count"]) + except Exception as e: + logger.exception(f"trigger '{tid}' firing failed") + ok = False + self._emit("failed", tid, detail=str(e)[:300]) + if ok: + self._resolve_dependents(tid) # success-only, outside the try so it can't flip a fired trigger to failed + return ok + + async def _run_action(self, tid: str, trig: dict, index: int, action: dict, ctx: dict, since: float, + fire: int | None = None) -> bool: + """Run one action of ``tid``'s fire number ``fire`` (its ``attempts`` when not given); whether it went + through.""" if "delay" in action: await self._delay(tid, trig, index, action, since) changelog_before = self._gw._query_changelog_id() @@ -1436,12 +1585,12 @@ async def _run_action(self, tid: str, trig: dict, index: int, action: dict, ctx: detail = {"notify": action["channel"], "to": note["to"], "notification_id": note["id"], "text_len": len(text)} else: - fire = await self._run_nl(tid, trig, index, action, ctx) - if fire is None: + done = await self._run_nl(tid, trig, index, action, ctx, fire=fire) + if done is None: return False - detail = {"nl": action["instruction"][:120], **_named_executor(action), "calls": fire["calls"], - **({"failed_calls": fire["failed_calls"]} if fire["failed_calls"] else {}), - **({"calls_shared_with": sorted(fire["shared_with"])} if fire["shared_with"] else {})} + detail = {"nl": action["instruction"][:120], **_named_executor(action), "calls": done["calls"], + **({"failed_calls": done["failed_calls"]} if done["failed_calls"] else {}), + **({"calls_shared_with": sorted(done["shared_with"])} if done["shared_with"] else {})} if "verify" in action and action["type"] != "nl": # only an action's own verify inherits its `as`; when.check state probes read as the pin if not await self._run_verify(tid, index, action["verify"], ctx, role=action.get("as")): @@ -1463,38 +1612,63 @@ async def _run_verify(self, tid: str, index: int, verify: dict, ctx: dict | None self._emit("verify_failed", tid, action_index=index) return False - async def _run_nl(self, tid: str, trig: dict, index: int, action: dict, ctx: dict | None = None) -> dict | None: - """The fire with the calls its executor made for it, by tool, or None when the action failed.""" + async def _run_nl(self, tid: str, trig: dict, index: int, action: dict, ctx: dict | None = None, + fire: int | None = None) -> dict | None: + """Fire number ``fire`` (``attempts`` when not given) with the calls its executor made for it, by tool, or None + when the action failed.""" instruction = action["instruction"] named = _named_executor(action) executor = self._executors.get(action.get("executor")) if executor is not None and "hours" in executor: await self._await_hours(tid, trig, index, executor["hours"], named) - fire = _in_flight_fire({"trigger_id": tid, "action_index": index, "fire": trig["attempts"]}) - fires = self._in_flight.setdefault(executor["role"], []) if executor is not None else [] - for other in fires: # an executor's calls carry only its role, so fires it realizes at once share their counts - other["shared_with"].add(tid) - fire["shared_with"].add(other["cause"]["trigger_id"]) - fires.append(fire) - try: - for attempt in (1, 2): - reply = await self._call_executor(executor, instruction, context_id=f"trigger-{tid}") - if reply is None: - self._emit("action_failed", tid, action_index=index, - detail=f"executor attempt {attempt} did not complete", **named) - return None - if "verify" not in action: - return fire - if await self._run_verify(tid, index, action["verify"], ctx): - return fire - if attempt == 1: - instruction = (f"{action['instruction']}\n\nYour previous attempt did not pass verification — " - f"the expected change is not observable in the environment. Do it again exactly as described.") - finally: - fires.remove(fire) + done = _in_flight_fire({"trigger_id": tid, "action_index": index, + "fire": trig["attempts"] if fire is None else fire}) + for attempt in (1, 2): + reply = await self._executor_turn(tid, trig, index, action.get("executor"), executor, instruction, done) + if reply is None: + self._emit("action_failed", tid, action_index=index, + detail=f"executor attempt {attempt} did not complete", **named) + return None + if "verify" not in action: + return done + if await self._run_verify(tid, index, action["verify"], ctx): + return done + if attempt == 1: + instruction = (f"{action['instruction']}\n\nYour previous attempt did not pass verification — " + f"the expected change is not observable in the environment. Do it again exactly as described.") self._emit("action_failed", tid, action_index=index, detail="verification failed after retry", **named) return None + async def _executor_turn(self, tid: str, trig: dict, index: int, name: str | None, executor: dict | None, + instruction: str, fire: dict) -> str | None: + """One call to the executor, listed in the trigger's ``calls`` with the end of its time limit while it runs. + + Calls in one conversation (the executor's URL and the trigger's context id) take turns: fires of one trigger + running side by side (``concurrent_fires``) wait for each other here, by design, so two turns never resume the + executor's session (a Claude Code conversation, say) at once; only their delays and hours run side by side. + ``fire`` (``_in_flight_fire``) is counted as being realized only while its call runs, so the calls the executor + makes for another fire while this one waits for its turn are never credited to this one.""" + context_id = f"trigger-{tid}" + if executor is None: + return await self._call_executor(executor, instruction, context_id=context_id) + timeout = self._executor_timeout(name, executor) + if timeout != executor["timeout_seconds"]: + executor = {**executor, "timeout_seconds": timeout} + lock = self._executor_locks.setdefault((executor["a2a_url"], context_id), asyncio.Lock()) + async with lock: + fires = self._in_flight.setdefault(executor["role"], []) + for other in fires: # an executor's calls carry only its role, so fires it realizes at once share counts + other["shared_with"].add(tid) + fire["shared_with"].add(other["cause"]["trigger_id"]) + fires.append(fire) + call = {"executor": name, "action_index": index, "deadline": _monotonic() + timeout} + trig["calls"].append(call) + try: + return await self._call_executor(executor, instruction, context_id=context_id) + finally: + trig["calls"].remove(call) + fires.remove(fire) + def caused_by(self, role: str) -> list[dict]: """The fires in flight for the executor with ``role``, all of them when several overlap; its tool calls are attributed to them. A fire is the trigger's n-th, failed ones counted.""" @@ -1509,20 +1683,18 @@ async def _delay(self, tid: str, trig: dict, index: int, action: dict, since: fl mean = _parse_duration(delay["mean"]).total_seconds() seconds = trig["delay_rngs"][index].expovariate(1.0 / mean) self._emit("delayed", tid, action_index=index, virtual_seconds=round(seconds, 3), **_named_executor(action)) - trig["waiting"] = True - trig["resumes"] = since + seconds + until = since + seconds + token = _begin_wait(trig, until) try: - await self._sleep_virtual(seconds - (self._gw._clock.elapsed() - since)) + await self._sleep_virtual(until) finally: - trig["waiting"] = False - trig["resumes"] = None + _end_wait(trig, token) async def _await_hours(self, tid: str, trig: dict, index: int, hours: dict, named: dict) -> None: """Wait until the executor's working hours are open now, reckoned each pass; not while the clock is unarmed.""" clock = self._gw._clock gen = opens = None - trig["waiting"] = True - trig["hours"] = hours + token = _begin_wait(trig, None, hours) try: while True: current = clock.generation # read before now(), so a re-arm in between is caught on the next pass @@ -1532,20 +1704,30 @@ async def _await_hours(self, tid: str, trig: dict, index: int, hours: dict, name return if current != gen or opening != opens: gen, opens = current, opening - trig["resumes"] = opens + trig["waits"][token]["resumes"] = opens self._emit("deferred", tid, action_index=index, until=_iso(opens), **named) - await self._sleep_virtual((opens - now).total_seconds(), generation=gen) + await self._sleep_virtual(opens, generation=gen) finally: - trig["waiting"] = False - trig["resumes"] = None - trig["hours"] = None - - async def _sleep_virtual(self, seconds: float, generation: int | None = None) -> None: - """Wait out ``seconds`` of virtual time as the clock runs (real time at its rate, 1 while unarmed, none while - frozen) or jumps ahead. Given the clock's ``generation``, stop early once a re-arm or clear moves it on.""" + _end_wait(trig, token) + + async def _sleep_virtual(self, until: float | datetime, generation: int | None = None) -> None: + """Wait until the clock gets to ``until``, a Clock.elapsed() reading or a virtual time on the current arm, as it + runs (real time at its rate, 1 while unarmed, none while frozen) or jumps ahead. ``until`` is the very mark the + waiting fire's resumes_at shows: a mark worked out here from a later reading of a running clock would lie past + it, and a clock frozen at resumes_at would hold the fire forever. Given the clock's ``generation``, stop early + once a re-arm or clear moves it on.""" clock = self._gw._clock - until = clock.elapsed() + seconds - while (left := until - clock.elapsed()) > 0 and (generation is None or clock.generation == generation): + while generation is None or clock.generation == generation: + if isinstance(until, datetime): + if (now := clock.now()) is None: + return + left = (until - now).total_seconds() + else: + left = until - clock.elapsed() + # A delay's resumes_at is its end to the microsecond, so an advance to it can fall a hair short; under a + # microsecond left is reached. + if left < _VIRTUAL_RESOLUTION_S: + return rate = clock.rate await self._nap(min(left / rate, _VIRTUAL_RECHECK_S) if rate else _VIRTUAL_RECHECK_S) @@ -1564,29 +1746,38 @@ async def _call_executor(self, executor: dict | None, instruction: str, context_ return None url = f"{executor['a2a_url']}/a2a" deadline = asyncio.get_running_loop().time() + executor["timeout_seconds"] - async with httpx.AsyncClient(timeout=30) as client: - send = await client.post(url, json={ + async with httpx.AsyncClient(timeout=_EXECUTOR_HTTP_TIMEOUT_S) as client: + # An executor holds message/send open until its task ends, so the read waits as long as the executor may + # take, and a little longer, so a reply that completes just past the time limit is still read. + read_s = max(_EXECUTOR_HTTP_TIMEOUT_S, executor["timeout_seconds"]) + _EXECUTOR_HTTP_TIMEOUT_S + send = await client.post(url, timeout=httpx.Timeout(_EXECUTOR_HTTP_TIMEOUT_S, read=read_s), json={ "jsonrpc": "2.0", "id": str(uuid.uuid4()), "method": "message/send", "params": {"message": {"role": "user", "messageId": uuid.uuid4().hex, "contextId": context_id, "parts": [{"kind": "text", "text": instruction}]}}, }) send.raise_for_status() - task_id = (send.json().get("result") or {}).get("id") + result = send.json().get("result") or {} + task_id = result.get("id") if not task_id: return None - while asyncio.get_running_loop().time() < deadline: - await asyncio.sleep(2) + status = result.get("status") or {} + # Polled until the time limit, then once more: a reply that completed in the last poll's gap is kept. + last = False + while status.get("state") not in _TERMINAL_A2A_STATES: + if last: + return None + last = asyncio.get_running_loop().time() >= deadline + if not last: + await asyncio.sleep(2) poll = await client.post(url, json={"jsonrpc": "2.0", "id": str(uuid.uuid4()), "method": "tasks/get", "params": {"id": task_id}}) poll.raise_for_status() status = ((poll.json().get("result") or {}).get("status") or {}) - if status.get("state") in _TERMINAL_A2A_STATES: - if status.get("state") != "completed": - return None - parts = (status.get("message") or {}).get("parts") or [] - return "\n".join(p.get("text", "") for p in parts if p.get("kind") == "text") - return None + if status.get("state") != "completed": + return None + parts = (status.get("message") or {}).get("parts") or [] + return "\n".join(p.get("text", "") for p in parts if p.get("kind") == "text") def _cancel(self, tid: str, trig: dict, **cause) -> None: """Retire a deadline its `unless` call answered, and the time triggers anchored on it, which can no longer fire.""" @@ -1598,6 +1789,27 @@ def _cancel(self, tid: str, trig: dict, **cause) -> None: self._cancel(dep_id, dep, anchor=tid) +def _uncount(trig: dict, ticket: dict) -> None: + """Take one fire out of ``trig``'s ``running``, once.""" + if ticket["counted"]: + ticket["counted"] = False + trig["running"] -= 1 + + +def _begin_wait(trig: dict, resumes: Any, hours: dict | None = None) -> object: + """Mark one fire of ``trig`` as waiting until ``resumes`` (a Clock.elapsed() reading, a datetime, or None until + known), for ``hours`` when it waits on its executor's working hours; returns the token ``_end_wait`` takes.""" + token = object() + trig.setdefault("waits", {})[token] = {"resumes": resumes, "hours": hours} + trig["waiting"] = True + return token + + +def _end_wait(trig: dict, token: object) -> None: + trig["waits"].pop(token, None) + trig["waiting"] = bool(trig["waits"]) + + def _interval_seconds(when: dict, rng: random.Random | None) -> float: """Virtual seconds from one mark of a recurring time trigger to the next, a stochastic one drawn from `rng`.""" every = when["every"] @@ -1615,6 +1827,12 @@ def _named_executor(action: dict) -> dict: return {"executor": action["executor"]} if action.get("executor") is not None else {} +def _replies_only(spec: dict) -> bool: + """Whether every action of a trigger is a contact's reply (``nl``) or a notification (``notify``, an app telling + the agent of the reply), the only kinds ``concurrent_fires`` runs side by side: neither reads and writes rows.""" + return all(action.get("type") in ("nl", "notify") for action in spec["actions"]) + + def _in_flight_fire(cause: dict) -> dict: """An nl fire as `_in_flight` holds it: its cause, its executor's calls by tool, and the triggers it overlapped.""" return {"cause": cause, "calls": {}, "failed_calls": {}, "shared_with": set()} diff --git a/src/agent_env/env/gateway/user_chat.py b/src/agent_env/env/gateway/user_chat.py new file mode 100644 index 000000000..b071c072d --- /dev/null +++ b/src/agent_env/env/gateway/user_chat.py @@ -0,0 +1,212 @@ +"""The user chat (urn:agentenv:user-chat/v1): how an always-on agent reaches its user. + +The roles a task names in `config` get `message_user`, and with `stay_quiet: true` also `stay_quiet`. Each message is +queued with the virtual time it was sent at; nothing here reaches the user by itself: the always_on step `take`s what +was sent after each agent turn and hands it to the user sim when the user would read it. Whatever else the agent writes +(its plain reply) never reaches the user. The tools are built in: once a role is named, role rules from +modify_env_tool_access don't hide them. + +`take` hands each message out once: {"id", "from", "at", "text"}, plus, with stay_quiet on, the `quiet` calls since the +last take. A `take_id` makes a retry safe. With `from` (a role) it takes only that role's messages and quiet calls, and +leaves the others queued. +""" +from __future__ import annotations + +import threading +from typing import Any, Callable + +from mcp.server.fastmcp.tools import Tool as FastMCPTool +from mcp.server.fastmcp.utilities.func_metadata import FuncMetadata +from mcp.types import CallToolResult, TextContent, ToolAnnotations + +from .caller import Caller +from .clock import Clock, _iso + +MESSAGE_USER_TOOL_NAME = "message_user" +STAY_QUIET_TOOL_NAME = "stay_quiet" +_MAX_TEXT_CHARS = 20000 +_MAX_TAKE_ID_CHARS = 128 +_KEPT_TAKES = 64 +_CONFIG_FIELDS = ("roles", "user_name", "stay_quiet") + +MESSAGE_USER_DESCRIPTION = ( + "Send a chat message to {user}, the person you work for. This is the only way they see anything from you: your " + "normal replies are kept in a log they never read. Send a message when there is something they need to know or " + "decide; send nothing when there isn't. You can send several messages in one turn. They read messages when " + "they're free, so a reply can take a while; you are woken when it arrives." +) +STAY_QUIET_DESCRIPTION = ( + "Call this when there is nothing worth telling {user} right now. It sends nothing; it records that you chose to " + "stay quiet this time." +) +_QUIET_SCHEMA: dict[str, Any] = {"type": "object", "properties": {}, "additionalProperties": False} +_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"text": {"type": "string", "description": "The message, as you'd text it."}}, + "required": ["text"], + "additionalProperties": False, +} + + +# Read-only for the world: they queue a message (or a note) for the harness and change no app's data, so, like get_time, +# they don't make the trigger engine re-evaluate every armed state trigger (TriggerEngine._is_readonly). An action +# trigger on either still fires. +_ANNOTATIONS = ToolAnnotations(readOnlyHint=True) + + +class UserChatError(ValueError): + """A bad config, take or tool argument.""" + + +class UserChat: + """The queue of messages roles sent their user; threadsafe.""" + + def __init__(self, clock: Callable[[], Clock]): + self._clock_of = clock + self._lock = threading.Lock() + self._roles: tuple[str, ...] = () + self._user_name = "your user" + self._queue: list[dict] = [] + self._sent = 0 + self._takes: dict[str, dict] = {} + self._stay_quiet = False + self._quiet: list[dict] = [] + + @property + def roles(self) -> tuple[str, ...]: + return self._roles + + @property + def user_name(self) -> str: + return self._user_name + + @property + def enabled(self) -> bool: + return bool(self._roles) + + @property + def stay_quiet(self) -> bool: + """Whether the configured roles also get stay_quiet (off unless a config turns it on).""" + return self.enabled and self._stay_quiet + + def configure(self, body: Any) -> dict: + """Name the roles that get message_user (`roles: []` turns it off) and how the tool names the user.""" + if not isinstance(body, dict): + raise UserChatError("body must be a JSON object") + unknown = sorted(set(body) - set(_CONFIG_FIELDS)) + if unknown: + raise UserChatError(f"config takes {list(_CONFIG_FIELDS)}, not {unknown}") + roles = body.get("roles", []) + if not isinstance(roles, list) or not all(isinstance(r, str) and r.strip() for r in roles): + raise UserChatError("roles must be a list of role names") + name = body.get("user_name") or "your user" + if not isinstance(name, str): + raise UserChatError("user_name must be a string") + stay_quiet = body.get("stay_quiet", False) + if not isinstance(stay_quiet, bool): + raise UserChatError("stay_quiet must be true or false") + with self._lock: + self._roles = tuple(dict.fromkeys(r.strip() for r in roles)) + self._user_name = name.strip() or "your user" + self._stay_quiet = stay_quiet + return self.state() + + def clear(self) -> dict: + with self._lock: + self._roles, self._user_name = (), "your user" + self._queue.clear() + self._takes.clear() + self._stay_quiet = False + self._quiet.clear() + return {"ok": True} + + def send(self, text: Any, *, role: str) -> dict: + if role not in self._roles: + raise UserChatError("message_user is not available here") + if not isinstance(text, str) or not text.strip(): + raise UserChatError("text must be the message to send") + if len(text) > _MAX_TEXT_CHARS: + raise UserChatError(f"text must be at most {_MAX_TEXT_CHARS} characters") + now = self._clock_of().now() + with self._lock: + self._sent += 1 + item = {"id": f"user-message-{self._sent}", "from": role, "at": _iso(now) if now else None, + "text": text, "status": "pending"} + self._queue.append(item) + return {"sent": True, "id": item["id"], "at": item["at"]} + + def quiet(self, *, role: str) -> dict: + """stay_quiet: nothing is sent; the call is kept (with its virtual time) for the next take's `quiet`.""" + if role not in self._roles or not self._stay_quiet: + raise UserChatError("stay_quiet is not available here") + now = self._clock_of().now() + with self._lock: + self._quiet.append({"from": role, "at": _iso(now) if now else None}) + return {"ok": True, "sent_to_user": False} + + def state(self) -> dict: + with self._lock: + out = {"roles": list(self._roles), "user_name": self._user_name, "sent": self._sent, + "pending": [_public(m) for m in self._queue if m["status"] == "pending"]} + if self._stay_quiet: + out["stay_quiet"] = True + return out + + def take(self, body: Any = None) -> dict: + body = body if isinstance(body, dict) else {} + take_id = body.get("take_id") + if take_id is not None and (not isinstance(take_id, str) or not take_id or len(take_id) > _MAX_TAKE_ID_CHARS): + raise UserChatError(f"take_id must be a non-empty string of at most {_MAX_TAKE_ID_CHARS} characters") + sender = body.get("from") + if sender is not None and (not isinstance(sender, str) or not sender.strip()): + raise UserChatError("from must be a role name") + with self._lock: + if take_id is not None and take_id in self._takes: + return self._takes[take_id] + items = [m for m in self._queue if m["status"] == "pending" and sender in (None, m["from"])] + for m in items: + m["status"] = "taken" + self._queue = [m for m in self._queue if m["status"] == "pending"] + result = {"items": [_public(m) for m in items]} + if self._stay_quiet: + result["quiet"] = [q for q in self._quiet if sender in (None, q["from"])] + self._quiet = [q for q in self._quiet if sender not in (None, q["from"])] + if take_id is not None: + self._takes[take_id] = result + while len(self._takes) > _KEPT_TAKES: + del self._takes[next(iter(self._takes))] + return result + + +def _public(m: dict) -> dict: + return {k: v for k, v in m.items() if k != "status"} + + +def build_user_chat_tool(chat: UserChat, make_arg_model: Callable[[str, dict], Any]) -> FastMCPTool: + """message_user, built by hand like get_time so a backing server's tool can't silently shadow it.""" + + async def message_user(text: str) -> dict | CallToolResult: + try: + return chat.send(text, role=Caller.current().role) + except UserChatError as e: + return CallToolResult(isError=True, content=[TextContent(type="text", text=str(e))]) + + return FastMCPTool(fn=message_user, name=MESSAGE_USER_TOOL_NAME, + description=MESSAGE_USER_DESCRIPTION.format(user=chat.user_name), parameters=_SCHEMA, + fn_metadata=FuncMetadata(arg_model=make_arg_model(MESSAGE_USER_TOOL_NAME, _SCHEMA)), + is_async=True, annotations=_ANNOTATIONS) + + +def build_stay_quiet_tool(chat: UserChat, make_arg_model: Callable[[str, dict], Any]) -> FastMCPTool: + """stay_quiet, built in like message_user; only registered when the config turns it on.""" + + async def stay_quiet() -> dict | CallToolResult: + try: + return chat.quiet(role=Caller.current().role) + except UserChatError as e: + return CallToolResult(isError=True, content=[TextContent(type="text", text=str(e))]) + + return FastMCPTool(fn=stay_quiet, name=STAY_QUIET_TOOL_NAME, + description=STAY_QUIET_DESCRIPTION.format(user=chat.user_name), parameters=_QUIET_SCHEMA, + fn_metadata=FuncMetadata(arg_model=make_arg_model(STAY_QUIET_TOOL_NAME, _QUIET_SCHEMA)), + is_async=True, annotations=_ANNOTATIONS) diff --git a/src/agent_env/explorer/ui/src/components/task-instance-viewer.tsx b/src/agent_env/explorer/ui/src/components/task-instance-viewer.tsx index 17d435de0..9c3a7a52c 100644 --- a/src/agent_env/explorer/ui/src/components/task-instance-viewer.tsx +++ b/src/agent_env/explorer/ui/src/components/task-instance-viewer.tsx @@ -467,6 +467,18 @@ export function TaskInstanceViewer({ () => parseTriggerRuntime(metadata), [metadata], ); + // always_on steps, by their type or by the run record they write under metadata.always_on[step_id]: a wake's closing + // text there is the agent's private note, never sent to the user, which the trajectory viewer labels as such. + const alwaysOnStepIds = useMemo(() => { + const record = metadata?.always_on; + const ids = new Set( + record && typeof record === 'object' ? Object.keys(record) : [], + ); + for (const s of taskSteps ?? []) { + if (s.type === 'always_on' && s.id) ids.add(s.id); + } + return ids; + }, [metadata, taskSteps]); // Authored trigger configs for the badge popovers. Empty when the surface passes slim step refs — popovers then show runtime history only. const authoredTriggers = useMemo( () => indexAuthoredTriggers(taskSteps), @@ -955,6 +967,9 @@ export function TaskInstanceViewer({ diff --git a/src/agent_env/explorer/ui/src/components/trajectory-viewer.tsx b/src/agent_env/explorer/ui/src/components/trajectory-viewer.tsx index 65e7f0f79..a58c7e59a 100644 --- a/src/agent_env/explorer/ui/src/components/trajectory-viewer.tsx +++ b/src/agent_env/explorer/ui/src/components/trajectory-viewer.tsx @@ -1804,6 +1804,9 @@ interface TrajectoryViewerProps { isInProgress?: boolean; // Object URL of this trajectory; only used to lazy-load screenshot-trimmed frames. screenshotBaseUri?: string; + // An always_on wake: its closing text is the agent's private note, never sent to the user (only message_user texts + // are), so it is labelled as such and shown quieter. + closingNote?: boolean; } export function TrajectoryViewer({ @@ -1811,6 +1814,7 @@ export function TrajectoryViewer({ timelineScrollRef, isInProgress = false, screenshotBaseUri, + closingNote = false, }: TrajectoryViewerProps) { const { model, @@ -2099,16 +2103,37 @@ export function TrajectoryViewer({ })()} {finalResponse && ( -
+
- {isInProgress ? 'Latest Response' : 'Final Response'} + {closingNote ? ( + + Closing note{' '} + + (private, not sent to the user) + + + ) : isInProgress ? ( + 'Latest Response' + ) : ( + 'Final Response' + )} {isInProgress && ( @@ -2116,7 +2141,13 @@ export function TrajectoryViewer({ )}
-
+
diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index a2e9efc50..067013b39 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -135,6 +135,9 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "register_agent_triggers": _needs_the_agent, "browser_connect_cdp": _needs_the_agent, "snapshot_agent_state": _needs_the_agent, + # An outside agent can't yet be woken by always_on's loop, nor play its user sim (the agent user_sim names). + "always_on": _needs_the_agent, + "user_sim": _needs_the_agent, "collect_artifacts": _collect_artifacts, "register_env_triggers": _register_env_triggers, "peer_agents": _peer_agents, diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 1b9a3f298..48d7971a5 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -54,6 +54,8 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep from agent_env.task_step.task_steps.modify_env_tool_access import ModifyEnvToolAccessStep from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + from agent_env.task_step.task_steps.always_on import AlwaysOnTaskStep + from agent_env.task_step.task_steps.user_sim import UserSimTaskStep from agent_env.task_step.task_steps.register_env_triggers import RegisterEnvTriggersStep from agent_env.task_step.task_steps.register_agent_triggers import RegisterAgentTriggersStep from agent_env.task_step.task_steps.sync_env_clock import SyncEnvClockTaskStep @@ -119,6 +121,8 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(LoadArtifactTaskStep): LoadArtifactTaskStep, _get_type(ModifyEnvToolAccessStep): ModifyEnvToolAccessStep, _get_type(PromptAgentTaskStep): PromptAgentTaskStep, + _get_type(AlwaysOnTaskStep): AlwaysOnTaskStep, + _get_type(UserSimTaskStep): UserSimTaskStep, _get_type(RegisterEnvTriggersStep): RegisterEnvTriggersStep, _get_type(RegisterAgentTriggersStep): RegisterAgentTriggersStep, _get_type(SyncEnvClockTaskStep): SyncEnvClockTaskStep, diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py new file mode 100644 index 000000000..84947bbe9 --- /dev/null +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -0,0 +1,1960 @@ +"""The always_on and user_sim steps: a task that runs over simulated days, the way you'd hand a job to a personal +assistant, with an LLM user sim. + +The ``user_sim`` step (``user_sim.py``, which documents the interface this step uses) says who the user is and records +it in the run's metadata under its ``name``; a ``deploy_agent`` step deploys the agent that plays them. The +``always_on`` step names it with ``user_sim`` and runs the loop: + +- The first wake is the user's request. Each wake opens with what came due for the agent on notifications/v1 (a + trigger's ``notify``, a scheduled check), labelled ``[time · channel]``, then the user's own messages, each labelled + with its time; while the task's clock is set, the agent can read it with the gateway's ``get_time``. +- Only what the agent sends with the gateway's ``message_user`` tool (user-chat/v1) reaches the user; its plain reply + goes to the log. Each wake should end with ``message_user`` or ``stay_quiet``: one that ends with closing text and + neither gets one ``nudge`` turn. ``stay_quiet`` and the nudge are on unless the task turns them off; the rule itself is + in the agent's system prompt. A reminder at the end of every wake (``wake_reminder``) is off by default: with + ``stay_quiet`` on, no wake in Opus 5.5 and GPT 6.1 runs of Surprise 30th ended in a message left in closing text. + The step takes only its agent's role's messages: another role's stay queued, and what the role sent before the run + is recorded apart (``user_chat_before_run``), never read by the user. +- The user sim is asked only after the agent sent something, at the time the user would read it (``schedule``), with + everything sent meanwhile in one read: the time, the agent's messages, what triggers sent the user's phone + (``notify`` to ``user_phone``) and the user's own lines (the opening request, ``user_says`` lines, ``final``). + Nothing meant for the agent is in it. Its answer is reply / no_reply / done; a reply, or a done with text, is held + for the response time. A schedule and response times are task time, so a user sim that sets either needs an armed + clock (sync_env_clock): without one the step refuses to start. Done ends a one-sitting run (after the agent's turn on its text); a run whose end_time falls + on a later day than its start goes on to end_time. +- Scripted user lines are triggers' ``notify`` actions to the ``user_says`` recipient: sent to the agent as the + user's message at their time. +- The clock runs at the task's speed (sync_env_clock) only during the agent's turns: the step freezes it between them + (clock/v1 set_rate), so settles and reads don't move it, and puts the task's rate back at the end. When nothing is + due, the step advances it to the earliest of: the next trigger event, the next notification, a held reply, a + pending read, or end_time, never past one; then waits for the world to settle. A contact's reply (an AI-played contact, a trigger's executor) counts as in + progress until it ends or its time limit passes; other trigger work gets ``settle_timeout_seconds``. Contacts whose + task sets no time limit get ``contact_timeout_seconds``, and one trigger's replies run side by side + (``contact_replies_in_parallel``). What comes due in the same minute wakes the agent once. +- The agent can schedule its own wake-ups (scheduled checks) when the run is multi-day, at least + ``scheduled_check_min_interval`` apart; ``scheduled_checks`` turns them on or off whatever the run's length. A + check's wake counts toward ``max_wakes``. +- At end_time there is always a final check-in: the user's ``final`` line, or the step's own (``FINAL_DEFAULT_TEXT``) + when the task sets none. An agent's turn that ends past end_time is followed by it, with whatever came due (a read + due before end_time goes first), never by an ordinary wake. The run's last wake, at end_time or the one + ``max_wakes`` makes the last, carries the last-turn notice (``final_reminder``). After it the user sim reads what + the agent sent (its wrap-up), no earlier than the task's time when the run ended, and the user's reaction, and any + reply still held, are recorded; then the run ends. +- A real-time budget keeps the run inside the hub's 8-hour job (``effective_time_budget`` gives the rule): once one + more wake might end past ``time_budget_seconds`` (7 h, lowered to fit the task's own timeouts), the agent gets its + closing wake, with the step's wrap-up line and the last-turn notice, and the run ends with end reason time_budget. +- End reasons: end_time, user_done (one sitting), max_wakes, time_budget, finished_nothing_pending or stuck (nothing + can wake the agent, with nothing pending or with something still pending), agent_turn_failed, agent_turn_timeout, + user_sim_error, step_error. With no end_time, a world that only repeats ends the run too (``IDLE_JUMPS_MAX``, in the + loop's docstring), so a recurring trigger that never wakes the agent can't move the clock forever. What was still pending is recorded (``pending_at_end``): replies held, messages the user + never read, user lines and alerts never delivered, the agent's checks not yet due, contact replies in progress. +- Graders read what the user saw. The prompt response is every message_user text in order, each with its pretend + time, never the agent's closing text; messages no turn took (a turn that timed out after sending) are taken at the + end and included, and listed as never read. The conversation (``metadata["a2a_conversations"][step_id]``) holds the user's + own messages and the agent's message_user texts, then the user's last word that never reached the agent and an end + marker with the end reason. An agent turn that doesn't complete (failed, canceled, rejected, timed out) ends the run + and sets the response's ``error_type`` (as prompt_agent does for a failed one), so verifiers skip the run. +- Who did what, and in what order, even under a frozen clock: each app change (urn:agentenv:changes/v1) is recorded + with who made it and the wake it was seen in, the agent's own with the wake that made it (``app_changes`` events); + each reply records the agent's message it answers (``reply_to``: its wake, time and text); and each read leaves a + record in ``metadata["post_turn_hooks"][step_id]`` in the shape the old psi-wake hook wrote, ``user_reads_at`` + being when the reply reached the agent, so checks built on those records grade the run instead of skipping it. A + reply that never reached it (her last read's, or one still held at the end) has ``delivered`` false and no + ``user_reads_at``. +- The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report + back); the full timeline goes to the object store, at the summary's ``object_url``. +- The agent: the message_user rules go in its system prompt when the step sets one and the agent's card takes it, + else on its first wake; its session is the step's ``context_id`` when set, as prompt_agent's is, else the run's own. +- When the step ends it puts back the gateway settings it changed for the run (the user chat's roles and stay_quiet, + the triggers' concurrent_fires and executor time limit, the roles with scheduled checks, the clock's rate), so the + task's later steps see them as they were. +""" +from __future__ import annotations + +import asyncio +import importlib.util +import json +import logging +import os +import re +import sys +import tempfile +import time +import uuid +from datetime import datetime, timedelta, timezone +from types import ModuleType +from typing import Any, ClassVar, Optional +from zoneinfo import ZoneInfo + +from a2a.types import TaskState + +from agent_env.a2a_agent import A2AAgent, conversation_store, protocol +from agent_env.a2a_agent.object_transfer import send_and_wait +from agent_env.artifact import FileArtifact +from agent_env.env.env import EnvCapabilityUnsupported +from agent_env.env.gateway.notifications import DEFAULT_MAX_ACTIVE +from agent_env.env.gateway.constants import ( + CHANGES_PAGE_MAX, + DEFAULT_ROLE, + EXT_CHANGES_URI, + EXT_CLOCK_URI, + EXT_NOTIFICATIONS_URI, + EXT_STATE_URI, + EXT_TRIGGERS_URI, + EXT_USER_CHAT_URI, +) +from agent_env.store import DuplicateKeyError, get_config +from agent_env.task_step.context import PromptResponse, TaskStepContext +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps import user_sim_versions +from agent_env.task_step.task_steps.user_sim import USER_SIMS_KEY +from agent_env.task_step.task_steps.notification_delivery import delivered_parts, label_time, take_due +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + +logger = logging.getLogger(__name__) + +ALWAYS_ON_KEY = "always_on" +USER_SAYS = "user_says" +USER_PHONE = "user_phone" +IN_FLIGHT = ("firing", "queued") +POLL_SECONDS = 0.5 +DAYS = ("mon", "tue", "wed", "thu", "fri", "sat", "sun") +FINAL_REMINDER = ("(This is the last time you'll be woken for this job. Only what you send with message_user reaches " + "{first}; your normal reply is not shown to them.)") +NUDGE_TEXT = ("Your closing text was not delivered. If the user should see it, send it with message_user; otherwise " + "call stay_quiet.") +WAKE_REMINDER_TEXT = "Reminder: only message_user reaches the user." +MESSAGE_RULES = ("The user never sees your normal replies. The only way to reach them is message_user. Text outside it " + "goes to a private log.") +QUIET_RULE = "If nothing is worth saying, call stay_quiet." +CHECKS_RULE = ("You only run when something wakes you. To do something at a later time, schedule a check with " + "schedule_check: you'll be woken then with your note.") +CHECK_MIN_INTERVAL = "PT3H" +EMPTY_REPLIES = frozenset({"(The assistant's reply was empty.)"}) # a harness's stand-in for an empty final reply +# The hub runs a step as one job of at most this long, retrying one that runs over; the step keeps this much of it spare. +JOB_LIMIT_SECONDS = 8 * 3600 +JOB_HEADROOM_SECONDS = 600 +DEFAULT_TIME_BUDGET_SECONDS = 7 * 3600 +# The time limit of a contact (a trigger's executor) whose task sets none; the gateway's own default is 120 s. +CONTACT_TIMEOUT_SECONDS = 300.0 +# Past a contact's time limit, how long the settle wait still gives its call to come back. +CONTACT_GRACE_SECONDS = 30.0 +# The hard caps that bound one turn of the loop (wake_reserve_seconds adds them up): past an agent's or user sim's turn +# limit, how long send_and_wait's own retries and polls still get before the turn is cut off; how long a trajectory +# fetch may take; how many user sim turns one read may take; and what the step's own gateway and store calls (takes, +# changes, the chat, clock reads) are allowed per wake and per read, beyond which the loop's turn is cut off. +TURN_GRACE_SECONDS = 60.0 +TRAJECTORY_FETCH_SECONDS = 120.0 +READ_ASKS = 2 +GATEWAY_ALLOWANCE_SECONDS = 300.0 +# At the end, how long taking the agent's messages no turn took may take (the run's record comes first). +LEFTOVER_TAKE_SECONDS = 60.0 +# The final check-in's line when the task sets no `final`: the step's, not the user's. +FINAL_DEFAULT_TEXT = "Time's up on this job. Send {first} a short wrap-up: what's done, and anything still open." +NOTHING_CAN_WAKE = "nothing_can_wake_the_agent" # until finish() tells finished_nothing_pending from stuck +ERROR_ENDS = ("agent_turn_failed", "agent_turn_timeout", "user_sim_error", "step_error") +# What each end reason means, for the end marker in the graded conversation; {wakes} and {first} are filled in. +END_NOTES = { + "end_time": "The run reached its end time.", + "user_done": "{first} signed off.", + "max_wakes": "The run was cut off at its cap of {wakes} wakes.", + "time_budget": "The run was cut off by its real-time budget.", + "finished_nothing_pending": "Nothing could wake the assistant again, and nothing was pending.", + "stuck": "Nothing could wake the assistant again, though something was still pending.", + "agent_turn_failed": "The assistant's turn failed.", + "agent_turn_timeout": "The assistant's turn timed out.", + "user_sim_error": "The user simulator failed.", + "step_error": "The run stopped on an error.", +} +# The run's metadata keeps a summary; the full timeline goes to the object store. When storing fails, the summary keeps +# this many of the last events instead, without these fields. +# The records a post-turn hook keeps (metadata["post_turn_hooks"][step id]), written in the shape the old psi-wake hook +# wrote them, so checks that read its reply times (user_reads_at) grade a run on this step. +HOOK_RECORDS_KEY = "post_turn_hooks" +HOOK_NAME = "always_on" +CHANGE_SUMMARY_CHARS = 200 +# The app changes one read of an env's changelog keeps in the timeline; the rest are counted (counts.app_changes counts +# them all, by who made them). +CHANGES_KEPT_PER_READ = 200 +TIMELINE_TAIL = 20 +# With no end_time, the run ends once this many clock jumps in a row have been to repeating triggers' marks alone (see +# the loop's docstring): the world can go on ticking, but nothing in it has woken the agent. +IDLE_JUMPS_MAX = 24 +# The clock jumps between two wakes the timeline keeps one by one; the rest are folded into one entry. +JUMPS_KEPT_PER_GAP = 20 +PENDING_KEPT = 20 +PENDING_TEXT_CHARS = 300 +# What _small keeps of nesting: past this many levels, a list or dict is summed up as its count. +KEPT_NESTING_LEVELS = 6 +# A user sim message longer than this, or one with a markdown heading or table, reads like an assistant's reply: its +# read is flagged (looks_like_assistant). +ASSISTANT_LIKE_MESSAGE_CHARS = 1200 +BULKY_EVENT_FIELDS = ("input", "raw", "first_answer", "read", "parts_text", "changes") +# What the prompt response keeps of each wake's parts (source_agent_per_turn_prompt_parts): each text part cut to +# TURN_TEXT_CHARS, and past TURN_TEXT_TOTAL_CHARS in all, a note for the rest; every wake's full text is in the stored +# timeline (its wake event's parts_text). +TURN_TEXT_CHARS = 4000 +TURN_TEXT_TOTAL_CHARS = 400_000 + + +# ================================================================================================================== +class AlwaysOnTaskStep(PromptAgentTaskStep): + type: ClassVar[str] = "always_on" + entity_refs = () + field_scopes = {k: v for k, v in PromptAgentTaskStep.field_scopes.items() if k != "user_model"} + + def __init__(self, id: str, version: Optional[int], *, user_sim: str, end_time: Optional[str] = None, + final: Optional[str] = None, max_wakes: int = 60, settle_timeout_seconds: float = 300, + user_timeout_seconds: int = 300, final_reminder: Optional[str] = FINAL_REMINDER, + stay_quiet: bool = True, nudge: bool = True, nudge_text: str = NUDGE_TEXT, + wake_reminder: bool = False, wake_reminder_text: str = WAKE_REMINDER_TEXT, + scheduled_checks: Optional[bool] = None, scheduled_check_min_interval: str = CHECK_MIN_INTERVAL, + contact_timeout_seconds: Optional[float] = CONTACT_TIMEOUT_SECONDS, + contact_replies_in_parallel: bool = True, time_budget_seconds: Optional[float] = None, + model: Optional[str] = None, model_params: Optional[dict] = None, effort: Optional[str] = None, + max_thinking_tokens: Optional[int] = None, harness: Optional[str] = None, + trajectory_output_prefix: Optional[str] = None, **prompt_fields: Any): + for unsupported in ("max_conversation_turns", "user_agent_name", "user_a2a_url", "post_turn_hooks", + "user_output_format", "user_model", "snapshot_config"): + prompt_fields.pop(unsupported, None) + super().__init__(id, version, max_conversation_turns=max(1, int(max_wakes)), model=model, + model_params=model_params, effort=effort, max_thinking_tokens=max_thinking_tokens, + harness=harness, trajectory_output_prefix=trajectory_output_prefix, **prompt_fields) + if not user_sim: + raise ValueError("always_on needs user_sim, the name of a user_sim step's user") + if end_time is not None: + _end_instant(end_time, datetime.now(timezone.utc)) # validates + self.user_sim = user_sim + self.end_time = end_time + self.final = final + self.max_wakes = int(max_wakes) + self.settle_timeout_seconds = float(settle_timeout_seconds) + self.user_timeout_seconds = int(user_timeout_seconds) + self.final_reminder = final_reminder + self.stay_quiet = bool(stay_quiet) + self.nudge = bool(nudge) + self.nudge_text = nudge_text + self.wake_reminder = bool(wake_reminder) + self.wake_reminder_text = wake_reminder_text + self.scheduled_checks = scheduled_checks + self.scheduled_check_min_interval = scheduled_check_min_interval + if contact_timeout_seconds is not None and not float(contact_timeout_seconds) > 0: + raise ValueError(f"contact_timeout_seconds must be positive (got {contact_timeout_seconds!r})") + self.contact_timeout_seconds = None if contact_timeout_seconds is None else float(contact_timeout_seconds) + self.contact_replies_in_parallel = bool(contact_replies_in_parallel) + self.time_budget_seconds = None if time_budget_seconds is None else float(time_budget_seconds) + self.effective_time_budget() # validates + + _OWN = ("user_sim", "end_time", "final", "max_wakes", "settle_timeout_seconds", "user_timeout_seconds", + "final_reminder", "stay_quiet", "nudge", "nudge_text", "wake_reminder", "wake_reminder_text", + "scheduled_checks", "scheduled_check_min_interval", "contact_timeout_seconds", "contact_replies_in_parallel", + "time_budget_seconds") + _PROMPT_FIELDS = ("prompt", "parts", "prompt_id", "agent_name", "timeout_seconds", "trajectory_output_prefix", + "model", "model_params", "system_prompt", "max_turns", "output_format", "max_thinking_tokens", + "effort", "harness", "agentenv_tools", "poll_interval_seconds", "context_id") + + def settle_cap_seconds(self) -> float: + """The longest one settle waits, whatever is still at work (queued fires, nl actions in a row, a contact whose + own time limit is longer): a contact's time limit and grace, then settle_timeout_seconds.""" + contact = self.contact_timeout_seconds or CONTACT_TIMEOUT_SECONDS + return contact + CONTACT_GRACE_SECONDS + self.settle_timeout_seconds + + def read_cap_seconds(self) -> float: + """The longest one read's answer may take: READ_ASKS user sim turns, each with its grace and trajectory + fetch; an ask past it is refused, and an answer past it ends the read as a user sim error.""" + return READ_ASKS * (self.user_timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS) + + def wake_reserve_seconds(self) -> float: + """The longest one turn of the loop, or the run's closing, can take, each part hard-capped: a clock jump's + settle (settle_cap_seconds), then a wake (the agent's turn and, with nudges on, its nudge turn, each at most + timeout_seconds + TURN_GRACE_SECONDS plus a trajectory fetch; one settle; GATEWAY_ALLOWANCE_SECONDS of the + step's own calls), then a read (read_cap_seconds and GATEWAY_ALLOWANCE_SECONDS). The loop cuts off a turn of + its own that runs past this (a step error), so it is a real bound.""" + turn = self.timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS + wake = turn * (2 if self.nudge else 1) + self.settle_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS + return self.settle_cap_seconds() + wake + self.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS + + def effective_time_budget(self) -> float: + """The real time after which the run closes: ``time_budget_seconds``, else 7 h, lowered so the closing still + fits the hub's job. The rule: + + The hub runs a step as one job of at most 8 h (``JOB_LIMIT_SECONDS``) and retries one that runs over, which + would start the step again against a world whose clock has already moved. So the step never starts a wake or a + read that might end past the budget: once the time spent plus one wake's worst case + (``wake_reserve_seconds``) reaches it, it gives the agent its closing wake instead (whatever is due, the + step's wrap-up line and the last-turn notice), lets the user sim read the wrap-up, and ends with end reason + ``time_budget``. A run therefore ends by the budget plus one wake's worst case, and that must fit the job + with ``JOB_HEADROOM_SECONDS`` to spare: the default budget is lowered to make it fit, and a budget a task sets + that can't is refused, as is a wake whose worst case leaves no room for a budget at all.""" + reserve = self.wake_reserve_seconds() + room = JOB_LIMIT_SECONDS - JOB_HEADROOM_SECONDS - reserve + if room <= reserve: + raise ValueError(f"always_on: one wake can take up to {reserve:.0f}s (wake_reserve_seconds: a settle, the " + f"agent's turn and nudge, a settle and a read), so no run fits the hub's " + f"{JOB_LIMIT_SECONDS // 3600} h job; lower timeout_seconds or user_timeout_seconds") + if self.time_budget_seconds is None: + return min(DEFAULT_TIME_BUDGET_SECONDS, room) + if not 0 < self.time_budget_seconds <= room: + raise ValueError(f"always_on: time_budget_seconds={self.time_budget_seconds:.0f} can't fit: with one wake's " + f"worst case ({reserve:.0f}s) after it, the run must close within {room:.0f}s to end " + f"inside the hub's {JOB_LIMIT_SECONDS // 3600} h job") + return self.time_budget_seconds + + def to_dict(self) -> dict: + base = TaskStep.to_dict(self) + for k in self._PROMPT_FIELDS: + base[k] = getattr(self, k) + for k in self._OWN: + base[k] = getattr(self, k) + return base + + @classmethod + def from_dict(cls, data: dict) -> AlwaysOnTaskStep: + fields = {k: data[k] for k in cls._PROMPT_FIELDS if k in data and data[k] is not None} + own = {k: data[k] for k in cls._OWN if k in data and data[k] is not None} + return cls(**cls._base_from_dict(data), retry_config=data.get("retry_config"), **fields, **own) + + async def _execute(self, context: TaskStepContext) -> TaskStepContext: + run = _Run(self, context) + try: + await run.go() + finally: + try: + await run.restore() + finally: + run.close() + return context + + +# ================================================================================================================== +class _Run: + """One always-on run: its state, its timeline and its loop.""" + + def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): + self.step = step + self.ctx = context + self.timeline: list[dict] = [] + self.t0_real = time.time() + self.wakes = 0 + self.end_reason: Optional[str] = None + self.unread: list[dict] = [] # the agent's messages the user hasn't read + self.read_at: Optional[datetime] = None + self.read_mode = "now" + self.held: list[dict] = [] # user replies held for their response time: {deliver_at, text} + self.you_sent: list[dict] = [] # the user's own lines since their last read: {at, text} + self.history: list[dict] = [] # every read and its answer, for the plug-in + self.replies = 0 + self.gave_up = False + self.final_sent = False + self.last_reply_at: Optional[datetime] = None + self.per_turn_trajectory_urls: list[Optional[str]] = [] + self.per_turn_parts: list[Optional[list[dict]]] = [] + self.tool_calls: Optional[int] = None + self.all_messages: list[dict] = [] + self.final_task_id = "" + self.error: Optional[str] = None + self.error_type: Optional[str] = None # set as prompt_agent sets it, so verifiers skip a run that broke + self.error_code: Optional[str] = None + self.chat_pending: list[dict] = [] # the agent's message_user texts not yet in the graded conversation + self.reaction: Optional[dict] = None # her last read's answer when she said nothing + self.change_cursors: dict[str, tuple[Any, int]] = {} # env id -> (env, the last changelog id read) + self.change_counts = {"by_agent": 0, "by_others": 0, "no_actor": 0} + self.hook_records: list[dict] = [] + self._plugin_paths: list[str] = [] + self._plugin_modules: list[str] = [] + self.nudges = 0 + self.zone = "UTC" + self.multi_day = False + self.checks: dict = {"on": False} + self.signed_off = False # a one-sitting run's user said done: no more reads + self.contacts: dict = {} + self.last_settle: dict = {} + self.pending: dict = {} + self.end_note = "" + self.time_budget = step.effective_time_budget() + self.idle_jumps = 0 # clock jumps in a row to repeating triggers' marks alone, since the last wake + self.idle_since: Optional[str] = None + self.clock_jumps = 0 + self.jumps_in_gap = 0 # clock jumps since the last wake, the first JUMPS_KEPT_PER_GAP of them logged + self.folded: Optional[dict] = None + self.clock: dict = {} # the task's clock rate, and whether the step freezes it between turns + self.clock_rates: list[tuple[Any, float]] = [] # (env, the task's rate) for each clock the step freezes + self.undo: list[tuple[str, Any]] = [] # (what, coroutine function): the gateway settings to put back at the end + self.rules_in_system_prompt = False # whether the agent took the rules in its system prompt + + async def restore(self) -> None: + """Put back what the step changed on the gateways, latest first, so later steps of the task see the settings + they had before it: the user chat's roles, user name and stay_quiet; the triggers' concurrent_fires and + executor time limit; the roles with scheduled checks; the clock's rate. Each is tried on its own; one that + fails is logged.""" + while self.undo: + what, put_back = self.undo.pop() + try: + await put_back() + except Exception as e: + logger.warning(f"always_on: could not put back {what} after the run: {type(e).__name__}: {e}") + + def _undo_config(self, env, uri: str, what: str, body: dict) -> None: + """At the end, set ``env``'s ``uri`` config back to ``body``, what it was before the step changed it.""" + async def put_back() -> None: + await _call(env, uri, "config", body, attempts=3) + self.undo.append((f"{what} on {env.env_id}", put_back)) + + def close(self) -> None: + for p in self._plugin_paths: + if os.path.exists(p): + os.unlink(p) + for m in self._plugin_modules: + sys.modules.pop(m, None) + + def spent(self) -> float: + """Real seconds since the step started.""" + return time.time() - self.t0_real + + # ---- record ---------------------------------------------------------------------------------------------- + def log(self, event: str, **fields: Any) -> dict: + """Add an event to the run's timeline, which keeps it whole; the process log gets only its name, ids, times, + counts and sizes (``_log_view``), since texts and reads carry what people wrote.""" + entry = {"event": event, "real_time": datetime.now(timezone.utc).isoformat(), + "real_elapsed_s": round(time.time() - self.t0_real, 1), **fields} + self.timeline.append(entry) + logger.info(f"always_on {event}: " + json.dumps(_log_view(fields), default=str)) + return entry + + # ---- setup ----------------------------------------------------------------------------------------------- + async def go(self) -> None: + step, ctx = self.step, self.ctx + self.agent = next((a for a in ctx.deployed_agents if a.agent_name == step.agent_name), None) + if self.agent is None: + raise RuntimeError(f"Agent with name '{step.agent_name}' not found in context.deployed_agents") + self.target_url = self.agent.a2a_url or self.agent.api_url + self.recipient = self.agent.role or DEFAULT_ROLE + us = (ctx.metadata.get(USER_SIMS_KEY) or {}).get(step.user_sim) + if us is None: + raise RuntimeError(f"always_on: no user_sim step named '{step.user_sim}' ran before this step") + self.us = us + self.zone = _task_zone(us, ctx.metadata.get("clock_configurations") or []) + self.person = {**self._person(us), "time_zone": self.zone} + self.version = us["user_version"] + self.user_agent = next((a for a in ctx.deployed_agents if a.agent_name == us["agent_name"]), None) + if self.user_agent is None: + raise RuntimeError(f"always_on: the user sim agent '{us['agent_name']}' is not deployed") + self.user_url = self.user_agent.a2a_url or self.user_agent.api_url + self.plugin = await self._load_plugin(us.get("plugin")) + self.tz = ZoneInfo(self.zone) + + self.clock_envs = [d for d in ctx.deployed_envs if d.supports(EXT_CLOCK_URI, "state")] + self.notification_envs = [d for d in ctx.deployed_envs if d.supports(EXT_NOTIFICATIONS_URI, "take")] + self.chat_envs = [d for d in ctx.deployed_envs if d.supports(EXT_USER_CHAT_URI, "take")] + if not self.chat_envs: + raise RuntimeError(f"always_on: no deployed env offers {EXT_USER_CHAT_URI} (message_user); rebuild its " + "gateway from a branch that has user_chat.py") + timed = [k for k in ("schedule", "response_times") if us.get(k)] + if timed and not await self._armed(): + # Read times and reply delays are task time; with no clock to jump, the loop would wait them out in real + # time, spinning on the clock it can't move. + raise ValueError(f"always_on: user sim '{step.user_sim}' sets {' and '.join(timed)}, which need the " + "task's clock, and no env's clock is armed; run sync_env_clock before this step, or drop " + f"{' and '.join(timed)} from the user_sim step") + now = await self.now() + self.opening = {"at": _iso(now), "text": self._effective_prompt()} + self.person["opening"] = self.opening + t0 = await self._t0() or now + self.end = _end_instant(step.end_time, t0) if step.end_time else None + self.multi_day = self.end is not None and self.end.astimezone(self.tz).date() > t0.astimezone(self.tz).date() + self.person["done_ends_run"] = not self.multi_day # for setup, which writes her prompt before any read + chat_config = {"roles": [self.recipient], "user_name": self.person.get("name") or "your user"} + if step.stay_quiet: + chat_config["stay_quiet"] = True + for env in self.chat_envs: + before = await _call(env, EXT_USER_CHAT_URI, "state") if env.supports(EXT_USER_CHAT_URI, "state") else {} + self._undo_config(env, EXT_USER_CHAT_URI, "the user chat's config", + {"roles": before.get("roles") or [], "user_name": before.get("user_name") or "your user", + **({"stay_quiet": True} if before.get("stay_quiet") else {})}) + await _call(env, EXT_USER_CHAT_URI, "config", chat_config) + await self._configure_clock() + await self._configure_checks() + await self._configure_contacts() + await self._configure_agents() + await self._start_changes() + + self.conversation_id = uuid.uuid4().hex + try: + conversation_store.create_conversation(conversation_id=self.conversation_id, + task_instance_id=ctx.instance_id or "", + source_agent_name=us["agent_name"], + target_agent_name=step.agent_name) + except DuplicateKeyError: + pass + ctx.metadata.setdefault("a2a_conversations", {})[step.id] = self.conversation_id + self.user_context_id = uuid.uuid4().hex # one user sim session per run + # The agent's session: the step's context_id, as prompt_agent uses it, so a run can go on with the session an + # earlier step left; else a new one. The graded conversation is the run's own either way. + self.solver_context_id = step.context_id or uuid.uuid4().hex + + # From here the conversation exists, so whatever fails still closes it and leaves the run's record. + try: + self.hook_records.append({"hook": HOOK_NAME, "turn": 0, "started_at": _iso(await self.now())}) + ctx.metadata.setdefault(HOOK_RECORDS_KEY, {})[step.id] = self.hook_records + now = await self.now() + self.log("start", virtual_time=_iso(now), end_time=_iso(self.end) if self.end else None, + agent=step.agent_name, agent_role=self.recipient, user_sim=step.user_sim, version=self.version, + user_model=self.user_model, max_wakes=step.max_wakes, time_zone=self.zone, + multi_day=self.multi_day, scheduled_checks=self.checks, clock_rate=await self._rate()) + # What the agent's role sent before the run (a task that gave it message_user earlier) isn't this run's: + # set it aside, recorded, so the first wake doesn't take it as its own. + before, quiet = await self._take_messages() + if before or quiet: + self.log("user_chat_before_run", messages=before, stay_quiet_calls=len(quiet)) + await self.loop(now) + except Exception as e: + self.error = f"{type(e).__name__}: {e}" + self.end_reason = self.end_reason or "step_error" + self.log("error", error=self.error) + raise + finally: + await self.finish() + + def _person(self, us: dict) -> dict: + name = us.get("display_name") or us["name"] + return {"key": us["name"], "name": name, "first": us.get("first_name") or name.split()[0], + "email": us.get("email"), "time_zone": us.get("time_zone"), + "description": us.get("description"), "versions": us.get("versions") or {}, + "output_format": us.get("output_format"), + "permissions": us.get("permissions"), "keep_private": us.get("keep_private")} + + async def _load_plugin(self, spec: Optional[dict]) -> ModuleType: + if not spec: + return user_sim_versions + artifact = FileArtifact.get(spec["file_artifact_id"], spec.get("file_artifact_version")) + source = await asyncio.to_thread(artifact.load) + fd, path = tempfile.mkstemp(suffix=".py") + os.write(fd, source) + os.close(fd) + self._plugin_paths.append(path) + name = f"user_sim_plugin_{uuid.uuid4().hex[:8]}" + mspec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(mspec) + sys.modules[name] = module + self._plugin_modules.append(name) + mspec.loader.exec_module(module) + if hasattr(module, "configure"): + module.configure(spec.get("params") or {}) + return module + + async def _configure_agents(self) -> None: + step, ctx = self.step, self.ctx + model, effort = step._resolve_model_and_effort(ctx) + self.model = model + overrides = ctx.metadata.get("user_overrides", {}) + card = self.agent.a2a_card or {} + desired = {"model": model, "system_prompt": self._system_prompt(), "effort": effort, + "harness": ctx.agent_harness or step.harness, + "max_turns": overrides.get("agent_max_turns") or step.max_turns, + "max_thinking_tokens": overrides.get("agent_max_thinking_tokens") or step.max_thinking_tokens, + "output_format": step.output_format, "timeout_seconds": step.timeout_seconds, + "task_id": ctx.metadata.get("task_id"), "agentenv_tools": step.agentenv_tools, + "builtin_tools": card.get("builtin_tools")} + desired = {k: v for k, v in desired.items() if v is not None} + model_params = get_config().get_model_params(step.model_params) + if model_params: + desired["model_params"] = model_params + negotiated = A2AAgent.negotiate_agent_config(card, desired) + if negotiated: + await protocol.post_agent_config(self.target_url + negotiated.endpoint, negotiated.fields) + ctx.record_model("solver", step.id, negotiated.fields.get("model") if negotiated else None) + self.rules_in_system_prompt = bool(desired.get("system_prompt") and negotiated + and "system_prompt" in negotiated.fields) + if desired.get("system_prompt") and not self.rules_in_system_prompt: + logger.warning("always_on: the agent's card did not accept system_prompt, so the message_user rules go " + "on its first wake instead") + + setup = self.plugin.setup(self.version, self.person) + self.user_model, self.user_model_from = self._user_model() + user_config = {"model": self.user_model, "system_prompt": setup.get("system_prompt"), + "output_format": setup.get("output_format")} + user_config = {k: v for k, v in user_config.items() if v is not None} + negotiated = A2AAgent.negotiate_agent_config(self.user_agent.a2a_card or {}, user_config) + if negotiated: + await protocol.post_agent_config(self.user_url + negotiated.endpoint, negotiated.fields) + accepted = sorted(negotiated.fields) if negotiated else [] + ctx.record_model("user simulator", step.id, negotiated.fields.get("model") if negotiated else None) + self.user_setup = {"model": self.user_model, "model_from": self.user_model_from, + "user_sim_step_model": self.us.get("model"), + "model_set": "model" in accepted, # else the agent ran its own + "system_prompt_chars": len(setup.get("system_prompt") or ""), + "output_format": setup.get("output_format"), "accepted_fields": accepted} + if "system_prompt" not in accepted and setup.get("system_prompt"): + logger.warning("always_on: the user sim agent's card did not accept system_prompt; it keeps its deploy one") + + async def _configure_clock(self) -> None: + """The clock runs at the task's rate only while the agent works (FD-5366): freeze it (rate 0) now, and between + turns, so a slow contact reply or user sim read doesn't move the task's time. With clock/v1 ``set_rate``, + which keeps the clock's marks (a re-arm with set_time, as sync_env_clock arms it, would re-anchor every + relative or recurring time trigger); the task's rate is put back at the end. A clock already frozen (rate 0) + is left alone; a gateway without set_rate keeps running between turns, as the record says.""" + envs = [] + for env in await self._armed(): + state = await _call(env, EXT_CLOCK_URI, "state") + rate = float(state.get("virtual_seconds_per_real_second") or 0) + if rate <= 0: + continue + if not env.supports(EXT_CLOCK_URI, "set_rate"): + envs.append({"env_id": env.env_id, "rate": rate, "frozen_between_turns": False, + "why": "the gateway has no clock set_rate"}) + continue + self.clock_rates.append((env, rate)) + envs.append({"env_id": env.env_id, "rate": rate, "frozen_between_turns": True}) + if self.clock_rates: + self.undo.append(("the clock's rate", lambda: self._clock_runs(True))) + await self._clock_runs(False) + self.clock = {"envs": envs} + + async def _clock_runs(self, running: bool) -> None: + """Run each clock the step freezes at the task's rate (``running``), or freeze it.""" + for env, rate in self.clock_rates: + await _call(env, EXT_CLOCK_URI, "set_rate", {"virtual_seconds_per_real_second": rate if running else 0}, + attempts=3) + + async def _agent_turn(self, parts: list[dict], msg_id: str, record) -> tuple[str, dict]: + """One turn of the agent on ``parts``: the task's clock runs at its rate while the agent works, and is frozen + again after.""" + step = self.step + await self._clock_runs(True) + try: + # Cut off past its limit and grace, whatever send_and_wait's own retries do: a TimeoutError, as its own. + return await asyncio.wait_for( + send_and_wait(self.target_url, parts, agent=self.agent, message_id=msg_id, + context_id=self.solver_context_id, timeout_seconds=step.timeout_seconds, + poll_interval_seconds=step.poll_interval_seconds, before_send=record), + step.timeout_seconds + TURN_GRACE_SECONDS) + finally: + await self._clock_runs(False) + + async def _configure_contacts(self) -> None: + """On each env with triggers, let one trigger's replies run side by side (``contact_replies_in_parallel``) and + give contacts that set no time limit ``contact_timeout_seconds``; a gateway too old for it keeps its ways.""" + step = self.step + body = {"concurrent_fires": step.contact_replies_in_parallel} + if step.contact_timeout_seconds is not None: + body["executor_timeout_seconds"] = step.contact_timeout_seconds + applied = [] + for env in self.ctx.deployed_envs: + if not env.supports(EXT_TRIGGERS_URI, "state"): + continue + if not env.supports(EXT_TRIGGERS_URI, "config"): + applied.append({"env_id": env.env_id, "applied": False, "why": "the gateway has no triggers config"}) + continue + before = (await _call(env, EXT_TRIGGERS_URI, "state")).get("settings") or {} + self._undo_config(env, EXT_TRIGGERS_URI, "the triggers' settings", + {k: v for k, v in before.items() + if k in ("concurrent_fires", "executor_timeout_seconds") and v is not None}) + got = await _call(env, EXT_TRIGGERS_URI, "config", body) + applied.append({"env_id": env.env_id, "applied": True, "concurrent_fires": got.get("concurrent_fires"), + "executor_timeout_seconds": got.get("executor_timeout_seconds")}) + self.contacts = {"replies_in_parallel": step.contact_replies_in_parallel, + "timeout_seconds": step.contact_timeout_seconds, "envs": applied} + + def rules(self) -> str: + """What the agent is told about reaching the user, and about waking itself when it can.""" + return " ".join([MESSAGE_RULES, *([QUIET_RULE] if self.step.stay_quiet else []), + *([CHECKS_RULE] if self.checks["on"] else [])]) + + def _user_model(self) -> tuple[str, str]: + """The user sim agent's model for this run, and where it came from: the user_sim step's own ``step_params`` + model, else the run's ``user_model`` (``task run`` / ``eval run --user-model``, a bundle's ``user_model``, as + prompt_agent's ``_resolve_user_model`` honours it), else the user_sim step's ``model``.""" + overrides = self.ctx.metadata.get("user_overrides") or {} + own = ((overrides.get("step_params") or {}).get(self.us.get("step_id")) or {}).get("model") + if own: + return own, "user_sim_step_params" + if overrides.get("user_model"): + return overrides["user_model"], "run_user_model" + return self.us["model"], "user_sim_step" + + async def _configure_checks(self) -> None: + """Give the agent's role scheduled checks (notifications/v1) on each env whose clock is set: by default when + the run is multi-day, else as ``scheduled_checks`` says. A role the task already gave checks keeps its own + settings; other roles the task named keep theirs.""" + step = self.step + wanted = self.multi_day if step.scheduled_checks is None else step.scheduled_checks + envs = [e for e in await self._armed() if e.supports(EXT_NOTIFICATIONS_URI, "config")] + applied = [] + for env in envs: + state = await _call(env, EXT_NOTIFICATIONS_URI, "state") + roles = list(state.get("roles") or []) + if self.recipient in roles: + applied.append({"env_id": env.env_id, "set_by": "task", "min_interval": state.get("min_interval"), + "max_active": state.get("max_active")}) + elif wanted: + body = {"roles": [*roles, self.recipient], + "max_active": state.get("max_active", DEFAULT_MAX_ACTIVE) if roles else DEFAULT_MAX_ACTIVE, + "min_interval": state.get("min_interval") if roles else step.scheduled_check_min_interval} + self._undo_config(env, EXT_NOTIFICATIONS_URI, "the roles with scheduled checks", + {"roles": roles, **{k: state[k] for k in ("max_active", "min_interval") + if state.get(k) is not None}}) + got = await _call(env, EXT_NOTIFICATIONS_URI, "config", body) + applied.append({"env_id": env.env_id, "set_by": "step", "min_interval": got.get("min_interval"), + "max_active": got.get("max_active")}) + self.checks = {"on": bool(applied), "envs": applied} + + def _system_prompt(self) -> Optional[str]: + """The step's system prompt with the rules after it; without one, or when the agent's card doesn't accept it, + the agent keeps its own and the first wake opens with the rules.""" + return f"{self.step.system_prompt.rstrip()}\n\n{self.rules()}" if self.step.system_prompt else None + + # ---- clock and world ------------------------------------------------------------------------------------ + async def _armed(self) -> list: + armed = [] + for env in self.clock_envs: + try: + if (await _call(env, EXT_CLOCK_URI, "state")).get("armed"): + armed.append(env) + except Exception as e: + logger.warning(f"clock state of {env.env_id} failed: {e}") + return armed + + async def now(self) -> datetime: + envs = await self._armed() + if not envs: + return datetime.now(timezone.utc) + return min([_utc((await _call(env, EXT_CLOCK_URI, "state"))["virtual_time"]) for env in envs]) + + async def _t0(self) -> Optional[datetime]: + envs = await self._armed() + return min([_utc((await _call(env, EXT_CLOCK_URI, "state"))["t0"]) for env in envs]) if envs else None + + async def _rate(self) -> Optional[float]: + envs = await self._armed() + return (await _call(envs[0], EXT_CLOCK_URI, "state")).get("virtual_seconds_per_real_second") if envs else None + + async def advance(self, target: datetime, reason: str) -> None: + before = await self.now() + if target <= before: + return + armed = await self._armed() + if not armed: # nothing to move: the loop would come straight back here, spinning in real time + raise RuntimeError(f"always_on: can't move the task's clock to {_iso(target)} ({reason}): no env's clock " + "is armed") + for env in armed: + try: + await _call(env, EXT_CLOCK_URI, "advance", {"virtual_time": _iso(target)}, attempts=2) + except Exception: + state = await _call(env, EXT_CLOCK_URI, "state") + if not (state.get("armed") and _utc(state["virtual_time"]) >= target): + raise + stuck = await self.settle() + self.clock_jumps += 1 + self.jumps_in_gap += 1 + if self.jumps_in_gap <= JUMPS_KEPT_PER_GAP: + self.log("clock_jump", **{"from": _iso(before), "to": _iso(target), "reason": reason, + "settled": not stuck, "still_at_work": stuck, "settle": self.last_settle}) + return + # Past the first jumps since the last wake, one entry sums the rest up, so a long gap can't grow the timeline. + if self.folded is None: + self.folded = self.log("clock_jumps_folded", count=0, **{"from": _iso(before)}, reasons={}, + unsettled=0, settle_seconds=0.0) + f = self.folded + f["count"] += 1 + f["to"] = _iso(target) + f["reasons"][reason] = f["reasons"].get(reason, 0) + 1 + f["unsettled"] += 1 if stuck else 0 + f["settle_seconds"] = round(f["settle_seconds"] + self.last_settle.get("seconds", 0), 1) + if stuck: + f["last_still_at_work"] = stuck + + async def settle(self) -> list[dict]: + """Wait until the world has settled: no trigger still at work, bar a fire waiting for a later time. A contact's + reply (a call to a trigger's executor) counts as in progress until it ends or its time limit passes (plus + ``CONTACT_GRACE_SECONDS``); any other work, an event due that an advance to now doesn't set going among it, + gets ``settle_timeout_seconds``, from the start or from the last contact call seen. A gateway that reports no + executor calls gets ``settle_timeout_seconds`` in all. However much is still at work (queued fires, nl + actions in a row, a contact whose own limit is longer), the wait never runs past ``settle_cap_seconds``. + Returns, by env, the triggers still at work when it gave up (empty when the world settled); ``last_settle`` + says how long it waited and how many contact calls it saw.""" + started = time.monotonic() + cap = started + self.step.settle_cap_seconds() # however much is still at work, never past this + envs = [e for e in await self._armed() if e.supports(EXT_TRIGGERS_URI, "state")] + waiting = {i: {"live_call_seen": started, "contact_until": started} for i in range(len(envs))} + stuck: list[dict] = [] + calls_seen = 0 + while waiting: + for i in sorted(waiting): + env, track = envs[i], waiting[i] + clock = await _call(env, EXT_CLOCK_URI, "state") + state = await _call(env, EXT_TRIGGERS_URI, "state") + now, mono = _utc(clock["virtual_time"]), time.monotonic() + rows = state.get("triggers", []) + live = [c for row in rows for c in row.get("executor_calls") or [] if c.get("seconds_left", 0) > 0] + if live: + calls_seen = max(calls_seen, len(live)) + track["live_call_seen"] = mono + track["contact_until"] = max(track["contact_until"], mono + CONTACT_GRACE_SECONDS + + max(c["seconds_left"] for c in live)) + busy = [row for row in rows if _busy(row, now)] + due = state.get("next_event_at") is not None and _utc(state["next_event_at"]) <= now + limit = min(cap, max(track["contact_until"], + track["live_call_seen"] + self.step.settle_timeout_seconds)) + if not due and not busy: + waiting.pop(i) + elif mono >= limit: # a due event that won't go, like other work, gets settle_timeout_seconds + stuck.append({"env_id": env.env_id, "triggers": [_at_work(row) for row in busy], + **({"due_not_going": state["next_event_at"]} if due else {})}) + waiting.pop(i) + elif due: # an event already due: an advance to now sets it going + try: + await _call(env, EXT_CLOCK_URI, "advance", {"virtual_time": _iso(now)}, attempts=1) + except Exception: + pass + if waiting: + self.ctx.heartbeat() + await asyncio.sleep(POLL_SECONDS) + self.last_settle = {"seconds": round(time.monotonic() - started, 1), "contact_calls_seen": calls_seen} + return stuck + + async def _next_world_event(self) -> Optional[datetime]: + out = [] + for env in await self._armed(): + if env.supports(EXT_TRIGGERS_URI, "state"): + st = await _call(env, EXT_TRIGGERS_URI, "state") + if st.get("next_event_at"): + out.append(_utc(st["next_event_at"])) + return min(out) if out else None + + async def _trigger_rows(self) -> list[dict]: + rows = [] + for env in await self._armed(): + if env.supports(EXT_TRIGGERS_URI, "state"): + rows += (await _call(env, EXT_TRIGGERS_URI, "state")).get("triggers", []) + return rows + + async def _only_repeating(self) -> bool: + """Whether all the world has ahead is recurring time triggers' marks: no trigger has a fire in flight (running, + queued, waiting out a delay or hours, calling a contact, evaluating) and none has a one-off mark to come.""" + for row in await self._trigger_rows(): + if (row.get("status") in IN_FLIGHT or row.get("running") or row.get("pending") or row.get("waits") + or row.get("waiting") or row.get("executor_calls") or row.get("evaluating")): + return False + when = row.get("when") or {} + if row.get("type") == "time" and row.get("status") == "armed" and "every" not in when \ + and (row.get("next_mark") or "after" not in when): + return False + return True + + async def _repeating_ids(self) -> list[str]: + return sorted({row["id"] for row in await self._trigger_rows() if row.get("type") == "time" + and row.get("status") == "armed" and "every" in (row.get("when") or {})}) + + async def _notes_state(self, to: str) -> list[dict]: + out = [] + for env in self.notification_envs: + if env.supports(EXT_NOTIFICATIONS_URI, "state"): + out.append(await _call(env, EXT_NOTIFICATIONS_URI, "state", {"to": to})) + return out + + async def _due_count(self, to: str, now: datetime) -> int: + n = 0 + for st in await self._notes_state(to): + n += sum(1 for c in st.get("checks", []) if c.get("status") == "active" and _utc(c["next"]) <= now) + n += sum(1 for p in st.get("pending_notifications", []) if not p.get("at") or _utc(p["at"]) <= now) + return n + + async def _next_due(self, to: str, now: datetime) -> Optional[datetime]: + """When the next check or notification for ``to`` comes due after ``now``.""" + times = [] + for st in await self._notes_state(to): + times += [_utc(c["next"]) for c in st.get("checks", []) if c.get("status") == "active"] + times += [_utc(p["at"]) for p in st.get("pending_notifications", []) if p.get("at")] + return min((t for t in times if t > now), default=None) + + # ---- the user's schedule --------------------------------------------------------------------------------- + def _schedule(self) -> Optional[dict]: + return self.us.get("schedule") or None + + def _available(self, t: datetime) -> tuple[datetime, str]: + """When the user would first look at a message that arrived at ``t``, and in what mode (morning, work, free): + while awake and not busy. Sleep may run past midnight (wake 08:00, sleep 00:30). No schedule: at once, mode + "now".""" + sched = self._schedule() + if not sched: + return t, "now" + tz = self.tz + wake, sleep = _hm(sched.get("wake", "07:30")), _hm(sched.get("sleep", "23:30")) + work = [_hm(x) for x in sched.get("work", ["09:00", "18:00"])] + work_days = set(sched.get("work_days", DAYS[:5])) + busy = [(datetime.fromisoformat(b["start"]).replace(tzinfo=tz), datetime.fromisoformat(b["end"]).replace(tzinfo=tz)) + for b in sched.get("busy") or []] + lt = t.astimezone(tz) + mode = None + # Each pass moves past a busy block or a night, or ends: a block once passed is behind, and a night always ends + # at a waking time; so the passes are bounded by two per block and one more. + for _ in range(2 * len(busy) + 2): + moved = False + for start, stop in busy: + if start <= lt < stop: + lt, moved = stop, True + hm = (lt.hour, lt.minute) + if _asleep(hm, wake, sleep): + day = lt.date() if hm < wake else lt.date() + timedelta(days=1) + lt, mode, moved = datetime(day.year, day.month, day.day, *wake, tzinfo=tz), "morning", True + if not moved: + break + else: + raise RuntimeError(f"always_on: the user's schedule never frees a time to read after {_iso(t)}") + if mode is None: + hm = (lt.hour, lt.minute) + mode = "work" if DAYS[lt.weekday()] in work_days and work[0] <= hm < work[1] else "free" + return lt.astimezone(timezone.utc), mode + + def _latency(self, mode: str) -> timedelta: + rt = self.us.get("response_times") or {} + if mode == "now" or not rt: + return timedelta(0) + return timedelta(minutes=float(rt.get(mode, rt.get("free", 0)))) + + async def _schedule_read(self) -> None: + if not self.unread or self.read_at is not None or self.gave_up or self.signed_off: + return + # A message has no time when the chat env's clock is unarmed; the task's time (another env's clock, else the + # computer's) stands in for it. + first_at = min(_utc(m["at"]) for m in self.unread if m.get("at")) if any(m.get("at") for m in self.unread) \ + else await self.now() + at, mode = self._available(first_at) + if self.last_reply_at and at < self.last_reply_at: + at, mode = self._available(self.last_reply_at) + self.read_at, self.read_mode = at, mode + self.log("user_read_scheduled", read_at=_iso(at), mode=mode, unread=len(self.unread)) + + # ---- the loop ------------------------------------------------------------------------------------------- + async def loop(self, now: datetime) -> None: + """Until the run ends: a read that is due, else a wake for what is due, else a clock jump to what comes next. + + A world that only repeats: with no end_time, when nothing the step tracks is ahead (no alert or check for the + agent, no user line, no held reply, no read) and no trigger has work in flight or a one-off mark to come, the + world can only go on ticking: its recurring triggers fire again and again. Each such jump after which nothing + woke the agent counts; ``IDLE_JUMPS_MAX`` of them in a row end the run (``nothing_can_wake``, which ``finish`` + turns into finished_nothing_pending or stuck). A wake or a read starts the count again. The rule only ever ends + a run in which a recurring trigger has fired that many times running without waking the agent, and it can't + end one that waits on a known event; with an end_time the run goes on to it, its jumps folded in the timeline + past ``JUMPS_KEPT_PER_GAP`` per gap.""" + step = self.step + await self._capped(self._open(), "the opening wake") + while self.end_reason is None: + if self.wakes >= step.max_wakes: + self.end_reason = "max_wakes" + break + if not await self._capped(self._next(), "a turn of the loop"): + break + await self._capped(self._last_read(), "the last read", step.read_cap_seconds() + GATEWAY_ALLOWANCE_SECONDS) + + async def _capped(self, work, what: str, seconds: Optional[float] = None): + """``work``, cut off past ``seconds`` (one turn of the loop's worst case, ``wake_reserve_seconds``): every part + is capped on its own, so only calls hanging past their allowance get here, and the run ends on a step error + rather than run past the hub's job.""" + seconds = self.step.wake_reserve_seconds() if seconds is None else seconds + timer = asyncio.timeout(seconds) + try: + async with timer: + return await work + except TimeoutError: + if not timer.expired(): + raise + self.end_reason = "step_error" + raise RuntimeError(f"always_on: {what} ran past its worst case ({seconds:.0f}s, wake_reserve_seconds)") + + async def _open(self) -> None: + """The opening wake: the user's request, with what came due for the agent.""" + self.you_sent.append(dict(self.opening)) + opened_at = _utc(self.opening["at"]) + await self._chat_user(self._opening_parts(lambda text: self._chat_part(opened_at, text, kind="opening"))) + due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] + await self.wake([*delivered_parts(due), + *self._opening_parts(lambda text: self._user_part(opened_at, text, opening=True))], + woke_by=[f"notification:{d['channel']}" for d in due] + ["user_request"], + notifications=[{k: d.get(k) for k in ("at", "channel", "source", "source_id", "to")} + for d in due]) + + async def _next(self) -> bool: + """One turn of the loop: the closing, a read, a wake or a clock jump; False once the run should end.""" + step = self.step + now = await self.now() + spent = self.spent() + if spent + step.wake_reserve_seconds() >= self.time_budget: + self.log("time_budget_reached", virtual_time=_iso(now), real_seconds=round(spent, 1), + time_budget_seconds=self.time_budget, wake_reserve_seconds=step.wake_reserve_seconds()) + await self._final(now, reason="time_budget") + return False + # 1. a read that is due: the user sim reads what the agent sent (one due only at or past end_time is the last + # read, after the final check-in) + if self.read_at is not None and self.read_at <= now and self.unread \ + and (self.end is None or self.read_at < self.end): + await self.read(now) + return True + # past end_time (the clock runs during the agent's turns, so one can end past it): the final check-in next, with + # whatever else is due, rather than ordinary wakes that would use up max_wakes without one + if self.end is not None and now >= self.end: + await self._final(now) + return False + # 2. anything due for the agent: notifications, scripted user lines, held replies; and what else comes + # due in the same minute, so it all arrives in one wake + if any(h["deliver_at"] <= now for h in self.held) or await self._due_count(self.recipient, now) \ + or await self._due_count(USER_SAYS, now): + later = await self._later(now) + same_minute = [t for t in later.values() if t < now.replace(second=0, microsecond=0) + + timedelta(minutes=1) and (self.end is None or t < self.end)] + if same_minute and await self._armed(): + await self.advance(min(same_minute), reason="same_minute") + return True + lines = await self._take_user_says() + replies = [h for h in self.held if h["deliver_at"] <= now] + self.held = [h for h in self.held if h["deliver_at"] > now] + await self._wake_with(now, lines, replies) + return True + # 3. nothing due: jump to the next thing + later = await self._later(now) + soonest = min(later.values()) if later else None + if self.end is not None and (soonest is None or soonest >= self.end): + await self._final(now) + return False + if soonest is None: + self.end_reason = NOTHING_CAN_WAKE # finished or stuck: finish() decides from what is pending + return False + if self.end is None and set(later) == {"world_event"} and await self._only_repeating(): + self.idle_jumps += 1 + self.idle_since = self.idle_since or _iso(now) + if self.idle_jumps > IDLE_JUMPS_MAX: + self.log("world_only_repeats", virtual_time=_iso(now), jumps=self.idle_jumps - 1, + since=self.idle_since, triggers=await self._repeating_ids()) + self.end_reason = NOTHING_CAN_WAKE + return False + else: + self.idle_jumps, self.idle_since = 0, None + why = sorted(k for k, v in later.items() if v == soonest) + await self.advance(soonest, reason="+".join(why)) + return True + + async def _later(self, now: datetime) -> dict[str, datetime]: + """What could wake the agent after ``now``, by kind, each at its earliest time.""" + candidates = {"world_event": await self._next_world_event(), + "agent_notification": await self._next_due(self.recipient, now), + "user_says": await self._next_due(USER_SAYS, now), + "held_reply": min((h["deliver_at"] for h in self.held), default=None), + "user_read": self.read_at if self.unread else None} + return {k: v for k, v in candidates.items() if v is not None and v > now} + + async def _final(self, now: datetime, reason: str = "end_time") -> None: + """The final check-in: at end_time, the user's ``final`` line, else ``FINAL_DEFAULT_TEXT`` (a line from the + step, not the user); with whatever else is due, and the last-turn notice. A reply held past now stays held, + to be recorded as pending.""" + step = self.step + if reason == "end_time" and self.end > now and await self._armed(): # with no clock, the check-in is now + await self.advance(self.end, reason="end_time") + now = await self.now() + self.final_sent = True + lines = await self._take_user_says() + replies = [h for h in self.held if h["deliver_at"] <= now] + self.held = [h for h in self.held if h["deliver_at"] > now] + notes = [] + if reason == "end_time" and step.final: + lines.append({"at": _iso(now), "text": step.final, "source": "final"}) + else: + notes.append(_step_part(FINAL_DEFAULT_TEXT.format(first=self.person["first"]), "final")) + await self._wake_with(now, lines, replies, last=True, notes=notes) + self.end_reason = self.end_reason or reason + + async def _last_read(self) -> None: + """After the agent's last turn, the user sim reads what it sent since her last read, at the time she would + (the world's clock stays where the run ended), so her reaction to its wrap-up is recorded; a reply she gives + is held and recorded as pending, never delivered. Not after done ends a one-sitting run, after she gave up, or + when the run ended on an error.""" + if not self.unread or self.signed_off or self.gave_up or self.end_reason in ERROR_ENDS: + return + await self._schedule_read() + now = await self.now() + at = self.read_at or now + if at < now: # she'd have looked during the last turn, which went on sending: she reads it all, from now + at, self.read_mode = self._available(now) + await self.read(at, last=True) + + def _prompt_parts(self) -> list[dict]: + """The task's prompt as A2A parts, its seed applied.""" + return self.step._apply_seed(self.step._effective_parts(), self.ctx.metadata.get("seed", {})) + + def _effective_prompt(self) -> str: + return "\n".join(p.get("text", "") for p in self._prompt_parts() if p.get("kind") == "text") + + def _opening_parts(self, labelled) -> list[dict]: + """The prompt as the opening wake and the user's chat carry it: its text as one labelled line from the user + (``labelled(text)``), where its first text part was, and every other part (a file, data) as it is, in the + prompt's order, as prompt_agent sends them.""" + out, text = [], self._effective_prompt() + for part in self._prompt_parts(): + if part.get("kind") != "text": + out.append(part) + elif text is not None: + out.append(labelled(text)) + text = None + return out + + def _user_part(self, at: datetime, text: str, opening: bool = False) -> dict: + first = self.person["first"] + body = re.sub(rf"^\s*{re.escape(first)}\s*:\s*", "", text) + return {"kind": "text", "text": f"[{label_time(_iso(at))}] {first}: {body}", + "metadata": {"agentenv": {"source": "user", "at": _iso(at), "opening": opening}}} + + # ---- the graded conversation: the user's chat ---------------------------------------------------------------- + # What graders read as the run's conversation (metadata["a2a_conversations"][step id]) is what the user saw: her + # own messages as user turns (the opening request, user_says and final lines, her replies), and the agent's + # message_user texts, each with its pretend time, as one agent message per run of them. Never the agent's closing + # text, the alerts that woke it, or the step's own lines; those are in the per-wake prompt parts and the timeline. + # It ends with her last word that never reached the agent, and a marker saying how the run ended. + + def _chat_part(self, at: datetime, text: str, **meta: Any) -> dict: + part = self._user_part(at, text) + part["metadata"]["agentenv"].update(meta) + return part + + async def _chat_flush(self) -> None: + """Write the agent's message_user texts not yet in the conversation, as one agent message.""" + if self.chat_pending: + parts, self.chat_pending = self.chat_pending, [] + conversation_store.complete_a2a_task(conversation_id=self.conversation_id, parts=parts, role="agent", + **(await self.step._env_clock(self.clock_envs))) + + async def _chat_user(self, parts: list[dict]) -> None: + """Write a user turn: what she said, after what the agent sent before it.""" + await self._chat_flush() + conversation_store.add_a2a_task(conversation_id=self.conversation_id, parts=parts, + a2a_task_id=uuid.uuid4().hex, role="user", + **(await self.step._env_clock(self.clock_envs))) + + async def _chat_end(self, now: datetime) -> None: + """Close the conversation: the agent's last messages, then one last user turn with her replies that never + reached the agent, her reaction to its last messages when she said nothing, and the end marker.""" + first = self.person["first"] + parts = [self._chat_part(h["deliver_at"], f"{h['text']} (sent after the assistant's last turn; it never saw " + "this)", kind="user_reply", delivered=False, **({"done": True} if h.get("done") else {})) + for h in self.held] + if self.reaction: + parts.append(_step_part(f"({first} {self.reaction['said']})", "user_reaction")) + parts[-1]["metadata"]["agentenv"].update(outcome=self.reaction["outcome"], at=self.reaction["at"]) + parts.append({"kind": "text", "text": f"[End of run · {label_time(_iso(now))}] {self.end_note} (A note from " + f"the run, not a message from {first}.)", + "metadata": {"agentenv": {"source": "end_marker", "end_reason": self.end_reason, + "at": _iso(now)}}}) + await self._chat_user(parts) + + async def _take_user_says(self) -> list[dict]: + items = await take_due(self.notification_envs, USER_SAYS) if self.notification_envs else [] + out = [] + for it in items: + text = "\n\n".join(p["text"] for p in it["parts"] if p.get("kind") == "text") + out.append({"at": it.get("at"), "text": text, "source": "user_says", "trigger_id": it.get("source_id")}) + return out + + async def _wake_with(self, now: datetime, lines: list[dict], replies: list[dict], last: bool = False, + notes: Optional[list[dict]] = None) -> None: + """Wake the agent with what came due for it, then the user's lines and replies in time order, then ``notes`` + (the step's own lines); ``last`` marks the run's last wake.""" + due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] + parts = delivered_parts(due) + user_msgs = sorted([{"at": l["at"] or _iso(now), "text": l["text"], "kind": l.get("source", "user_says"), + "trigger_id": l.get("trigger_id")} for l in lines] + + [{"at": _iso(r["deliver_at"]), "text": r["text"], "kind": "user_reply", + "done": r.get("done", False)} for r in replies], + key=lambda m: _utc(m["at"])) + for r in replies: + if r.get("hook") is not None: # it reaches the agent now: the reply time the old psi-wake hook recorded + r["hook"].update(delivered=True, user_reads_at=_iso(r["deliver_at"]), delivered_in_wake=self.wakes + 1) + if user_msgs: + await self._chat_user([self._chat_part(_utc(m["at"]), m["text"], kind=m["kind"], + **({"done": True} if m.get("done") else {})) for m in user_msgs]) + for m in user_msgs: + parts.append(self._user_part(_utc(m["at"]), m["text"])) + if m["kind"] != "user_reply": + self.you_sent.append({"at": m["at"], "text": m["text"]}) + self.log("scripted_user_line", virtual_time=m["at"], text=m["text"], kind=m["kind"], + trigger_id=m.get("trigger_id")) + else: + self.log("user_reply_delivered", virtual_time=m["at"], text=m["text"], done=m["done"]) + parts += notes or [] + woke_by = [f"notification:{d['channel']}" for d in due] + [f"{m['kind']}" for m in user_msgs] + woke_by += [f"step:{n['metadata']['agentenv']['source']}" for n in notes or []] + await self.wake(parts, woke_by=woke_by, notifications=[{k: d.get(k) for k in + ("at", "channel", "source", "source_id", "to")} + for d in due], last=last) + if self.signed_off and any(m.get("done") for m in user_msgs): + self.end_reason = self.end_reason or "user_done" + + async def wake(self, parts: list[dict], woke_by: list[str], notifications: Optional[list] = None, + last: bool = False) -> None: + """One wake: the agent's turn on ``parts``, then what it sent the user, then the settle wait. The wake that + ``last`` marks, or the one ``max_wakes`` makes the last, carries the last-turn notice (``final_reminder``).""" + step = self.step + sent_at = await self.now() + if self.wakes == 0 and not self.rules_in_system_prompt: # no system prompt took them: the first wake does + parts = [_step_part(self.rules(), "rules"), *parts] + last = last or self.wakes + 1 >= step.max_wakes + if last and step.final_reminder: + parts.append(_step_part(step.final_reminder.format(first=self.person["first"]), "final_reminder")) + if step.wake_reminder: + parts = [*parts, _step_part(step.wake_reminder_text, "reminder")] + self.wakes += 1 + self.idle_jumps, self.idle_since = 0, None + self.jumps_in_gap, self.folded = 0, None + msg_id = uuid.uuid4().hex + await self._read_changes("before_turn") + + def record() -> None: + self.per_turn_parts.append(self._kept_parts(parts)) + + self.log("wake", n=self.wakes, virtual_time=_iso(sent_at), woke_by=woke_by, notifications=notifications or [], + last=last, parts_text="\n".join(p.get("text", "") for p in parts if p.get("kind") == "text")) + t = time.time() + try: + task_id, result = await self._agent_turn(parts, msg_id, record) + except TimeoutError as e: + self._failed("agent_turn_timeout", f"agent turn timed out after {step.timeout_seconds}s: {e}") + self.log("agent_turn_end", n=self.wakes, state="timeout", real_seconds=round(time.time() - t, 1)) + return + state = result["status"]["state"] + status_msg = (result.get("status") or {}).get("message") or {} + term = protocol.TerminalResponse.from_message(status_msg) + if term.tool_call_count is not None: + self.tool_calls = (self.tool_calls or 0) + term.tool_call_count + messages, quiet = await self._take_messages() + await self._read_changes("turn") + traj = None + card = self.agent.a2a_card or {} + traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) + if traj_ext: + try: + traj = await asyncio.wait_for( + step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), msg_id, + card=card, sandbox_type=self.agent.sandbox_type), TRAJECTORY_FETCH_SECONDS) + except Exception as e: + logger.warning(f"wake {self.wakes}: trajectory fetch failed (continuing): {e}") + self.per_turn_trajectory_urls.append(traj) + self.final_task_id = task_id + ended = await self.now() + plain = term.response_text or "" + extra = {"stay_quiet_calls": len(quiet)} if step.stay_quiet else {} + self.log("agent_turn_end", n=self.wakes, state=str(state), virtual_time=_iso(ended), + virtual_seconds=round((ended - sent_at).total_seconds()), real_seconds=round(time.time() - t, 1), + tool_calls=term.tool_call_count, plain_reply=plain, messages_to_user=len(messages), + trajectory_url=traj, **extra) + self._sent(messages) + if not _completed(state): # failed, canceled or rejected: the turn didn't run to its end + self._failed(term.error_type or f"agent_turn_{_state_name(state)}", + term.error_message or term.response_text or f"agent turn {_state_name(state)}", + term.error_code) + return + if step.nudge and not messages and not quiet and _closing_text(plain): + await self._nudge(plain) + stuck = await self.settle() + if stuck: + self.log("settle_timeout", wake=self.wakes, still_at_work=stuck, settle=self.last_settle) + await self._read_changes("after_turn") + await self._schedule_read() + + def _kept_parts(self, parts: list[dict]) -> list[dict]: + """What the prompt response keeps of one wake's parts: TURN_TEXT_CHARS of each text, until + TURN_TEXT_TOTAL_CHARS in all; then a note. Other parts (files, data) as they are.""" + kept, used = [], sum(len(p.get("text") or "") for turn in self.per_turn_parts for p in turn or []) + for part in parts: + if part.get("kind") != "text": + kept.append(part) + continue + text = part.get("text") or "" + room = min(TURN_TEXT_CHARS, TURN_TEXT_TOTAL_CHARS - used) + if room <= 0: + kept.append({"kind": "text", "text": "(cut: the run's stored timeline has this wake's full text)"}) + break + if len(text) > room: + part = {**part, "text": text[:room] + "… (cut)"} + used += len(part["text"]) + kept.append(part) + return kept + + async def _nudge(self, closing: str) -> None: + """The wake ended with closing text and neither message_user nor stay_quiet, so give the agent one more short + turn with ``nudge_text``; record what the closing text was and what it did after.""" + step = self.step + self.nudges += 1 + parts = [_step_part(step.nudge_text, "nudge")] + sent_at = await self.now() + msg_id = uuid.uuid4().hex + + def record() -> None: + self.per_turn_parts.append(self._kept_parts(parts)) + + t = time.time() + try: + task_id, result = await self._agent_turn(parts, msg_id, record) + except TimeoutError as e: + self._failed("agent_turn_timeout", f"agent nudge turn timed out after {step.timeout_seconds}s: {e}") + self.log("nudge", n=self.nudges, wake=self.wakes, virtual_time=_iso(sent_at), closing_text=closing, + state="timeout", outcome="timeout", real_seconds=round(time.time() - t, 1)) + return + state = result["status"]["state"] + status_msg = (result.get("status") or {}).get("message") or {} + term = protocol.TerminalResponse.from_message(status_msg) + if term.tool_call_count is not None: + self.tool_calls = (self.tool_calls or 0) + term.tool_call_count + messages, quiet = await self._take_messages() + await self._read_changes("nudge") + traj = None + card = self.agent.a2a_card or {} + traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) + if traj_ext: + try: + traj = await asyncio.wait_for( + step._fetch_trajectory(self.target_url, traj_ext, task_id, step._trajectory_prefix(), msg_id, + card=card, sandbox_type=self.agent.sandbox_type), TRAJECTORY_FETCH_SECONDS) + except Exception as e: + logger.warning(f"nudge after wake {self.wakes}: trajectory fetch failed (continuing): {e}") + self.per_turn_trajectory_urls.append(traj) + self.final_task_id = task_id + plain = term.response_text or "" + outcome = "message_user" if messages else "stay_quiet" if quiet else "neither" + self.log("nudge", n=self.nudges, wake=self.wakes, virtual_time=_iso(sent_at), closing_text=closing, + state=str(state), outcome=outcome, messages_to_user=len(messages), stay_quiet_calls=len(quiet), + plain_reply_after=plain, tool_calls=term.tool_call_count, real_seconds=round(time.time() - t, 1), + trajectory_url=traj) + self._sent(messages, after_nudge=True) + if not _completed(state): + self._failed(term.error_type or f"agent_turn_{_state_name(state)}", + term.error_message or term.response_text or f"agent nudge turn {_state_name(state)}", + term.error_code) + + # ---- app changes ------------------------------------------------------------------------------------------------- + async def _start_changes(self) -> None: + """Where each env's changelog stands before the first wake (urn:agentenv:state/v1's changelog_id), so the + run reads only its own changes; an env that keeps no changelog is skipped.""" + for env in self.ctx.deployed_envs: + if not (env.supports(EXT_CHANGES_URI, "get") and env.supports(EXT_STATE_URI, "get")): + continue + try: + changelog_id = (await _call(env, EXT_STATE_URI, "get", attempts=2)).get("changelog_id") + if changelog_id is None: # an empty changelog, or none: only the second refuses a read + await _call(env, EXT_CHANGES_URI, "get", {"after_id": 0, "limit": 1}, attempts=1) + except Exception as e: + logger.info(f"always_on: {env.env_id} keeps no changelog it serves; its changes go unrecorded: {e}") + continue + self.change_cursors[env.env_id] = (env, int(changelog_id or 0)) + + async def _read_changes(self, phase: str) -> None: + """Record the app changes made since the last read (urn:agentenv:changes/v1), each with who made it. One the + agent made (a tool call or website request of its role) carries the wake that made it (``wake``, with + ``phase`` "turn" or "nudge"); others say when they were seen: before the wake's turn (clock jumps), during it, + or after it (contacts and triggers while the world settled). A change with no actor is one the gateway didn't + serve, a website reached directly say.""" + for env_id, (env, cursor) in list(self.change_cursors.items()): + rows: list[dict] = [] + try: + while True: + got = await _call(env, EXT_CHANGES_URI, "get", {"after_id": cursor, "limit": CHANGES_PAGE_MAX}, + attempts=2) + page = got.get("changes") or [] + rows += page + if page: + cursor = page[-1]["id"] + if not page or not got.get("has_more"): + break + except Exception as e: + logger.warning(f"always_on: reading {env_id}'s changes failed (continuing): {e}") + self.change_cursors[env_id] = (env, cursor) + if rows: + changes = [self._change(env_id, row, phase) for row in rows] + more = len(changes) - CHANGES_KEPT_PER_READ + self.log("app_changes", wake=self.wakes, phase=phase, env_id=env_id, count=len(changes), + changes=changes[:CHANGES_KEPT_PER_READ], **({"not_kept": more} if more > 0 else {})) + + def _change(self, env_id: str, row: dict, phase: str) -> dict: + actors = [{k: v for k, v in a.items() if k in ("kind", "role", "tool", "trigger_id", "request", "caused_by", + "failed") and v not in (None, [], {})} + for a in row.get("actors") or []] + mine = any(a.get("role") == self.recipient and a.get("kind") in ("tool_call", "request") for a in actors) + by = "agent" if mine else "others" if actors else None + self.change_counts["by_agent" if mine else "by_others" if actors else "no_actor"] += 1 + summary = row.get("summary") + if isinstance(summary, str) and len(summary) > CHANGE_SUMMARY_CHARS: + summary = summary[:CHANGE_SUMMARY_CHARS] + "…" + return {"env_id": env_id, "id": row.get("id"), "table": f"{row.get('schema_name')}.{row.get('table_name')}", + "op": row.get("operation"), "row_id": row.get("row_id"), "summary": summary, + "at": row.get("timestamp"), "by": by, "wake": self.wakes, "phase": phase, "actors": actors} + + async def _take_messages(self) -> tuple[list[dict], list[dict]]: + """What the agent sent the user with message_user this turn, oldest first, and its stay_quiet calls: its role's + only, so another role's stay queued for whoever takes them.""" + messages, quiet = [], [] + for env in self.chat_envs: + got = await _call(env, EXT_USER_CHAT_URI, "take", {"take_id": uuid.uuid4().hex, "from": self.recipient}) + messages += got.get("items") or [] + quiet += got.get("quiet") or [] + messages.sort(key=lambda m: m.get("at") or "") + return messages, quiet + + def _sent(self, messages: list[dict], after_nudge: bool = False, leftover: bool = False) -> None: + """Record what the agent sent the user: for her next read, the graded chat and the timeline. ``leftover`` + marks messages no turn took (taken at the end), which she never reads.""" + for m in messages: + self.log("message_to_user", virtual_time=m.get("at"), text=m["text"], wake=self.wakes, + **({"after_nudge": True} if after_nudge else {}), **({"leftover": True} if leftover else {})) + sent = {"at": m.get("at"), "text": m["text"], "wake": self.wakes} + self.unread.append(sent) + self.all_messages.append(sent) + self.chat_pending.append({"kind": "text", "text": f"[{label_time(m.get('at'))}] {m['text']}", + "metadata": {"agentenv": {"source": "message_user", "at": m.get("at"), + "wake": self.wakes}}}) + + def _failed(self, error_type: str, error: str, error_code: Optional[str] = None) -> None: + """An agent turn didn't complete (failed, canceled, rejected or timed out): the run ends, and its prompt + response carries ``error_type`` the way prompt_agent's does, so verifiers skip it instead of grading it as a + normal run.""" + self.error, self.error_type, self.error_code = error, error_type, error_code + self.end_reason = "agent_turn_timeout" if error_type == "agent_turn_timeout" else "agent_turn_failed" + + async def read(self, now: datetime, last: bool = False) -> None: + """The user sim reads what the agent sent since her last read, at ``now``; ``last`` marks the read after the + run's last wake, whose answer can no longer change how the run ends.""" + phone = [] + if self.notification_envs: + for it in await take_due(self.notification_envs, USER_PHONE): + phone.append({"at": it.get("at"), "channel": it["channel"], + "text": "\n\n".join(p["text"] for p in it["parts"] if p.get("kind") == "text")}) + self.idle_jumps, self.idle_since = 0, None + seen = list(self.unread) + read = {"time": _iso(now), "assistant_messages": [{"at": m.get("at"), "text": m["text"]} for m in seen], + "phone": phone, + "you_sent": list(self.you_sent), "done_ends_run": not self.multi_day} + self.unread, self.you_sent, self.read_at = [], [], None + calls = [] + cap = self.step.read_cap_seconds() + deadline = time.monotonic() + cap + + async def ask(text: str) -> dict: + got = await self._ask_user_sim(text, deadline) + calls.append(got) + return got + + t = time.time() + try: + ans = await asyncio.wait_for(self.plugin.answer(self.version, self.person, self.history, read, ask), cap) + except TimeoutError: # the plug-in ran past the read's cap: a user sim error, as a failed ask is + ans = {"outcome": "no_reply", "message": ""} + calls.append({"text": None, "structured": None, "error": f"the read ran past its cap ({cap:.0f}s)"}) + self.history.append({"read": read, "answer": ans}) + mode = self.read_mode + latency = self._latency(mode) + flagged = (len(ans.get("message") or "") > ASSISTANT_LIKE_MESSAGE_CHARS + or bool(re.search(r"^\s*(#|\|)", ans.get("message") or "", re.M))) + entry = self.log("user_sim_read", virtual_time=_iso(now), version=self.version, read=read, + input=ans.get("input"), raw=ans.get("raw"), outcome=ans.get("outcome"), + message=ans.get("message"), used=ans.get("used"), score=ans.get("score"), + schema_errors=ans.get("schema_errors"), mode=mode, + real_seconds=round(time.time() - t, 1), looks_like_assistant=flagged, + user_sim_error=(calls[-1].get("error") if calls else None), + user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls], + user_sim_turns=ans.get("turns", len(calls)), shown=ans.get("shown"), + past_turn_cap=ans.get("past_turn_cap"), last_read=last, + retried=ans.get("retried"), first_answer=ans.get("first_answer")) + outcome = ans.get("outcome") + said = (ans.get("message") or "").strip() + hook = {"hook": HOOK_NAME, "turn": self.wakes, "read_at": _iso(now), + "wrote_at": min((m["at"] for m in seen if m.get("at")), default=None), "outcome": outcome} + if outcome in ("reply", "done") and said: + deliver = now + latency + reply_to = _reply_to(ans, seen) + entry.update(deliver_at=_iso(deliver), reply_to=reply_to, reply_to_candidates=len(seen)) + # Not delivered until a wake carries it, which sets user_reads_at; one never delivered (her last read's + # reply, or one still held at the end) keeps delivered False and no user_reads_at, so a check can't count + # it as received. + hook.update(delivered=False, deliver_at=_iso(deliver), reply_to=_cut(reply_to)) + self.held.append({"deliver_at": deliver, "text": ans["message"], "done": outcome == "done", + "hook": hook}) + self.last_reply_at = deliver + self.hook_records.append(hook) + if outcome == "reply" and said: + self.replies += 1 + max_replies = self.us.get("max_replies") + if max_replies and self.replies >= int(max_replies): + self.gave_up = True + self.log("user_gave_up", replies=self.replies) + if not said and outcome in ("no_reply", "done") and (last or (outcome == "done" and not self.multi_day)): + self.reaction = {"outcome": outcome, "at": _iso(now), + "said": "read it and signed off without a message." if outcome == "done" + else "read it and didn't reply."} + if outcome == "done" and not last: + entry["ends_run"] = not self.multi_day + if not self.multi_day and said: + self.signed_off = True + elif not self.multi_day: + self.end_reason = "user_done" + if calls and calls[-1].get("error") and not last: + self.end_reason = self.end_reason or "user_sim_error" + + async def _ask_user_sim(self, text: str, deadline: float) -> dict: + """One user sim turn on ``text``, cut off at its time limit and grace or at the read's ``deadline`` + (time.monotonic()), whichever comes first.""" + msg_id = uuid.uuid4().hex + limit = min(self.step.user_timeout_seconds + TURN_GRACE_SECONDS, deadline - time.monotonic()) + if limit <= 0: + return {"text": None, "structured": None, "error": "the read's time limit passed"} + try: + user_task_id, result = await asyncio.wait_for(send_and_wait( + self.user_url, [{"kind": "text", "text": text}], + agent=self.user_agent if self.user_agent.sandbox_id else None, message_id=msg_id, + context_id=self.user_context_id, timeout_seconds=min(self.step.user_timeout_seconds, limit), + poll_interval_seconds=self.step.poll_interval_seconds), limit) + except TimeoutError as e: + return {"text": None, "structured": None, "error": f"timeout: {e}"} + status = (result.get("status") or {}) + if status.get("state") != TaskState.completed and str(status.get("state")) != "completed": + return {"text": None, "structured": None, "error": f"state {status.get('state')}"} + msg = status.get("message") or {} + term = protocol.TerminalResponse.from_message(msg) + raw = next((p["text"] for p in msg.get("parts") or [] if p.get("kind") == "text"), term.response_text) + structured = term.structured_output + if structured is None: + try: + structured = json.loads(raw) + except (TypeError, ValueError): + structured = None + traj = None + card = self.user_agent.a2a_card or {} + traj_ext = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) + if traj_ext: + try: + traj = await asyncio.wait_for( + self.step._fetch_trajectory(self.user_url, traj_ext, user_task_id, + self.step._trajectory_prefix() + "user_sim/", msg_id, + card=card, sandbox_type=self.user_agent.sandbox_type), + max(0.0, min(TRAJECTORY_FETCH_SECONDS, deadline - time.monotonic()))) + except Exception as e: + logger.warning(f"user sim trajectory fetch failed (continuing): {e}") + return {"text": raw, "structured": structured if isinstance(structured, dict) else None, + "trajectory_url": traj} + + # ---- the end -------------------------------------------------------------------------------------------- + async def _contacts_at_work(self) -> list[dict]: + """By env, the triggers still at work now: contact replies in progress or waiting for a later time.""" + out = [] + for env in self.ctx.deployed_envs: + if not env.supports(EXT_TRIGGERS_URI, "state"): + continue + try: + state = await _call(env, EXT_TRIGGERS_URI, "state", attempts=2) + except Exception as e: + out.append({"env_id": env.env_id, "error": f"{type(e).__name__}: {str(e)[:200]}"}) + continue + rows = [_at_work(row) for row in state.get("triggers", []) + if row.get("status") in IN_FLIGHT or row.get("running") or row.get("executor_calls") + or row.get("pending")] + if rows: + out.append({"env_id": env.env_id, "triggers": rows}) + return out + + async def _pending_at_end(self) -> dict: + """What was still pending when the run ended: her replies held for their response time, the agent's messages + she never read, user lines and alerts for the agent never delivered, its scheduled checks not yet due, and + contact replies still in progress.""" + alerts, checks = [], [] + for st in await self._notes_state(self.recipient): + alerts += [{k: n.get(k) for k in ("at", "channel", "text", "trigger_id")} + for n in st.get("pending_notifications", [])] + checks += [{k: c.get(k) for k in ("next", "note", "every")} for c in st.get("checks", []) + if c.get("status") == "active"] + lines = [{k: n.get(k) for k in ("at", "text", "trigger_id")} + for st in await self._notes_state(USER_SAYS) for n in st.get("pending_notifications", [])] + return {"held_replies": [{"at": _iso(h["deliver_at"]), "text": h["text"], "done": h.get("done", False)} + for h in self.held], + "unread_by_user": [dict(m) for m in self.unread], + "user_lines_not_delivered": lines, + "alerts_never_delivered": alerts, + "scheduled_checks_not_due": checks, + "contact_replies_in_progress": await self._contacts_at_work()} + + async def finish(self) -> None: + step, ctx = self.step, self.ctx + now = await self.now() + try: # what the agent sent and no turn took: a turn that timed out or failed mid-way + leftover, _ = await asyncio.wait_for(self._take_messages(), LEFTOVER_TAKE_SECONDS) + self._sent(leftover, leftover=True) + except Exception as e: + logger.warning(f"always_on: could not take the agent's last messages: {type(e).__name__}: {e}") + for h in self.held: + if h.get("hook") is not None: + h["hook"]["undelivered"] = "the run ended before it reached the agent" + await self._read_changes("end") + try: + self.pending = await self._pending_at_end() + except Exception as e: # the record matters more than this part of it + logger.warning(f"always_on: could not read what was pending at the end: {e}") + self.pending = {"error": [{"error": f"{type(e).__name__}: {str(e)[:200]}"}]} + if self.end_reason == NOTHING_CAN_WAKE: + # Scheduled checks would have woken the agent, so they aren't what makes a run stuck. + stuck = any(items for kind, items in self.pending.items() if kind != "scheduled_checks_not_due") + self.end_reason = "stuck" if stuck else "finished_nothing_pending" + self.end_note = END_NOTES.get(self.end_reason, self.end_reason or "").format(first=self.person["first"], + wakes=self.wakes) + if self.end_reason in ERROR_ENDS and not self.error_type: + self.error_type = self.end_reason + self.log("end", reason=self.end_reason, note=self.end_note, virtual_time=_iso(now), wakes=self.wakes, + replies=self.replies, unread_at_end=len(self.unread), held_at_end=len(self.held), + pending=self.pending) + try: + await self._chat_end(now) + except Exception as e: + logger.warning(f"always_on: could not close the graded conversation: {e}") + conv = conversation_store.get_conversation(self.conversation_id) + if conv and conv.get("status") == "active": + conversation_store.mark_closed(self.conversation_id) + await step._persist_env_trigger_state(ctx) + counts = { + "wakes": self.wakes, + "messages_to_user": len(self.all_messages), + "user_sim_reads": sum(1 for e in self.timeline if e["event"] == "user_sim_read"), + "user_replies": self.replies, + "user_no_replies": sum(1 for e in self.timeline if e["event"] == "user_sim_read" + and e.get("outcome") == "no_reply"), + "user_dones": sum(1 for e in self.timeline if e["event"] == "user_sim_read" and e.get("outcome") == "done"), + "scripted_user_lines": sum(1 for e in self.timeline if e["event"] == "scripted_user_line"), + "clock_jumps": self.clock_jumps, + "silent_wakes": sum(1 for e in self.timeline if e["event"] == "agent_turn_end" + and not e.get("messages_to_user")), + # reads whose answer didn't match the user sim's answer format (the plug-in's schema_errors), counting a + # read the plug-in retried after a malformed first answer even when the retry came back well formed + "user_sim_malformed_answers": sum(1 for e in self.timeline if e["event"] == "user_sim_read" + and (e.get("schema_errors") or e.get("retried"))), + } + if self.change_cursors: + counts["app_changes"] = dict(self.change_counts) + if step.nudge: + nudges = [e for e in self.timeline if e["event"] == "nudge"] + counts["nudges"] = len(nudges) + counts["nudge_outcomes"] = {o: sum(1 for e in nudges if e.get("outcome") == o) + for o in sorted({e.get("outcome") for e in nudges})} + if step.stay_quiet: + counts["stay_quiet_wakes"] = sum(1 for e in self.timeline if e["event"] == "agent_turn_end" + and e.get("stay_quiet_calls")) + record = {"step_id": step.id, "end_reason": self.end_reason, "end_note": self.end_note, "error": self.error, + "error_type": self.error_type, "counts": counts, + "user_sim": {"name": step.user_sim, "version": self.version, + "agent_name": self.us.get("agent_name"), **self.user_setup}, + "end_time": _iso(self.end) if self.end else None, "time_zone": self.zone, + "multi_day": self.multi_day, "real_seconds": round(time.time() - self.t0_real, 1), + "timeline_events": len(self.timeline), "pending_at_end": _bounded(self.pending)} + record["settings"] = {"stay_quiet": step.stay_quiet, "nudge": step.nudge, "wake_reminder": step.wake_reminder, + "clock": self.clock, + "rules_via": "system_prompt" if self.rules_in_system_prompt else "first_wake", + "scheduled_checks": self.checks, "contacts": self.contacts, + "time_budget_seconds": self.time_budget, + "wake_reserve_seconds": step.wake_reserve_seconds()} + await self._store_timeline(record) + ctx.metadata.setdefault(ALWAYS_ON_KEY, {})[step.id] = record + # The graded response is the user's whole chat from the agent, each message with its pretend time; never its + # closing text, which she never saw. + self._append_prompt_response("\n\n".join(f"[{label_time(m.get('at'))}] {m['text']}" for m in self.all_messages)) + + async def _store_timeline(self, record: dict) -> None: + """Store the run's full record, its timeline included, and give ``record`` (what stays in the run's metadata, + kept small for the job's report) its ``object_url``. When it can't be stored, ``record`` keeps the last + ``TIMELINE_TAIL`` events instead, their bulky fields dropped.""" + step, ctx = self.step, self.ctx + try: + store = get_config().get_object_store() + key = (f"{get_config().get_artifact_key_prefix()}always_on_timeline/" + f"instance_id={ctx.instance_id or 'adhoc'}/{step.id}-{uuid.uuid4().hex[:8]}.json") + full = {**record, "timeline": self.timeline} + record["object_url"] = await asyncio.to_thread(store.put, key, + json.dumps(full, indent=1, default=str).encode(), + content_type="application/json") + except Exception as e: + logger.warning(f"always_on: could not store the timeline: {e}") + record["timeline_store_error"] = f"{type(e).__name__}: {str(e)[:300]}" + record["timeline_tail"] = [_small({k: v for k, v in event.items() if k not in BULKY_EVENT_FIELDS}) + for event in self.timeline[-TIMELINE_TAIL:]] + + def _append_prompt_response(self, final_text: str) -> None: + step, ctx = self.step, self.ctx + traj = next((u for u in reversed(self.per_turn_trajectory_urls) if u), None) + ctx.prompt_responses.append(PromptResponse( + prompt_id=step.prompt_id, response=final_text, prompt_text=step.prompt, + agent_trajectory_object_url=traj, agent_trajectory_object_prefix=step._trajectory_prefix(), + target_agent_per_turn_trajectory_object_urls=self.per_turn_trajectory_urls or None, + source_agent_per_turn_prompt_parts=self.per_turn_parts or None, tool_call_count=self.tool_calls, + model=self.model, error_type=self.error_type, error_code=self.error_code, error_message=self.error, + agent_session_id=self.final_task_id, + a2a_context_id=self.solver_context_id, agent_name=step.agent_name, step_id=step.id)) + + +# ================================================================================================================== +# What a judge reads of the run beyond the user's chat (rubrics_verifier, for an always_on step's prompt). +JUDGE_CHANGES_PER_WAKE = 10 +JUDGE_WAKES = 300 +_WOKE_BY = {"user_request": "the user's request", "user_reply": "the user's reply", "user_says": "a user message", + "final": "the user's final line", "step:final": "the end of the job"} + + +def render_for_judge(record: dict) -> str: + """The run wake by wake for a judge: each wake's pretend time, what woke the agent, how much it sent the user and + its tool calls, and the app changes made around it, the agent's own marked; then how the run ended and what was + still pending. From the stored timeline at ``record["object_url"]`` (else the record's tail).""" + timeline = record.get("timeline_tail") or [] + if record.get("object_url"): + try: + url = record["object_url"] + timeline = json.loads(get_config().get_object_store_at(url).get(url))["timeline"] + except Exception as e: + return f"## The run, wake by wake\n(The run's record could not be read: {type(e).__name__}: {e})" + turns = {e["n"]: e for e in timeline if e["event"] == "agent_turn_end"} + nudges = {e["wake"]: e for e in timeline if e["event"] == "nudge"} + changes: dict[int, list[dict]] = {} + for e in timeline: + if e["event"] == "app_changes": + changes.setdefault(e["wake"], []).extend(e.get("changes") or []) + lines = ["## The run, wake by wake", + "The assistant ran only when something woke it. Each wake below gives its pretend time, what woke it, how " + "many messages it sent the user (their text is in the conversation) and its tool calls, then the app " + "changes made around it: the assistant's own (\"it changed\"), others' (contacts, triggers), and ones " + "with no known actor (made outside the env's gateway, such as on a website reached directly), each with " + "when it was seen. The user saw only the messages the assistant sent."] + wakes = [e for e in timeline if e["event"] == "wake"] + for wake in wakes[:JUDGE_WAKES]: + n = wake["n"] + woke = ", ".join(_WOKE_BY.get(w, f"an alert ({w.split(':', 1)[1]})" if w.startswith("notification:") else w) + for w in wake.get("woke_by") or []) or "nothing named" + turn = turns.get(n) or {} + sent = (turn.get("messages_to_user") or 0) + (nudges.get(n, {}).get("messages_to_user") or 0) + calls = f" · {turn['tool_calls']} tool calls" if turn.get("tool_calls") is not None else "" + state = f" · its turn {turn['state']}" if turn.get("state") not in (None, "completed", "TaskState.completed") \ + else "" + lines.append(f"- Wake {n} · {label_time(wake.get('virtual_time'))} · woken by {woke} · sent the user " + f"{sent} message{'s' if sent != 1 else ''}{calls}{state}") + made = changes.get(n, []) + for c in made[:JUDGE_CHANGES_PER_WAKE]: + if c.get("by") == "agent": + who = "it changed" + elif c.get("by"): + who = f"{_actor(c)} changed ({c.get('phase')})" + else: + who = f"no known actor changed ({c.get('phase')})" + summary = f" ({c['summary']})" if c.get("summary") else "" + lines.append(f" - {who}: {c.get('op')} {c.get('table')} row {c.get('row_id')}{summary}") + if len(made) > JUDGE_CHANGES_PER_WAKE: + lines.append(f" - and {len(made) - JUDGE_CHANGES_PER_WAKE} more changes") + if len(wakes) > JUDGE_WAKES: + lines.append(f"(and {len(wakes) - JUDGE_WAKES} more wakes)") + pending = {kind: v.get("count", 0) for kind, v in (record.get("pending_at_end") or {}).items() + if isinstance(v, dict) and v.get("count")} + lines += ["", "## How the run ended", + f"{record.get('end_note') or ''} (end reason: {record.get('end_reason')})".strip()] + if pending: + lines.append("Still pending at the end: " + ", ".join(f"{k.replace('_', ' ')}: {v}" + for k, v in pending.items())) + return "\n".join(lines) + + +def _actor(change: dict) -> str: + actor = (change.get("actors") or [{}])[0] + if actor.get("trigger_id"): + return f"trigger {actor['trigger_id']}" + if actor.get("caused_by"): + return f"{actor.get('role')} (for trigger {actor['caused_by'][0].get('trigger_id')})" + return str(actor.get("role") or actor.get("kind") or "someone") + + +# The longest wait between two tries of a gateway control call (_call backs off doubling, from 1 s, up to this). +CALL_RETRY_MAX_SLEEP_SECONDS = 20 + + +async def _call(env, uri: str, method: str, params: Optional[dict] = None, *, attempts: int = 6, + timeout: int = 60) -> Any: + """A gateway control call, retried on a transient failure (a busy gateway can take seconds to answer). + Every call the step retries is safe to repeat: reads, config, advance (checked against the clock after) and takes + with a take_id.""" + last: Optional[Exception] = None + for attempt in range(attempts): + try: + return await env.invoke(uri, method, params, timeout=timeout) + except EnvCapabilityUnsupported: + raise + except Exception as e: + last = e + if attempt + 1 < attempts: + logger.warning(f"always_on: {method} on {getattr(env, 'env_id', env)} failed ({type(e).__name__}: " + f"{e}); retry {attempt + 1}/{attempts - 1}") + await asyncio.sleep(min(2 ** attempt, CALL_RETRY_MAX_SLEEP_SECONDS)) + raise last + + +def _task_zone(us: dict, clocks: list[dict]) -> str: + """The user's time zone, else the one sync_env_clock gave the world; refused when the two disagree.""" + zones = sorted({c["timezone"] for c in clocks if c.get("timezone")}) + zone = us.get("time_zone") + if zone and zones and zones != [zone]: + raise ValueError(f"always_on: user sim '{us['name']}' is in {zone} but sync_env_clock set {zones}") + if not zone and len(zones) > 1: + raise ValueError(f"always_on: sync_env_clock set several time zones {zones}; give the user_sim step one") + return zone or (zones[0] if zones else "UTC") + + +def _step_part(text: str, source: str) -> dict: + """A line the step adds to a wake, marked so a reader can tell it from what the user or the world sent.""" + return {"kind": "text", "text": text, "metadata": {"agentenv": {"source": source}}} + + +def _state_name(state: Any) -> str: + """An A2A task state's name ("completed", "failed", "canceled", "rejected"), from the enum or its string.""" + return str(getattr(state, "value", state)) + + +def _completed(state: Any) -> bool: + return state == TaskState.completed or _state_name(state) == "completed" + + +def _closing_text(plain: str) -> str: + text = plain.strip() + return "" if text in EMPTY_REPLIES else text + + +def _busy(row: dict, now: datetime) -> bool: + """Whether a trigger is still at work at ``now``: evaluating, calling an executor, or with a fire in flight that + isn't waiting for a later time.""" + if row.get("evaluating") or row.get("executor_calls"): + return True + if "running" in row: # a gateway that reports each fire in flight and when each waiting one goes on + later = sum(1 for w in row.get("waits") or [] if w.get("resumes_at") and _utc(w["resumes_at"]) > now) + return row["running"] > later + if row.get("status") not in IN_FLIGHT and not row.get("pending"): + return False + return not (row.get("waiting") and row.get("resumes_at") and _utc(row["resumes_at"]) > now) + + +# Words she quotes name one of the agent's messages only when they run to at least this many characters, so a short +# quote ("ok") can't match one by chance. +QUOTE_MIN_CHARS = 8 + + +def _reply_to(answer: dict, seen: list[dict]) -> Optional[dict]: + """Which of the agent's messages in a read her answer responds to: the one the plug-in names in ``replying_to`` + (its 1-based number in the read, or words quoted from it); else the one her ``used`` quotes; else the only message + in the read; else the only one that asks something. None when it can't be told. ``matched_by`` says which rule + found it. Whether she approved what it asked is for the grader to read from her message.""" + def found(m: dict, how: str) -> dict: + return {"wake": m.get("wake"), "at": m.get("at"), "text": m["text"], "matched_by": how} + + def quoting(quote: Any) -> list[dict]: + words = " ".join(str(quote or "").split()).lower() + return [m for m in seen if len(words) >= QUOTE_MIN_CHARS and words in " ".join(m["text"].split()).lower()] + + named = answer.get("replying_to") + if isinstance(named, int) and not isinstance(named, bool) and 1 <= named <= len(seen): + return found(seen[named - 1], "replying_to") + if isinstance(named, str) and len(hits := quoting(named)) == 1: + return found(hits[0], "replying_to") + if isinstance(answer.get("used"), str) and len(hits := quoting(answer["used"])) == 1: + return found(hits[0], "used") + if len(seen) == 1: + return found(seen[0], "only_message") + if len(asks := [m for m in seen if "?" in m["text"]]) == 1: + return found(asks[0], "only_question") + return None + + +def _cut(item: Optional[dict]) -> Optional[dict]: + """``item`` with its text cut to PENDING_TEXT_CHARS, for what the run's metadata keeps.""" + if not item or len(item.get("text") or "") <= PENDING_TEXT_CHARS: + return item + return {**item, "text": item["text"][:PENDING_TEXT_CHARS] + "…"} + + +def _bounded(pending: dict) -> dict: + """``pending`` as the run's metadata keeps it: each list's count, and its first ``PENDING_KEPT`` items, kept + small (``_small``) however deep; the stored timeline's end event has them all.""" + return {kind: {"count": len(items), "items": [_small(item) for item in items[:PENDING_KEPT]]} + for kind, items in pending.items()} + + +def _small(value: Any, depth: int = 0) -> Any: + """``value`` kept small for the run's metadata, at any depth: strings cut to ``PENDING_TEXT_CHARS``, lists to their + first ``PENDING_KEPT`` items and a count of the rest, nesting past ``KEPT_NESTING_LEVELS`` levels summed up.""" + if isinstance(value, str): + return value[:PENDING_TEXT_CHARS] + "…" if len(value) > PENDING_TEXT_CHARS else value + if depth >= KEPT_NESTING_LEVELS and isinstance(value, (list, dict)): + return f"({len(value)} {'items' if isinstance(value, list) else 'fields'})" + if isinstance(value, dict): + return {k: _small(v, depth + 1) for k, v in value.items()} + if isinstance(value, (list, tuple)): + kept = [_small(v, depth + 1) for v in value[:PENDING_KEPT]] + return kept + [f"(and {len(value) - PENDING_KEPT} more)"] if len(value) > PENDING_KEPT else kept + return value + + +# The timeline fields the process log may show as they are: names, ids, times, states and settings, never what a person +# or the agent wrote. Any other string is logged as its length, a list as its count, a dict as its keys' view. +_LOGGED_AS_IS = frozenset({ + "virtual_time", "from", "to", "reason", "state", "outcome", "mode", "read_at", "deliver_at", "end_time", "agent", + "agent_role", "user_sim", "version", "user_model", "time_zone", "kind", "trigger_id", "env_id", "phase", + "error_type", "matched_by", "at", "wake", "n", "trajectory_url", "object_url"}) +# Past this length even one of those fields is logged as its length. +LOGGED_TEXT_CHARS = 200 + + +def _log_view(fields: dict, depth: int = 0) -> dict: + """``fields`` as the process log may show them: see ``_LOGGED_AS_IS``. Bodies and addresses are PII; the full + content stays in the stored timeline.""" + out: dict = {} + for k, v in fields.items(): + if v is None or isinstance(v, (bool, int, float)): + out[k] = v + elif isinstance(v, str): + out[k] = v if k in _LOGGED_AS_IS and len(v) <= LOGGED_TEXT_CHARS else {"chars": len(v)} + elif isinstance(v, (list, tuple)): + out[k] = {"items": len(v)} + elif isinstance(v, dict): + out[k] = _log_view(v, depth + 1) if depth < 1 else {"keys": len(v)} + else: + out[k] = type(v).__name__ + return out + + +def _at_work(row: dict) -> dict: + """A trigger still at work, as the run's record keeps it.""" + return {k: row.get(k) for k in ("id", "status", "running", "pending", "executor_calls", "waits", "resumes_at") + if row.get(k) not in (None, [], 0)} + + +def _asleep(hm: tuple[int, int], wake: tuple[int, int], sleep: tuple[int, int]) -> bool: + """Whether local time ``hm`` falls in the night from ``sleep`` to ``wake``; a sleep time after midnight (00:30 with + a wake at 08:00) makes the night the span between them. Equal times: never asleep.""" + if sleep > wake: + return hm >= sleep or hm < wake + return sleep <= hm < wake + + +def _hm(text: str) -> tuple[int, int]: + h, m = text.split(":") + return int(h), int(m) + + +def _utc(text: str) -> datetime: + at = datetime.fromisoformat(str(text).replace("Z", "+00:00")) + if at.tzinfo is None: + raise ValueError(f"{text!r} has no UTC offset") + return at.astimezone(timezone.utc) + + +def _iso(value: Optional[datetime]) -> Optional[str]: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if value else None + + +def _end_instant(end_time: str, t0: datetime) -> datetime: + if isinstance(end_time, str) and end_time.startswith("P"): + m = re.fullmatch(r"P(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?", end_time) + if m is None or not any(m.groups()): + raise ValueError(f"invalid ISO 8601 duration {end_time!r}") + w, d, h, mi, s = (float(g or 0) for g in m.groups()) + return t0 + timedelta(weeks=w, days=d, hours=h, minutes=mi, seconds=s) + return _utc(end_time) diff --git a/src/agent_env/task_step/task_steps/user_sim.py b/src/agent_env/task_step/task_steps/user_sim.py new file mode 100644 index 000000000..11140e614 --- /dev/null +++ b/src/agent_env/task_step/task_steps/user_sim.py @@ -0,0 +1,122 @@ +"""The user_sim step: who the user is, once per task. Its settings go into ``metadata["user_sims"][name]``, where the +always_on step reads them. This docstring is the whole interface between the two. + +What always_on reads from the settings (``metadata["user_sims"][name]``): + +- ``name``; ``display_name`` (else ``name``) and ``first_name`` (else the display name's first word): how the agent's + wakes and the plug-in name the user. +- ``agent_name``: the deployed agent that plays the user (``ask`` below sends to it); ``model``: set on that agent. +- ``user_version``: the version handed to the plug-in; ``plugin``: ``{"file_artifact_id", "file_artifact_version", + "params"}`` naming a plug-in file, else the built-in ``user_sim_versions``. +- ``time_zone`` (IANA): the user's zone; ``schedule`` (``wake``, ``sleep``, ``work``, ``work_days``, ``busy``): when + the user reads; ``response_times`` (minutes by ``morning`` / ``work`` / ``free``): how long a reply is held; + ``max_replies``: after that many replies the user stops reading. +- ``email``, ``description``, ``versions``, ``output_format``, ``permissions``, ``keep_private``: passed to the + plug-in in ``person`` only. + +A plug-in is one Python module with: + + configure(params: dict) -> None optional, called once with plugin.params + setup(version: str, person: dict) -> {"system_prompt": str | None, "output_format": dict | None} + async answer(version: str, person: dict, history: list[dict], read: dict, ask) -> dict + +``setup`` runs once per run; the step sets ``model``, ``system_prompt`` and ``output_format`` on the user sim agent. +``person`` is ``{"key": name, "name", "first", "email", "time_zone", "description", "versions", "output_format", +"permissions", "keep_private", "opening"}``, with ``time_zone`` the run's zone (the user's, else the world clock's, +else UTC) and ``opening`` the user's opening request, ``{"at": RFC 3339 UTC, "text"}``, so ``setup`` can write it +into the system prompt. The opening is also the first line of the first read's ``you_sent``. + +``answer`` runs once per read, at the time the user would read. ``read`` is ``{"time": RFC 3339 UTC, +"assistant_messages": [{"at", "text"}], "phone": [{"at", "channel", "text"}], "you_sent": [{"at", "text"}], +"done_ends_run": bool}``: the agent's message_user texts since the last read, oldest first; what triggers sent +``user_phone`` since the last read, its sender in ``channel`` (``"Gmail · Carol Reyes (Mom)"``) as in the agent's +alerts; the user's own lines (the opening request, ``user_says`` lines, ``final``) each at the time it was sent; never +anything meant for the agent; and whether done ends the run (one-sitting) or is only recorded (multi-day). The user +sim reads only after the agent has used message_user. ``history`` is every earlier +``{"read", "answer"}``, oldest first. ``ask(text)`` sends one turn to the user sim agent (one session per run) and +returns ``{"text", "structured", "trajectory_url"}``, or ``{"text": None, "structured": None, "error"}``. + +``answer`` returns ``{"outcome": "reply" | "no_reply" | "done", "message": str, "used": str | None}``, and may add +``input``, ``raw``, ``score``, ``schema_errors``, ``turns``, ``shown`` and ``past_turn_cap``, which the step records. +A ``reply`` or ``done`` with text is held for the response time, then wakes the agent as the user's message; the +recorded extras (``used``, ``score``, ...) never reach the agent. ``done`` ends a one-sitting run after the agent's +turn on that message, or at once when it has no text, with no further reads; in a multi-day run it is only recorded. +When the last ``ask`` of a read returned an error, the run ends (``user_sim_error``). +""" +from __future__ import annotations + +import logging +from typing import ClassVar, Optional +from zoneinfo import ZoneInfo + +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep + +logger = logging.getLogger(__name__) + +USER_SIMS_KEY = "user_sims" + + +class UserSimTaskStep(TaskStep): + """Who the user is, once per task: their settings go into the run's metadata under ``name``.""" + + type: ClassVar[str] = "user_sim" + entity_refs = (EntityRef.artifact("plugin.file_artifact_id", version_field="file_artifact_version", + artifact_type="file"),) + + def __init__(self, id: str, version: Optional[int], *, name: str, model: str, agent_name: Optional[str] = None, + user_version: str = "ours", display_name: Optional[str] = None, first_name: Optional[str] = None, + description: Optional[str] = None, email: Optional[str] = None, time_zone: Optional[str] = None, + schedule: Optional[dict] = None, response_times: Optional[dict] = None, + max_replies: Optional[int] = None, permissions: Optional[dict] = None, + keep_private: Optional[list] = None, output_format: Optional[dict] = None, + versions: Optional[dict] = None, plugin: Optional[dict] = None, depends_on=None, + fail_task_on_error: bool = True, retry_config=None): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error, + retry_config=retry_config) + if not model: + raise ValueError("user_sim.model is required") + if time_zone is not None: + ZoneInfo(time_zone) + self.name = name + self.model = model + self.agent_name = agent_name or name + self.user_version = user_version + self.display_name = display_name + self.first_name = first_name + self.description = description + self.email = email + self.time_zone = time_zone + self.schedule = schedule + self.response_times = response_times + self.max_replies = max_replies + self.permissions = permissions + self.keep_private = keep_private + self.output_format = output_format + self.versions = versions or {} + self.plugin = plugin + + _FIELDS = ("name", "model", "agent_name", "user_version", "display_name", "first_name", "description", "email", + "time_zone", "schedule", "response_times", "max_replies", "permissions", "keep_private", "output_format", + "versions", "plugin") + + def to_dict(self) -> dict: + return {**super().to_dict(), **{k: getattr(self, k) for k in self._FIELDS}} + + @classmethod + def from_dict(cls, data: dict) -> UserSimTaskStep: + return cls(**cls._base_from_dict(data), retry_config=data.get("retry_config"), + **{k: data.get(k) for k in cls._FIELDS if k in data}) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + overrides = self.step_param_overrides(context) + version = overrides.get("user_version") or self.user_version + if not any(a.agent_name == self.agent_name for a in context.deployed_agents): + raise RuntimeError(f"user_sim '{self.name}': no deployed agent named '{self.agent_name}'; deploy it first") + settings = {k: getattr(self, k) for k in self._FIELDS} + settings.update(user_version=version, step_id=self.id, + model=overrides.get("model") or self.model) + context.metadata.setdefault(USER_SIMS_KEY, {})[self.name] = settings + logger.info(f"user_sim '{self.name}': version {version}, model {settings['model']}, agent {self.agent_name}") + return context diff --git a/src/agent_env/task_step/task_steps/user_sim_versions.py b/src/agent_env/task_step/task_steps/user_sim_versions.py new file mode 100644 index 000000000..f0f9fd77d --- /dev/null +++ b/src/agent_env/task_step/task_steps/user_sim_versions.py @@ -0,0 +1,254 @@ +"""The user sim plug-in the always_on step calls when a user_sim step names none: each version decides the user sim's +system prompt and answer format (``setup``), renders its own input for one read and parses the answer (``answer``). + +The interface, which any plug-in a ``user_sim`` step names (a file artifact) must offer: + + setup(version, person) -> {"system_prompt": str | None, "output_format": dict | None} + async answer(version, person, history, read, ask) -> {"outcome": "reply" | "no_reply" | "done", "message": str, + "used": str | None, "input": str, "raw": ..., ...} + +``person`` is the user_sim step's settings (name, first, time_zone, description, versions, ...). A plug-in may also +offer ``configure(params)``, called once with the step's ``plugin.params``. ``history`` is every +earlier read, oldest first, each ``{"read": , "answer": }``. ``read`` is this read: +``{"time": RFC 3339, "assistant_messages": [{"at", "text"}], "phone": [{"at", "channel", "text"}], +"you_sent": [{"at", "text"}]}``. ``ask(text)`` sends the rendered text to the deployed user sim (one session per run) +and returns ``{"text": raw text, "structured": parsed object or None}``. + +Versions here: ``ours`` (the default) and its ablations ours_no_feed, ours_no_labels, ours_no_time, ours_no_used and +ours_always_replies, and ``today`` (the description as the whole prompt, the agent's raw text as the input, answers +{message, done}). ``ours`` shows each read labelled (the time, what's new on the phone, what the user sent, what the +assistant sent) and answers {outcome, message, used}; its prompt is ``versions.ours.system_prompt`` when the person +has one, else the rules below followed by the description, which then says only who the person is. +""" +from __future__ import annotations + +import json +from datetime import datetime, timezone +from typing import Any, Awaitable, Callable, Optional +from zoneinfo import ZoneInfo + +# prompt_agent's default user-sim output format, used when a task sets none. +ML_DEFAULT_FORMAT = {"type": "json_schema", "schema": {"type": "object", "properties": { + "message": {"type": "string", "description": "Your next reply"}, + "done": {"type": "boolean", "description": "Set true when you're satisfied and the conversation should end."}}, + "required": ["message", "done"]}} + +_OURS_PROPS = { + "outcome": {"type": "string", "enum": ["reply", "no_reply", "done"]}, + "message": {"type": "string"}, + "used": {"type": "string"}, + "score": {"type": "integer", "description": "Only with outcome done: your 1-5 rating"}, +} +FOOTER_Q = "Your AI assistant sent you this. As {first}, decide: reply, don't reply, or say you're done." +FOOTER_Q_ALWAYS = "Your AI assistant sent you this. As {first}, decide: reply, or say you're done." + +VERSIONS = ("today", "ours", "ours_no_feed", "ours_no_labels", "ours_no_time", "ours_no_used", "ours_always_replies") + + +def _ours_schema(version: str) -> dict: + props = dict(_OURS_PROPS) + required = ["outcome", "message", "used"] + if version == "ours_no_used": + props.pop("used") + required.remove("used") + if version == "ours_always_replies": + props["outcome"] = {"type": "string", "enum": ["reply", "done"]} + return {"type": "json_schema", "schema": {"type": "object", "additionalProperties": False, "properties": props, + "required": required}} + + +def _json_line(version: str) -> str: + outcomes = '"reply" | "done"' if version == "ours_always_replies" else '"reply" | "no_reply" | "done"' + used = '' if version == "ours_no_used" else ', "used": "..."' + return '{"outcome": ' + outcomes + ', "message": "..."' + used + '}' + + +def setup(version: str, person: dict) -> dict: + if version not in VERSIONS: + raise ValueError(f"unknown user sim version {version!r}; this plug-in has {list(VERSIONS)}") + if version == "today": + return {"system_prompt": person.get("description"), + "output_format": person.get("output_format") or ML_DEFAULT_FORMAT} + ours = (person.get("versions") or {}).get("ours") or {} + prompt = ours.get("system_prompt") + if not prompt: + if not person.get("description"): + raise ValueError(f"user sim {person.get('name')!r} needs a description or versions.ours.system_prompt") + return {"system_prompt": ours_prompt(version, person), "output_format": _ours_schema(version)} + if version == "ours_no_used": + prompt = _drop_used(prompt) + if version == "ours_always_replies": + prompt = _drop_no_reply(prompt) + return {"system_prompt": prompt, "output_format": _ours_schema(version)} + + +def ours_prompt(version: str, person: dict) -> str: + """How a person reads and answers in ``ours``, then who they are (the description).""" + full = person.get("name") or "the user" + first = person.get("first") or full.split()[0] + feed, clock = version != "ours_no_feed", version != "ours_no_time" + shows = [*(["the time"] if clock else []), *(["what's new on your phone"] if feed else []), "what you sent", + "what your AI assistant sent"] + lines = [f"You are {full}, a real person messaging your personal AI assistant. You are not the AI assistant and " + f"not a tester. Each message you get shows {', '.join(shows[:-1])}, and {shows[-1]}. Reply only as " + f"{first}. Don't do your assistant's work: never look things up for it or hand over facts it should find " + "itself. You can't open your apps" + ("; what's new on your phone is listed in each message." if feed + else ".")] + if feed: + lines.append("What's new on your phone is for you: your assistant reads your inbox itself. Don't pass it on or " + "make calls about it unless the assistant asks, or your description says to.") + if clock: + lines.append("Go by the time each message gives. Any other date you see is the computer's clock, not today's.") + lines.append("If the assistant asks something your description doesn't cover: for a choice, choose the way you " + "would; for a fact about you or anyone else that isn't written there, say you don't know. Never " + "guess facts.") + lines.append('Only answer "done" with a short sign-off. If your message answers a question or tells the assistant ' + 'to do something, answer "reply": a "done" message never reaches the assistant.') + used = ('In "used", give the tag or quote the words of your description you relied on, or "none". ' + if version != "ours_no_used" else "") + lines.append(used + "Answer in JSON:\n" + _json_line(version)) + return "\n".join(lines) + "\n\nWHO YOU ARE\n" + person["description"].strip() + "\n" + + +def _drop_used(prompt: str) -> str: + """Remove the lines that explain the `used` field (the tagged persona lines stay).""" + return "\n".join(ln for ln in prompt.splitlines() if '"used"' not in ln and "`used`" not in ln + and not ln.lower().lstrip().startswith("used:")) + + +def _drop_no_reply(prompt: str) -> str: + return prompt.replace('"reply" | "no_reply" | "done"', '"reply" | "done"').replace( + 'reply, don\'t reply, or say you\'re done', "reply, or say you're done") + + +# ---- rendering ---------------------------------------------------------------------------------------------------- +def _utc(text: str) -> datetime: + return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(timezone.utc) + + +def _local(text: Optional[str], tz: ZoneInfo) -> Optional[datetime]: + return _utc(text).astimezone(tz) if text else None + + +def _hm(d: datetime) -> str: + return d.strftime("%I:%M %p").lstrip("0") + + +def _stamp(t: datetime, now: datetime) -> str: + days = (now.date() - t.date()).days + if days == 0: + return _hm(t) + if days < 6: + return f"{t.strftime('%a')} {_hm(t)}" + return f"{t.strftime('%a %b')} {t.day}, {_hm(t)}" + + +def render_today(read: dict) -> str: + """The assistant's raw message text, nothing else (several messages joined by a blank line).""" + return "\n\n".join(m["text"].strip() for m in read["assistant_messages"]) + + +def render_ours(version: str, person: dict, history: list[dict], read: dict) -> str: + tz = ZoneInfo(person.get("time_zone") or "UTC") + now = _local(read["time"], tz) + first = person.get("first") or person.get("name") or "you" + labels = version != "ours_no_labels" + out: list[str] = [] + if version != "ours_no_time": + out.append(f"It's {now.strftime('%a, %b')} {now.day}, {_hm(now)}.") + last = _last_messaged(history, read) + if last: + lt = _local(last, tz) + days = (now.date() - lt.date()).days + when = (f"at {_hm(lt)}" if days == 0 else f"on {lt.strftime('%a')} at {_hm(lt)}" if days < 6 + else f"on {lt.strftime('%a %b')} {lt.day} at {_hm(lt)}") + out.append(f"You last messaged your AI assistant {when}.") + if version != "ours_no_feed": + if read.get("phone"): + out.append("\nNEW ON YOUR PHONE (since you last checked):" if labels else "") + for e in read["phone"]: + head, *rest = e["text"].split("\n") + t = _local(e.get("at"), tz) or now + out.append(f" [{_stamp(t, now)} · {e['channel']}] {head}" + "".join(f"\n {x}" for x in rest)) + earlier = [e for h in history for e in h["read"].get("phone") or []] + if earlier: + out.append("\nEARLIER ON YOUR PHONE (you've already seen these):" if labels else "") + for e in earlier: + t = _local(e.get("at"), tz) or now + out.append(f" [{_stamp(t, now)} · {e['channel']}] {e['text'].splitlines()[0] if e['text'] else ''}") + if read.get("you_sent"): + out.append("\nYOU SENT YOUR AI ASSISTANT:" if labels else "") + for s in read["you_sent"]: + t = _local(s.get("at"), tz) + out.append((f" [{_stamp(t, now)}] " if t else " ") + s["text"]) + out.append("\nNEW FROM YOUR AI ASSISTANT (this is what you're answering):" if labels else "") + for m in read["assistant_messages"]: + t = _local(m.get("at"), tz) + body = "\n".join(" " + ln if ln.strip() else "" for ln in m["text"].strip().splitlines()) + out.append((f" [{_stamp(t, now)}]\n" if t else "") + body) + q = FOOTER_Q_ALWAYS if version == "ours_always_replies" else FOOTER_Q + out.append("\n" + q.format(first=first) + "\nAnswer in JSON: " + _json_line(version)) + return "\n".join(x for x in out if x != "" or labels) + + +def _last_messaged(history: list[dict], read: dict) -> Optional[str]: + """When the person last sent their assistant anything before this read: their scripted lines (this read's + included) and their earlier replies.""" + times = [s["at"] for h in history for s in h["read"].get("you_sent") or [] if s.get("at")] + times += [s["at"] for s in read.get("you_sent") or [] if s.get("at")] + times += [h["read"]["time"] for h in history if (h.get("answer") or {}).get("outcome") in ("reply", "done")] + times = [t for t in times if _utc(t) <= _utc(read["time"])] + return max(times, key=_utc) if times else None + + +# ---- answers ------------------------------------------------------------------------------------------------------ +def _schema_errors(obj: Any, schema: dict) -> list[str]: + if not isinstance(obj, dict): + return ["not an object"] + errs = [f"missing {k}" for k in schema.get("required", []) if k not in obj] + props = schema.get("properties", {}) + if schema.get("additionalProperties") is False: + errs += [f"extra key {k}" for k in obj if k not in props] + for k, v in obj.items(): + p = props.get(k) or {} + if "enum" in p and v not in p["enum"]: + errs.append(f"{k}: {v!r} not in enum") + return errs + + +def _parse_text(text: str) -> Optional[dict]: + s = (text or "").replace("```json", "").replace("```", "") + dec, i, found = json.JSONDecoder(), s.find("{"), None + while i != -1: + try: + obj, end = dec.raw_decode(s, i) + if isinstance(obj, dict) and ("message" in obj or "outcome" in obj): + found = obj + i = s.find("{", end) + except ValueError: + i = s.find("{", i + 1) + return found + + +async def answer(version: str, person: dict, history: list[dict], read: dict, + ask: Callable[[str], Awaitable[dict]]) -> dict: + text = render_today(read) if version == "today" else render_ours(version, person, history, read) + got = await ask(text) + obj = got.get("structured") + if not isinstance(obj, dict): + obj = _parse_text(got.get("text") or "") + fmt = setup(version, person)["output_format"] + errors = _schema_errors(obj, fmt["schema"]) if obj is not None else ["no JSON object in the answer"] + out = {"version": version, "input": text, "raw": got.get("text"), "structured": obj, "schema_errors": errors} + if obj is None: + # A malformed answer reaches the assistant as raw text, as prompt_agent does today. + return {**out, "outcome": "reply", "message": got.get("text") or "", "used": None} + if version == "today": + done = obj.get("done") is True + return {**out, "outcome": "done" if done else "reply", "message": str(obj.get("message") or ""), + "used": None, "done": obj.get("done")} + outcome = obj.get("outcome") + if outcome not in ("reply", "no_reply", "done"): + outcome = "reply" + return {**out, "outcome": outcome, "message": str(obj.get("message") or ""), "used": obj.get("used"), + "score": obj.get("score")} diff --git a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py index 443e86103..f5cbf9432 100644 --- a/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py +++ b/src/agent_env/task_step/task_steps/verifiers/rubrics_verifier.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +import copy import json import logging import re @@ -623,6 +624,29 @@ def from_dict(cls, data: dict) -> RubricsVerifierTaskStep: judge_batch_concurrency=data.get("judge_batch_concurrency"), ) + # An always_on step's prompt: what this verifier grades it from, set on the copy ``_for_always_on`` makes. + _always_on_wakes: Optional[str] = None + + async def _for_always_on(self, context: TaskStepContext, record: dict) -> "RubricsVerifierTaskStep": + """This verifier as it grades an always_on step's prompt, which may run 40–60 wakes: from the user's chat (the + prompt's response and conversation) and the run's context (the trigger log, and the run wake by wake with the + app changes each made), never every wake's trajectory, which may not fit and costs a lot. A copy, since one + step instance is shared across concurrent rollouts; every other prompt is graded as before.""" + if self._spec().cites_evidence: + raise RuntimeError(f"Verifier '{self.verifier_id}': output_format={self.output_format.value} cites " + "per-criterion screenshots, and an always_on step's prompt is graded without its " + "wakes' trajectories.") + if "grade_up_to_turn" in self.step_param_overrides(context) or self.grade_up_to_turn is not None: + raise RuntimeError(f"Verifier '{self.verifier_id}': grade_up_to_turn can't cut an always_on step's " + "prompt, whose conversation turns are not its wakes.") + from agent_env.task_step.task_steps.always_on import render_for_judge # its module imports this one's peers + + twin = copy.copy(self) + twin.use_trajectory = False + twin.run_context_for_judge = self.run_context_for_judge or list(RUN_CONTEXT_PARTS) + twin._always_on_wakes = await asyncio.to_thread(render_for_judge, record) + return twin + def _resolved_criteria(self, context: TaskStepContext) -> list[dict]: """Criteria for this run: a per-run ``step_params`` override replaces the stored list. @@ -637,6 +661,15 @@ async def execute(self, context: TaskStepContext) -> TaskStepContext: prompt_response = next((pr for pr in context.prompt_responses if pr.prompt_id == self.prompt_id), None) if prompt_response is None: raise RuntimeError(f"PromptResponse with prompt_id='{self.prompt_id}' not found in context") + always_on = (context.metadata.get("always_on") or {}).get(prompt_response.step_id) + if always_on is not None and self._always_on_wakes is None: + if prompt_response.error_type: # skipped before the always_on settings are checked, as any errored prompt + logger.warning(f"Skipping verification '{self.verifier_id}': prompt had " + f"error_type={prompt_response.error_type}") + self._write_skipped_entries( + context, "prompt_error", f"Skipped: prompt had error_type={prompt_response.error_type}") + return context + return await (await self._for_always_on(context, always_on)).execute(context) overrides_for_step = self.step_param_overrides(context) grade_up_to_turn = ( _validated_turn(overrides_for_step["grade_up_to_turn"]) @@ -1545,6 +1578,8 @@ def _build_eval_prompt( base += "\n\n" + render_run_context(run_context, self.run_context_for_judge, message_max_chars=self.RUN_CONTEXT_MESSAGE_MAX_CHARS, firings_max=self.RUN_CONTEXT_FIRINGS_MAX) + if self._always_on_wakes: + base += "\n\n" + self._always_on_wakes if context_facts: # Facts before the policy: they are inputs, the policy is how to read them. base += "\n\n## Context facts\n" + self._CONTEXT_FACTS_LEAD + "\n" + "\n".join( diff --git a/tst/unit/env/gateway/clock_advance_test.py b/tst/unit/env/gateway/clock_advance_test.py index 0a76031f9..5c15cef05 100644 --- a/tst/unit/env/gateway/clock_advance_test.py +++ b/tst/unit/env/gateway/clock_advance_test.py @@ -123,6 +123,61 @@ def test_the_card_offers_advance(): "request": {"type": "object", "properties": {"virtual_time": {"type": "string"}}, "required": ["virtual_time"]}} +def test_set_rate_keeps_the_time_t0_and_generation_and_runs_on_at_the_new_rate(): + clock, now = _armed(rate=60) + now.tick(60) # an hour at 60x + gen, lived = clock.generation, clock.elapsed() + state = clock.set_rate(0) + assert (state["virtual_time"], state["t0"], state["virtual_seconds_per_real_second"]) == ( + "2026-01-01T01:00:00Z", T0, 0) + now.tick(600) + assert clock.read()["virtual_time"] == "2026-01-01T01:00:00Z" and clock.generation == gen + clock.set_rate(2) + now.tick(30) + assert clock.read()["virtual_time"] == "2026-01-01T01:01:00Z" + assert clock.elapsed() == pytest.approx(lived + 60) + clock.advance("2026-01-02T00:00:00Z") + assert clock.read()["virtual_time"] == "2026-01-02T00:00:00Z" + + +def test_set_rate_refuses_a_bad_rate_and_an_unarmed_clock(): + clock, _ = _armed() + with pytest.raises(ClockError, match="non-negative"): + clock.set_rate(-1) + with pytest.raises(ClockError, match="must be a number"): + clock.set_rate(None) + with pytest.raises(ClockError, match="not armed"): + Clock(now_fn=FakeNow()).set_rate(0) + + +@pytest.mark.asyncio +async def test_a_relative_mark_keeps_its_place_across_rate_changes(): + """A re-arm (set_time) would re-anchor a relative or recurring mark to the new t0; set_rate leaves it put.""" + now = FakeNow() + clock = Clock(now_fn=now) + engine = TriggerEngine(_FakeGateway(clock)) + clock.set_time(T0, rate=60) + engine.register({"watch_roles": ["default"], "triggers": [ + {"id": "rel", "when": {"type": "time", "at": "PT2H"}, "actions": [_perm()]}]}) + _tick_call(engine, now, 1) + for rate in (0, 60, 0, 60): + clock.set_rate(rate) + _tick_call(engine, now, 30) # 30 real s: half an hour at 60x, nothing while frozen + await _settle(engine) + assert clock.read()["virtual_time"] == "2026-01-01T01:01:00Z" + assert engine.state()["triggers"][0]["next_mark"] == "2026-01-01T02:00:00Z" + clock.set_rate(60) + _tick_call(engine, now, 60) + await _settle(engine) + assert engine.state()["triggers"][0]["status"] == "fired" + assert "reanchored" not in {e["kind"] for e in engine.state()["events"]} + + +def test_the_card_offers_set_rate(): + clock_ext = next(e for e in GATEWAY_EXTENSIONS if e["uri"] == EXT_CLOCK_URI) + assert clock_ext["params"]["methods"]["set_rate"]["endpoint"] == "/clock/rate" + + @pytest.fixture def gateway(tmp_path, monkeypatch) -> Gateway: monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) @@ -165,3 +220,10 @@ def test_route_refuses_a_backwards_advance(gateway): gateway._clock.set_time("2026-01-04T08:00:00Z", rate=0) resp = _put(gateway, "/clock/advance", {"virtual_time": T0}) assert resp.status_code == 400 and "only moves forward" in resp.json()["error"] + + +def test_route_sets_the_rate_of_an_armed_clock(gateway): + gateway._clock.set_time(T0, rate=0) + resp = _put(gateway, "/clock/rate", {"virtual_seconds_per_real_second": 60}) + assert resp.status_code == 200 and resp.json()["virtual_seconds_per_real_second"] == 60 + assert _put(gateway, "/clock/rate", {"virtual_seconds_per_real_second": -1}).status_code == 400 diff --git a/tst/unit/env/gateway/gateway_card_test.py b/tst/unit/env/gateway/gateway_card_test.py index 48979ddfa..1d8eb883c 100644 --- a/tst/unit/env/gateway/gateway_card_test.py +++ b/tst/unit/env/gateway/gateway_card_test.py @@ -201,12 +201,12 @@ def test_rewrite_child_card_drops_the_childs_interfaces(): assert _gateway({"mcp-slack": "http://slack:18765"})._rewrite_child_card("mcp-slack", child)["additionalInterfaces"] == [] -def test_gateway_card_lists_its_nine_extensions(): +def test_gateway_card_lists_its_ten_extensions(): assert _gateway({})._gateway_extensions() is gateway_constants.GATEWAY_EXTENSIONS assert [e["uri"] for e in _gateway({})._gateway_extensions()] == [ "urn:agentenv:disable-tool/v1", "urn:agentenv:enable-tool/v1", "urn:agentenv:triggers/v1", "urn:agentenv:clock/v1", "urn:agentenv:trajectory/v1", "urn:agentenv:step/v1", "urn:agentenv:state/v1", - "urn:agentenv:changes/v1", "urn:agentenv:notifications/v1", + "urn:agentenv:changes/v1", "urn:agentenv:notifications/v1", "urn:agentenv:user-chat/v1", ] @@ -342,11 +342,11 @@ async def test_env_data_plane_unknown_method(): def test_triggers_advertised_as_one_extension(): - """Triggers ride a single urn:agentenv:triggers/v1 (register/remove/clear/state) — no separate get URI.""" + """Triggers ride a single urn:agentenv:triggers/v1 (register/config/remove/clear/state) — no separate get URI.""" exts = _gateway({})._gateway_extensions() trigger = [e for e in exts if e["uri"] == "urn:agentenv:triggers/v1"] assert len(trigger) == 1 - assert set(trigger[0]["params"]["methods"]) == {"register", "remove", "clear", "state"} + assert set(trigger[0]["params"]["methods"]) == {"register", "config", "remove", "clear", "state"} assert not any(e["uri"] == "urn:agentenv:get-triggers/v1" for e in exts) diff --git a/tst/unit/env/gateway/triggers_fires_test.py b/tst/unit/env/gateway/triggers_fires_test.py new file mode 100644 index 000000000..2df5d56b6 --- /dev/null +++ b/tst/unit/env/gateway/triggers_fires_test.py @@ -0,0 +1,492 @@ +"""What the trigger engine's state says of each fire in flight (running, waits, executor_calls), and its opt-in +settings (config): a repeat trigger's fires side by side, and a time limit for executors that set none.""" +from __future__ import annotations + +import asyncio +import random +from datetime import timedelta +from types import SimpleNamespace + +import httpx +import pytest + +from agent_env.env.gateway import gateway as gw_mod +from agent_env.env.gateway.clock import Clock, _iso, _parse_rfc3339 +from agent_env.env.gateway.constants import EXT_TRIGGERS_URI, GATEWAY_EXTENSIONS +from agent_env.env.gateway.gateway import Gateway +from agent_env.env.gateway.triggers import TriggerEngine, TriggerError +from tst.unit.env.gateway.next_event_test import MONDAY, _engine, _next, _row, _until +from tst.unit.env.gateway.triggers_time_test import _FakeGateway, _perm, _result + + +def _reply_trigger(tid: str = "reply", *, delay=None, executor: dict | None = None) -> dict: + action = {"type": "nl", "executor": "bob", "instruction": "answer Linda", **({"delay": delay} if delay else {})} + return {"watch_roles": ["default"], + "executors": {"bob": {"a2a_url": "http://bob", "role": "bob", **(executor or {})}}, + "triggers": [{"id": tid, "when": {"type": "action", "tool": "send_text", "repeat": True}, + "actions": [action]}]} + + +class _Executor: + """A stand-in executor: each call waits until `release()` lets it answer; records each call and how many ran at + once.""" + + def __init__(self, monkeypatch) -> None: + self.calls: list[dict] = [] + self.running = self.most_at_once = 0 + self.go = asyncio.Event() + executor = self + + async def call(engine, ex, instruction, context_id): + executor.calls.append({"timeout_seconds": ex["timeout_seconds"], "context_id": context_id, + "at": engine._gw._clock.now()}) + executor.running += 1 + executor.most_at_once = max(executor.most_at_once, executor.running) + try: + await executor.go.wait() + finally: + executor.running -= 1 + return "replied" + + monkeypatch.setattr(TriggerEngine, "_call_executor", call) + + def release(self) -> None: + self.go.set() + + +# ---- executor calls in state -------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_state_lists_an_executor_call_in_progress_with_the_time_left_of_its_limit(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.register(_reply_trigger(executor={"timeout_seconds": 300})) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + + row = _row(engine, "reply") + [call] = row["executor_calls"] + assert (call["executor"], call["action_index"], row["running"]) == ("bob", 0, 1) + assert 295 <= call["seconds_left"] <= 300 + + bob.release() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert _row(engine, "reply")["executor_calls"] == [] and _row(engine, "reply")["running"] == 0 + + +# ---- concurrent_fires ----------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_by_default_a_second_call_queues_behind_the_fire_running(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.register(_reply_trigger()) + engine.on_tool_call("default", "send_text", {}, _result()) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + + row = _row(engine, "reply") + assert (row["status"], row["pending"], row["running"], len(bob.calls)) == ("queued", 1, 1, 1) + bob.release() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert len(bob.calls) == 2 and bob.most_at_once == 1 + + +@pytest.mark.asyncio +async def test_side_by_side_each_fire_waits_out_its_own_delay(monkeypatch): + """A quick reply doesn't wait behind a slow one: each fire's delay runs from its own provoking call.""" + bob = _Executor(monkeypatch) + bob.release() + engine, clock, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register({**_reply_trigger(delay="PT2H")}) + engine.on_tool_call("default", "send_text", {}, _result()) + clock.advance("2026-03-02T01:00:00Z") + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(_row(engine, "reply")["waits"]) == 2) + + row = _row(engine, "reply") + assert (row["status"], row["running"], row["pending"]) == ("firing", 2, 0) + assert [w["resumes_at"] for w in row["waits"]] == ["2026-03-02T02:00:00Z", "2026-03-02T03:00:00Z"] + assert row["resumes_at"] == _next(engine) == "2026-03-02T02:00:00Z" + + clock.advance("2026-03-02T02:00:00Z") + assert engine.clock_advanced() == ["reply"] + await _until(lambda: len(bob.calls) == 1 and len(_row(engine, "reply")["waits"]) == 1) + assert _next(engine) == "2026-03-02T03:00:00Z" and _row(engine, "reply")["running"] == 1 + + clock.advance("2026-03-02T03:00:00Z") + engine.clock_advanced() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert [c["at"].isoformat() for c in bob.calls] == ["2026-03-02T02:00:00+00:00", "2026-03-02T03:00:00+00:00"] + assert _row(engine, "reply")["fire_count"] == 2 and _next(engine) is None + + +@pytest.mark.asyncio +async def test_side_by_side_fires_of_one_trigger_take_turns_with_its_executor_conversation(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register(_reply_trigger()) + for _ in range(3): + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + await asyncio.sleep(0.05) + + assert _row(engine, "reply")["running"] == 3 and len(bob.calls) == 1 + bob.release() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert len(bob.calls) == 3 and bob.most_at_once == 1 + assert {c["context_id"] for c in bob.calls} == {"trigger-reply"} + + +@pytest.mark.asyncio +async def test_two_triggers_reach_their_executors_at_once(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.register({"watch_roles": ["default"], "executors": {"bob": {"a2a_url": "http://bob", "role": "bob"}}, + "triggers": [{"id": tid, "when": {"type": "action", "tool": "send_text"}, + "actions": [{"type": "nl", "executor": "bob", "instruction": "answer"}]} + for tid in ("group", "dm")]}) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(bob.calls) == 2) + + assert bob.most_at_once == 2 + bob.release() + await _until(lambda: all(_row(engine, t)["status"] == "fired" for t in ("group", "dm"))) + + +@pytest.mark.asyncio +async def test_removing_a_trigger_cancels_its_fires_side_by_side(monkeypatch): + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register(_reply_trigger(delay="PT1H")) + engine.on_tool_call("default", "send_text", {}, _result()) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(_row(engine, "reply")["waits"]) == 2) + tasks = [engine._triggers["reply"]["task"], *engine._triggers["reply"]["alongside"]] + + engine.remove({"ids": ["reply"]}) + await asyncio.gather(*tasks, return_exceptions=True) + + assert all(t.cancelled() for t in tasks) and bob.calls == [] + + +@pytest.mark.asyncio +async def test_side_by_side_is_only_for_contact_replies_a_tool_action_trigger_still_queues(monkeypatch): + """A tool action may read a row and write it back; two of its fires at once could lose a write, so it queues.""" + bob = _Executor(monkeypatch) + engine, _, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register({"watch_roles": ["default"], + "executors": {"bob": {"a2a_url": "http://bob", "role": "bob"}}, + "triggers": [{"id": "mirror", "when": {"type": "action", "tool": "send_text", "repeat": True}, + "actions": [{"type": "nl", "executor": "bob", "instruction": "answer"}, _perm()]}]}) + engine.on_tool_call("default", "send_text", {}, _result()) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + + row = _row(engine, "mirror") + assert (row["status"], row["pending"], row["running"]) == ("queued", 1, 1) + bob.release() + await _until(lambda: _row(engine, "mirror")["status"] == "armed") + assert len(bob.calls) == 2 + + +@pytest.mark.asyncio +async def test_a_reply_that_notifies_the_agent_runs_side_by_side_so_its_due_time_shows_and_it_lands_on_time( + monkeypatch): + """A contact's reply followed by a notify, as a host's reply on a booking site: the second fire, with the shorter + delay, used to queue behind the first, unseen in next_event_at, and only drew its delay once the first ended, 17 + minutes late. Its delay now runs from its own call, its resumes_at shows, and it lands then, the first still + waiting.""" + bob = _Executor(monkeypatch) + bob.release() + notes: list[str] = [] + engine, clock, _ = _engine() + def notify(channel, text, trigger_id, to): + notes.append(_iso(clock.now())) + return {"id": f"n{len(notes)}", "to": to} + + engine._gw._notifications = SimpleNamespace(notify=notify) + engine.config({"concurrent_fires": True}) + trigger = _reply_trigger(delay={"dist": "exp", "mean": "PT40M", "seed": 30}) + trigger["triggers"][0]["actions"].append({"type": "notify", "channel": "Stayncook", "text": "New message"}) + engine.register(trigger) + draws = random.Random(30) + slow, quick = draws.expovariate(1 / 2400), draws.expovariate(1 / 2400) # 1859 s, then 819 s + engine.on_tool_call("default", "send_text", {}, _result()) + clock.advance("2026-03-02T00:00:01Z") + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(_row(engine, "reply")["waits"]) == 2) + + row = _row(engine, "reply") + due = _iso(_parse_rfc3339("2026-03-02T00:00:01Z") + timedelta(seconds=quick)) + assert (row["status"], row["pending"], row["running"]) == ("firing", 0, 2) + assert _next(engine) == due < _iso(_parse_rfc3339(MONDAY) + timedelta(seconds=slow)) + + clock.advance(due) + engine.clock_advanced() + await _until(lambda: len(notes) == 1) + assert [_iso(c["at"]) for c in bob.calls] == [due] and notes == [due] + assert _row(engine, "reply")["running"] == 1 + + +@pytest.mark.asyncio +async def test_each_fire_side_by_side_keeps_its_own_number_and_only_its_own_executor_calls(monkeypatch): + """Fire 1 waits out its delay while fire 2 starts: fire 1's call still says it is fire 1. Fire 2, waiting for its + turn on the executor conversation, is not credited with the calls the executor makes for fire 1.""" + seen: list[list[int]] = [] + engine, clock, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register(_reply_trigger(delay="PT1H")) + + async def call(eng, ex, instruction, context_id): + seen.append([c["fire"] for c in eng.caused_by("bob")]) + eng.on_tool_call("bob", "messaging_send", {}, _result()) # the executor's own tool call, made for this fire + await asyncio.sleep(0.05) + return "replied" + + monkeypatch.setattr(TriggerEngine, "_call_executor", call) + engine.on_tool_call("default", "send_text", {}, _result()) + clock.advance("2026-03-02T00:30:00Z") + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(_row(engine, "reply")["waits"]) == 2) + clock.advance("2026-03-02T01:30:00Z") # both delays end: the two calls take turns + engine.clock_advanced() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + + assert seen == [[1], [2]] + oks = [e["detail"] for e in engine._events if e["kind"] == "action_ok"] + assert [d["calls"] for d in oks] == [{"messaging_send": 1}, {"messaging_send": 1}] + assert not any("calls_shared_with" in d for d in oks) + + +@pytest.mark.asyncio +async def test_a_call_queued_once_side_by_side_is_turned_off_still_fires(monkeypatch): + """Fires side by side don't drain a queue; turned off meanwhile, a queued call used to be left behind for good.""" + bob = _Executor(monkeypatch) + bob.release() + engine, clock, _ = _engine() + engine.config({"concurrent_fires": True}) + engine.register(_reply_trigger(delay="PT1H")) + engine.on_tool_call("default", "send_text", {}, _result()) # the first fire, resumes 01:00 + clock.advance("2026-03-02T00:30:00Z") + engine.on_tool_call("default", "send_text", {}, _result()) # side by side, resumes 01:30 + clock.advance("2026-03-02T01:00:00Z") + engine.clock_advanced() + await _until(lambda: len(bob.calls) == 1 and _row(engine, "reply")["running"] == 1) + + engine.config({}) # side by side off + engine.on_tool_call("default", "send_text", {}, _result()) # queued behind the fire left + assert (_row(engine, "reply")["status"], _row(engine, "reply")["pending"]) == ("queued", 1) + clock.advance("2026-03-02T01:30:00Z") + engine.clock_advanced() + await _until(lambda: len(bob.calls) == 2 and _row(engine, "reply")["pending"] == 0) + clock.advance("2026-03-02T02:00:00Z") # its delay runs from its own call + engine.clock_advanced() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert len(bob.calls) == 3 and _row(engine, "reply")["fire_count"] == 3 + + +def test_a_fire_that_can_t_be_scheduled_leaves_no_count(): + engine, _, _ = _engine() + engine.register(_reply_trigger()) + trig = engine._triggers["reply"] + + async def fire(ticket): + return None + + with pytest.raises(RuntimeError): # no running event loop + engine._start_fire(trig, fire) + assert trig["running"] == 0 + + +# ---- executor_timeout_seconds --------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_configured_limit_applies_only_to_executors_that_set_none(monkeypatch): + bob = _Executor(monkeypatch) + bob.release() + engine, _, _ = _engine() + engine.config({"executor_timeout_seconds": 300}) + engine.register({"watch_roles": ["default"], + "executors": {"bob": {"a2a_url": "http://bob", "role": "bob"}, + "ann": {"a2a_url": "http://ann", "role": "ann", "timeout_seconds": 120}}, + "triggers": [{"id": tid, "when": {"type": "action", "tool": "send_text"}, + "actions": [{"type": "nl", "executor": tid, "instruction": "answer"}]} + for tid in ("bob", "ann")]}) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: len(bob.calls) == 2) + + assert sorted((c["context_id"], c["timeout_seconds"]) for c in bob.calls) == [ + ("trigger-ann", 120), ("trigger-bob", 300)] + assert engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": 300.0} + + +@pytest.mark.asyncio +async def test_without_config_an_executor_that_sets_no_limit_keeps_120_s(monkeypatch): + bob = _Executor(monkeypatch) + bob.release() + engine, _, _ = _engine() + engine.register(_reply_trigger()) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: bob.calls) + assert bob.calls[0]["timeout_seconds"] == 120 + + +@pytest.mark.parametrize("body", [{"concurrent_fires": "yes"}, {"executor_timeout_seconds": 0}, + {"executor_timeout_seconds": True}, {"executor_timeout_seconds": float("inf")}, + {"parallel": True}, []]) +def test_bad_settings_are_refused(body): + engine, _, _ = _engine() + with pytest.raises(TriggerError): + engine.config(body) + + +def test_settings_are_off_until_set_and_clear_resets_them(): + engine, _, _ = _engine() + assert engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": None} + assert engine.config({"concurrent_fires": True, "executor_timeout_seconds": 300})["concurrent_fires"] is True + engine.config({"concurrent_fires": True}) + assert engine.state()["settings"] == {"concurrent_fires": True, "executor_timeout_seconds": None} + engine.clear() + assert engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": None} + + +def test_the_card_offers_config(): + triggers = next(e for e in GATEWAY_EXTENSIONS if e["uri"] == EXT_TRIGGERS_URI) + assert triggers["params"]["methods"]["config"]["endpoint"] == "/triggers/config" + assert "fires_in_state" in triggers["params"]["features"] + + +@pytest.mark.asyncio +async def test_a_fire_counts_as_running_from_its_detection_before_its_task_first_runs(monkeypatch): + bob = _Executor(monkeypatch) + bob.release() + engine, clock, _ = _engine() + engine.register({"watch_roles": ["default"], "triggers": [ + {"id": "ping", "when": {"type": "time", "at": "PT1M"}, "actions": [_perm()]}]}) + clock.advance("2026-03-02T00:01:00Z") + + assert engine.clock_advanced() == ["ping"] + assert (_row(engine, "ping")["status"], _row(engine, "ping")["running"]) == ("firing", 1) + await _until(lambda: _row(engine, "ping")["status"] == "fired") + assert _row(engine, "ping")["running"] == 0 + + +@pytest.mark.asyncio +async def test_a_fire_cancelled_before_it_began_leaves_no_count(monkeypatch): + _Executor(monkeypatch) + engine, _, _ = _engine() + engine.register(_reply_trigger()) + engine.on_tool_call("default", "send_text", {}, _result()) + trig = engine._triggers["reply"] + trig["task"].cancel() + await asyncio.gather(trig["task"], return_exceptions=True) + await asyncio.sleep(0) + + assert trig["running"] == 0 + + +@pytest.mark.asyncio +async def test_a_fire_advanced_to_its_resumes_at_goes_on_though_its_delay_ends_between_microseconds(monkeypatch): + """On a frozen clock, nothing else moves time: an advance to the rounded resumes_at must be enough.""" + bob = _Executor(monkeypatch) + bob.release() + engine, clock, _ = _engine() + engine.register(_reply_trigger(delay={"dist": "exp", "mean": "PT40M", "seed": 30})) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: _row(engine, "reply")["waits"]) + resumes_at = _row(engine, "reply")["resumes_at"] + assert resumes_at == "2026-03-02T00:30:58.882035Z" # the draw is 1858.882035... s + + clock.advance(resumes_at) + engine.clock_advanced() + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert len(bob.calls) == 1 + + +# ---- a clock that runs only during the agent's turns ---------------------------------------------------------------- + +class _RunningNow: + """Real time under a running clock: it moves on a microsecond at every read, as time.monotonic does between two.""" + + def __init__(self) -> None: + self.t = 1000.0 + + def __call__(self) -> float: + self.t += 1e-6 + return self.t + + +def _running_engine(rate: float) -> tuple[TriggerEngine, Clock]: + clock = Clock(now_fn=_RunningNow()) + clock.set_time(MONDAY, rate=rate) + return TriggerEngine(_FakeGateway(clock)), clock + + +@pytest.mark.parametrize("past", [0, 90], ids=["to its resumes_at", "past it"]) +@pytest.mark.asyncio +async def test_a_delay_begun_on_a_running_clock_goes_on_when_the_frozen_clock_is_advanced_to_its_resumes_at( + monkeypatch, past): + """The always_on step's jump: a contact's reply starts its delay during the agent's turn, with the clock running + at the task's rate; between turns the step freezes it (set_rate 0) and advances it to the reply's resumes_at. + Nothing else moves the clock then, so that must be enough.""" + bob = _Executor(monkeypatch) + bob.release() + engine, clock = _running_engine(rate=60) + engine.register(_reply_trigger(delay={"dist": "exp", "mean": "PT40M", "seed": 30})) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: _row(engine, "reply")["waits"]) + clock.set_rate(0) + to = _parse_rfc3339(_row(engine, "reply")["resumes_at"]) + timedelta(seconds=past) + + clock.advance(_iso(to)) + assert engine.clock_advanced() == ["reply"] + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert [c["at"] for c in bob.calls] == [to] + + +@pytest.mark.asyncio +async def test_a_fire_waiting_on_a_running_clock_for_its_executor_s_hours_goes_on_when_the_frozen_clock_reaches_them( + monkeypatch): + bob = _Executor(monkeypatch) + bob.release() + engine, clock = _running_engine(rate=60) + engine.register(_reply_trigger(executor={"hours": {"start": "09:00", "end": "17:00"}})) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: _row(engine, "reply")["waits"]) + clock.set_rate(0) + assert _row(engine, "reply")["resumes_at"] == "2026-03-02T09:00:00Z" + + clock.advance("2026-03-02T09:00:00Z") + assert engine.clock_advanced() == ["reply"] + await _until(lambda: _row(engine, "reply")["status"] == "armed") + assert [_iso(c["at"]) for c in bob.calls] == ["2026-03-02T09:00:00Z"] + + +@pytest.mark.asyncio +async def test_a_rate_change_wakes_a_waiting_fire_to_go_on_when_the_clock_gets_to_its_time(monkeypatch, tmp_path): + """On a frozen clock a waiting fire looks at it again only every _VIRTUAL_RECHECK_S real seconds; the next turn's + set_rate wakes it to wait out the rest at the new rate, so it goes on when the clock gets to its time, not up to + that long after (five virtual minutes at 60x).""" + monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) + gw = Gateway(host="127.0.0.1", port=0, server_name="AgentEnvGateway", internal_mcp_servers=[]) + bob = _Executor(monkeypatch) + bob.release() + gw._clock.set_time(MONDAY, rate=0) + engine = gw._trigger_engine + engine.register(_reply_trigger(delay="PT1M")) + engine.on_tool_call("default", "send_text", {}, _result()) + await _until(lambda: _row(engine, "reply")["waits"]) + try: + transport = httpx.ASGITransport(app=gw._mcp.streamable_http_app()) + async with httpx.AsyncClient(transport=transport, base_url="http://gateway") as client: + r = await client.put("/clock/rate", json={"virtual_seconds_per_real_second": 3600}) + r.raise_for_status() + await _until(lambda: bob.calls, seconds=1) # a virtual minute at 3600x is a sixtieth of a real second + finally: + await engine.cancel_scheduled() + gw._close_trajectory_file() diff --git a/tst/unit/env/gateway/triggers_test.py b/tst/unit/env/gateway/triggers_test.py index f6897f64f..6a56d57af 100644 --- a/tst/unit/env/gateway/triggers_test.py +++ b/tst/unit/env/gateway/triggers_test.py @@ -2,8 +2,10 @@ from __future__ import annotations import asyncio +import contextlib import json import random +import re from datetime import datetime, timedelta, timezone from zoneinfo import ZoneInfo @@ -487,7 +489,7 @@ async def test_a_stochastic_delay_draws_the_same_sequence_from_its_seed(engine, await asyncio.sleep(0.1) rng = random.Random(7) - assert waits == [rng.expovariate(1.0 / 600) for _ in range(3)] + assert waits == pytest.approx([rng.expovariate(1.0 / 600) for _ in range(3)]) # each waited to its own mark assert len(engine._test_state["slack"]) == 3 @@ -498,8 +500,8 @@ async def test_a_re_arm_during_one_mark_s_delay_does_not_shorten_the_next(engine "args": {"channel": "#c", "text": "hi"}, "delay": "PT90M"}]}]}) record = TriggerEngine._sleep_virtual - async def re_arm_a_day_back_after_the_first_wait(self, seconds, generation=None): - await record(self, seconds, generation) + async def re_arm_a_day_back_after_the_first_wait(self, until, generation=None): + await record(self, until, generation) if len(virtual_time) == 1: self._gw._clock.set_time("2026-10-13T10:30:00Z") @@ -1111,6 +1113,104 @@ def do_POST(self): server.shutdown() +@contextlib.asynccontextmanager +async def _executor_answering_after(seconds: float): + """A loopback executor whose message/send, like a deployed one's, answers only once its task ends `seconds` later; + its task polls answer at once. Yields its URL.""" + from pytest_socket import enable_socket + + enable_socket() # re-disabled by the autouse fixture's next setup + handlers: set[asyncio.Task] = set() + + async def serve(reader, writer): + handlers.add(asyncio.current_task()) + try: + while True: # one connection carries the send and then the polls + head = await reader.readuntil(b"\r\n\r\n") + req = json.loads(await reader.readexactly(int(re.search(rb"(?i)content-length: *(\d+)", head)[1]))) + if req["method"] == "message/send": + await asyncio.sleep(seconds) + body = json.dumps({"jsonrpc": "2.0", "id": req["id"], "result": {"id": "task-1", "status": { + "state": "completed", "message": {"parts": [{"kind": "text", "text": "replied"}]}}}}).encode() + writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: %d\r\n\r\n%s" + % (len(body), body)) + await writer.drain() + except (asyncio.IncompleteReadError, ConnectionError): + pass # the caller hung up + finally: + writer.close() + + server = await asyncio.start_server(serve, "127.0.0.1", 0) + try: + yield f"http://127.0.0.1:{server.sockets[0].getsockname()[1]}" + finally: + server.close() + for task in handlers: + task.cancel() + await asyncio.gather(*handlers, return_exceptions=True) + + +def _reply_then_post(url: str, timeout_seconds: float) -> dict: + return {"executor": {"a2a_url": url, "role": "executor", "timeout_seconds": timeout_seconds}, + "triggers": [{"id": "x", "when": {"type": "action", "tool": "t"}, + "actions": [{"type": "nl", "instruction": "reply to the client"}, + {"type": "tool", "tool": "slack_send_message", + "args": {"channel": "#c", "text": "the client replied"}}]}]} + + +async def _settled(engine, tid: str, within: float = 10) -> None: + for _ in range(int(within / 0.05)): + if engine._triggers[tid]["status"] not in ("firing", "queued"): + return + await asyncio.sleep(0.05) + raise AssertionError(f"trigger {tid!r} still {engine._triggers[tid]['status']} after {within}s") + + +@pytest.mark.asyncio +async def test_an_executor_slower_than_the_http_timeout_is_waited_out_and_the_later_actions_run(engine, monkeypatch): + """FD-5333: a contact slower than 30 s used to fail its fire on a read timeout, though its reply still landed.""" + monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.2) # stands in for the 30 s + async with _executor_answering_after(1.0) as url: + engine.register(_reply_then_post(url, timeout_seconds=10)) + engine.on_tool_call("default", "t", {}, _result()) + await asyncio.sleep(0.4) + assert engine._triggers["x"]["status"] == "firing" # past the HTTP timeout, still in flight + await _settled(engine, "x") + + assert engine._triggers["x"]["status"] == "fired", engine.state()["events"][-3:] + assert [e["action_index"] for e in engine._events if e["kind"] == "action_ok"] == [0, 1] + assert engine._test_gw.calls == [("slack_send_message", {"channel": "#c", "text": "the client replied"})] + + +@pytest.mark.asyncio +async def test_a_reply_that_completes_just_past_the_executor_s_limit_is_kept(engine, monkeypatch): + """A deployed executor answers message/send when its task ends; one that ends a moment past the limit used to be + cut off by the read timeout, its completed reply dropped.""" + monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.5) + async with _executor_answering_after(1.2) as url: + engine.register(_reply_then_post(url, timeout_seconds=1.0)) + engine.on_tool_call("default", "t", {}, _result()) + await _settled(engine, "x", within=3) + + assert engine._triggers["x"]["status"] == "fired", engine.state()["events"][-3:] + assert engine._test_gw.calls == [("slack_send_message", {"channel": "#c", "text": "the client replied"})] + + +@pytest.mark.asyncio +async def test_an_executor_slower_than_its_own_timeout_still_fails_the_fire(engine, monkeypatch): + monkeypatch.setattr(triggers_module, "_EXECUTOR_HTTP_TIMEOUT_S", 0.2) + async with _executor_answering_after(3.0) as url: + engine.register(_reply_then_post(url, timeout_seconds=1.0)) + engine.on_tool_call("default", "t", {}, _result()) + await asyncio.sleep(0.4) + assert engine._triggers["x"]["status"] == "firing" + await _settled(engine, "x", within=2.5) + + [failure] = [e for e in engine._events if e["kind"] == "action_failed"] + assert (failure["action_index"], failure["detail"]) == (0, "ReadTimeout") + assert engine._triggers["x"]["status"] == "armed" and engine._test_gw.calls == [] + + @pytest.mark.asyncio async def test_broken_check_is_fail_open(engine): engine.register({"triggers": [ @@ -1199,7 +1299,7 @@ async def test_waiting_virtual_time_follows_the_clock_rate(engine, monkeypatch, stepped = _SteppedClock(engine, monkeypatch) if rate is not None: stepped.clock.set_time("2026-10-14T08:00:00Z", rate=rate) - await engine._sleep_virtual(virtual_seconds) + await engine._sleep_virtual(stepped.clock.elapsed() + virtual_seconds) assert stepped.slept == steps @@ -1208,7 +1308,7 @@ async def test_waiting_virtual_time_holds_while_frozen_and_follows_a_new_rate(en stepped = _SteppedClock(engine, monkeypatch) stepped.at[1] = lambda: stepped.clock.set_time("2026-10-14T08:00:00Z", rate=0) # after 5 unarmed seconds stepped.at[3] = lambda: stepped.clock.set_time("2026-10-14T08:00:00Z", rate=4) # where it froze, now at 4x - await engine._sleep_virtual(13) + await engine._sleep_virtual(stepped.clock.elapsed() + 13) assert stepped.slept == [5.0, 5.0, 5.0, 2.0] @@ -1219,7 +1319,7 @@ async def test_waiting_virtual_time_counts_a_re_arm_forward_but_not_back(engine, stepped = _SteppedClock(engine, monkeypatch) stepped.clock.set_time("2026-10-14T08:00:00Z") stepped.at[1] = lambda: stepped.clock.set_time(re_arm_to) # at 08:00:05: 3 s ahead, or a day back - await engine._sleep_virtual(10) + await engine._sleep_virtual(stepped.clock.elapsed() + 10) assert stepped.slept == steps @@ -1228,7 +1328,7 @@ async def test_a_wait_tied_to_a_clock_generation_ends_when_the_clock_moves_on(en stepped = _SteppedClock(engine, monkeypatch) stepped.clock.set_time("2026-10-14T08:00:00Z") stepped.at[1] = stepped.clock.clear - await engine._sleep_virtual(3600, generation=stepped.clock.generation) + await engine._sleep_virtual(stepped.clock.elapsed() + 3600, generation=stepped.clock.generation) assert stepped.slept == [5.0] @@ -1276,7 +1376,8 @@ def _fake_waits(engine, monkeypatch, armed_at=None) -> _Waits: waited = _Waits() waited.advance = lambda seconds: source.__setitem__("t", source["t"] + seconds / clock.rate) - async def fake_sleep(self, seconds, generation=None): + async def fake_sleep(self, until, generation=None): + seconds = (until - clock.now()).total_seconds() if isinstance(until, datetime) else until - clock.elapsed() waited.append(seconds) waited.advance(max(0.0, seconds)) diff --git a/tst/unit/env/gateway/user_chat_test.py b/tst/unit/env/gateway/user_chat_test.py new file mode 100644 index 000000000..c286e9021 --- /dev/null +++ b/tst/unit/env/gateway/user_chat_test.py @@ -0,0 +1,181 @@ +"""Unit tests for the user chat (urn:agentenv:user-chat/v1): message_user and stay_quiet, built in for the roles a +config names.""" +from __future__ import annotations + +import asyncio + +import httpx +import pytest +from mcp.types import Tool as MCPTool + +from agent_env.env.gateway import gateway as gw_mod +from agent_env.env.gateway.clock import Clock +from agent_env.env.gateway.constants import AGENT_ENV_ROLE_HEADER, EXT_USER_CHAT_URI, GATEWAY_EXTENSIONS +from agent_env.env.gateway.gateway import Gateway + +T0 = "2027-02-01T15:00:00Z" + + +@pytest.fixture +def gateway(tmp_path, monkeypatch): + monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) + gw = Gateway(host="127.0.0.1", port=0, server_name="AgentEnvGateway", internal_mcp_servers=[]) + clock = Clock() + clock.set_time(T0, 0) + gw._clock = clock + gw._server_tools["mcp-svc"] = [MCPTool(name="svc_thing", description="", inputSchema={"type": "object"})] + gw._tools_discovered = True + yield gw + gw._close_trajectory_file() + + +def _req(gw, method, path, body=None, role=None): + async def _do(): + transport = httpx.ASGITransport(app=gw._mcp.streamable_http_app()) + async with httpx.AsyncClient(transport=transport, base_url="http://gateway") as client: + return await client.request(method, path, json=body, + headers={AGENT_ENV_ROLE_HEADER: role} if role else None) + return asyncio.run(_do()) + + +def _names(gw, role): + return {t["name"] for t in _req(gw, "POST", "/step", {"action": "list_tools"}, role=role).json()["tools"]} + + +def _call(gw, tool, arguments, role): + return _req(gw, "POST", "/step", {"action": "call_tool", "tool_name": tool, "arguments": arguments}, role=role) + + +def test_the_card_offers_config_state_take_and_clear(): + [ext] = [e for e in GATEWAY_EXTENSIONS if e["uri"] == EXT_USER_CHAT_URI] + assert set(ext["params"]["methods"]) == {"config", "state", "take", "clear"} + assert "stay_quiet" in ext["params"]["methods"]["config"]["request"]["properties"] + + +def test_message_user_reaches_a_configured_role_through_a_deny_all(gateway): + _req(gateway, "POST", "/tools/disable", {"role": "psi_agent", "tools": ["*"]}) + assert "message_user" not in _names(gateway, "psi_agent") + assert _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "user_name": "Tess"}).status_code == 200 + + names = _names(gateway, "psi_agent") + assert "message_user" in names and "svc_thing" not in names + assert "message_user" not in _names(gateway, "default") + assert _call(gateway, "message_user", {"text": "hi Tess, two questions"}, "psi_agent").status_code == 200 + assert _call(gateway, "message_user", {"text": "x"}, "default").status_code == 403 + taken = _req(gateway, "POST", "/user_chat/take", {"take_id": "a"}).json()["items"] + assert [(m["from"], m["at"], m["text"]) for m in taken] == [("psi_agent", T0, "hi Tess, two questions")] + assert _req(gateway, "POST", "/user_chat/take", {"take_id": "a"}).json()["items"] == taken + assert _req(gateway, "POST", "/user_chat/take", {}).json()["items"] == [] + + +def test_stay_quiet_is_off_unless_the_config_turns_it_on(gateway): + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "user_name": "Tess"}) + assert "message_user" in _names(gateway, "psi_agent") and "stay_quiet" not in _names(gateway, "psi_agent") + called = _call(gateway, "stay_quiet", {}, "psi_agent") + assert called.status_code != 200 or called.json().get("isError") + assert _req(gateway, "POST", "/user_chat/take", {"take_id": "a"}).json() == {"items": []} + + +def test_stay_quiet_sends_nothing_and_is_recorded_for_the_next_take(gateway): + _req(gateway, "POST", "/tools/disable", {"role": "psi_agent", "tools": ["*"]}) + st = _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "user_name": "Tess", "stay_quiet": True}) + assert st.status_code == 200 and st.json()["stay_quiet"] is True + assert {"message_user", "stay_quiet"} <= _names(gateway, "psi_agent") + assert "stay_quiet" not in _names(gateway, "default") + + assert _call(gateway, "stay_quiet", {}, "psi_agent").status_code == 200 + assert _call(gateway, "stay_quiet", {}, "default").status_code == 403 + taken = _req(gateway, "POST", "/user_chat/take", {"take_id": "b"}).json() + assert taken == {"items": [], "quiet": [{"from": "psi_agent", "at": T0}]} + assert _req(gateway, "POST", "/user_chat/take", {"take_id": "b"}).json() == taken + assert _req(gateway, "POST", "/user_chat/take", {}).json() == {"items": [], "quiet": []} + [desc] = [t["description"] for t in + _req(gateway, "POST", "/step", {"action": "list_tools"}, role="psi_agent").json()["tools"] + if t["name"] == "stay_quiet"] + assert "Tess" in desc + + +def test_a_take_for_one_role_leaves_the_others_messages_and_quiet_calls_queued(gateway): + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent", "other"], "stay_quiet": True}) + _call(gateway, "message_user", {"text": "from psi"}, "psi_agent") + _call(gateway, "message_user", {"text": "from other"}, "other") + _call(gateway, "stay_quiet", {}, "other") + + taken = _req(gateway, "POST", "/user_chat/take", {"from": "psi_agent"}).json() + assert [(m["from"], m["text"]) for m in taken["items"]] == [("psi_agent", "from psi")] and taken["quiet"] == [] + assert [m["text"] for m in _req(gateway, "GET", "/user_chat/state").json()["pending"]] == ["from other"] + rest = _req(gateway, "POST", "/user_chat/take", {}).json() + assert [(m["from"], m["text"]) for m in rest["items"]] == [("other", "from other")] + assert [q["from"] for q in rest["quiet"]] == ["other"] + + +@pytest.mark.parametrize("who", ["", 3]) +def test_a_take_from_no_role_is_refused(gateway, who): + assert _req(gateway, "POST", "/user_chat/take", {"from": who}).status_code == 400 + + +def test_a_new_config_or_clear_removes_stay_quiet(gateway): + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "stay_quiet": True}) + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) + assert "stay_quiet" not in _names(gateway, "psi_agent") and "message_user" in _names(gateway, "psi_agent") + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "stay_quiet": True}) + _req(gateway, "POST", "/user_chat/clear", {}) + assert not {"stay_quiet", "message_user"} & _names(gateway, "psi_agent") + + +def test_turning_the_chat_off_or_clearing_it_leaves_a_backing_server_s_stay_quiet(gateway): + gateway._tool_server_urls["stay_quiet"] = "http://backing" + gateway._create_and_register_proxy_tool("http://backing", MCPTool( + name="stay_quiet", description="backing", inputSchema={"type": "object", "properties": {}})) + tools = gateway._mcp._tool_manager._tools + + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "stay_quiet": True}) + _req(gateway, "POST", "/user_chat/config", {"roles": []}) + assert tools["stay_quiet"].description == "backing" and "message_user" not in tools + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) + _req(gateway, "POST", "/user_chat/clear", {}) + assert tools["stay_quiet"].description == "backing" and "message_user" not in tools + + +@pytest.mark.parametrize("body", [{"roles": ["psi_agent"], "stay_quiet": "yes"}, {"roles": "psi_agent"}, + {"roles": ["psi_agent"], "colour": "red"}]) +def test_a_bad_config_is_refused(gateway, body): + assert _req(gateway, "POST", "/user_chat/config", body).status_code == 400 + + +def test_an_unconfigured_gateway_lists_neither_tool(gateway): + assert not {"message_user", "stay_quiet"} & _names(gateway, "default") + + +def test_get_time_reaches_a_configured_role_through_a_deny_all_while_the_clock_is_set(gateway): + _req(gateway, "POST", "/tools/disable", {"role": "psi_agent", "tools": ["*"]}) + _req(gateway, "POST", "/tools/disable", {"role": "other", "tools": ["*"]}) + assert "get_time" not in _names(gateway, "psi_agent") + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) + assert "get_time" not in _names(gateway, "psi_agent") # no clock set, so no get_time at all + + assert _req(gateway, "PUT", "/clock/set-time", {"virtual_time": T0}).status_code == 200 + assert "get_time" in _names(gateway, "psi_agent") and "get_time" not in _names(gateway, "other") + called = _call(gateway, "get_time", {}, "psi_agent") + assert called.status_code == 200 and T0 in called.text + _req(gateway, "POST", "/user_chat/clear", {}) + assert "get_time" not in _names(gateway, "psi_agent") + + +def test_the_check_tools_reach_a_configured_role_through_a_deny_all_once_notifications_name_it(gateway): + checks = {"schedule_check", "list_scheduled_checks", "cancel_scheduled_check"} + _req(gateway, "POST", "/tools/disable", {"role": "psi_agent", "tools": ["*"]}) + _req(gateway, "POST", "/notifications/config", {"roles": ["psi_agent"]}) + assert not checks & _names(gateway, "psi_agent") # the task's deny-all, as before + + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"]}) + assert checks <= _names(gateway, "psi_agent") + _req(gateway, "POST", "/notifications/config", {"roles": []}) + assert not checks & _names(gateway, "psi_agent") + + +def test_message_user_and_stay_quiet_are_read_only_for_the_world_so_they_don_t_re_evaluate_state_triggers(gateway): + _req(gateway, "POST", "/user_chat/config", {"roles": ["psi_agent"], "stay_quiet": True}) + engine = gateway._trigger_engine + assert engine._is_readonly("message_user") and engine._is_readonly("stay_quiet") + assert not engine._is_readonly("svc_thing") diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py new file mode 100644 index 000000000..d460a97ea --- /dev/null +++ b/tst/unit/task_step/always_on_harness.py @@ -0,0 +1,290 @@ +"""A harness for the always_on step: an in-process gateway (clock, triggers, notifications, user chat) behind a real +DeployedGatewayEnv, an agent whose turns are scripted and act on that gateway as its role, and a user sim plug-in whose +answers are scripted. Shared by the always_on step's tests.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any, Callable, Optional + +import httpx +from mcp.types import CallToolResult, TextContent, Tool + +from agent_env.a2a_agent import A2AAgent, conversation_store +from agent_env.config import get_config +from agent_env.env.env import DeployedGatewayEnv +from agent_env.env.gateway import gateway as gw_mod +from agent_env.env.gateway.constants import AGENT_ENV_ROLE_HEADER, GATEWAY_EXTENSIONS, WELL_KNOWN_PATH +from agent_env.env.gateway.gateway import Gateway +from agent_env.env.gateway.triggers import TriggerEngine +from agent_env.task_step.context import DeployedAgent, TaskStepContext +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.user_sim import USER_SIMS_KEY + +GATEWAY = "http://gateway.test" +SOLVER = "http://solver.test" +USER_SIM = "http://user-sim.test" +T0 = "2026-11-09T14:00:00Z" # Mon Nov 9, 9:00 AM in New York +ROLE = "default" + +Turn = Callable[[int, list[str], "Rig"], Optional[str]] +# An agent card whose agent-config extension takes the settings the step sends. +CONFIG_CARD = {"capabilities": {"extensions": [{"uri": A2AAgent.EXT_AGENT_CONFIG, "params": { + "endpoint": "/ext/agent-config", "methods": {"set": {"request": {"supported": [ + "model", "system_prompt", "effort", "output_format", "timeout_seconds", "max_turns"]}}}}}]}} + + +class Fail: + """What an agent's turn returns to end in the failed state, as a harness reports a turn that broke; or in another + state that isn't completed (``state``: canceled, rejected).""" + + def __init__(self, message: str = "the CLI crashed", error_type: Optional[str] = None, + state: str = "failed") -> None: + self.message, self.error_type, self.state = message, error_type, state + + +class Rig: + """The world, the agent and the user sim for one always_on run. + + ``agent(n, texts, rig)`` plays the agent's nth turn (nudges included) given the texts it was sent, acting through + ``message``, ``quiet`` and ``schedule``, and returns its closing text; by default it stays quiet. ``answers`` are the + user sim's answers to its + reads, in order; once they run out it answers no_reply. Every message the agent got is in ``sent``, every read the + user sim got in ``reads``. A ``plugin`` replaces the scripted one; what it sends the user sim agent with ``ask`` is + in ``user_calls``, answered in turn by ``user_texts`` (then no_reply). ``during`` has, for each of the agent's + turns, the gateway's settings and the agent's tools as the agent saw them. With ``takes_config`` the agent's card + takes its config (``CONFIG_CARD``), whose posts land in ``configs``.""" + + def __init__(self, monkeypatch, tmp_path, *, agent: Optional[Turn] = None, answers: tuple = (), rate: float = 0, + t0: str = T0, plugin: Any = None, user_texts: tuple = (), takes_config: bool = False): + monkeypatch.setattr(gw_mod, "GATEWAY_TRAJECTORY_FILE", str(tmp_path / "trajectory.jsonl")) + self.gw = Gateway(host="127.0.0.1", port=0, server_name="AgentEnvGateway", internal_mcp_servers=[]) + self.gw._tools_discovered = True + real_client = httpx.AsyncClient + transport = httpx.ASGITransport(app=self.gw._mcp.streamable_http_app()) + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: real_client(*a, **{**kw, "transport": transport})) + self.http = lambda: real_client(transport=transport, base_url=GATEWAY) + self.t0, self.rate = t0, rate + self.agent = agent or (lambda n, texts, rig: (rig.quiet(), "")[1]) + self.answers = list(answers) + self.sent: list[list[str]] = [] + self.parts: list[list[dict]] = [] + self.reads: list[dict] = [] + self.setups: list[dict] = [] + self.plugin = plugin + self.user_texts = list(user_texts) + self.user_calls: list[dict] = [] + self.contact_calls: list[dict] = [] + self.during: list[dict] = [] + self.solver_contexts: list[Optional[str]] = [] + self.takes_config = takes_config + self.configs: list[tuple[str, dict]] = [] + self._contacts: dict[str, dict] = {} + self._monkeypatch = monkeypatch + monkeypatch.setattr(ao, "POLL_SECONDS", 0) + monkeypatch.setattr(ao, "send_and_wait", self._send_and_wait) + monkeypatch.setattr(ao._Run, "_load_plugin", self._load_plugin) + if takes_config: + async def post(url, fields): + self.configs.append((url, fields)) + monkeypatch.setattr(ao.protocol, "post_agent_config", post) + + # ---- the world ----------------------------------------------------------------------------------------------- + async def arm(self) -> None: + """Arm the gateway clock the way sync_env_clock does.""" + async with self.http() as client: + r = await client.put("/clock/set-time", json={"virtual_time": self.t0, + "virtual_seconds_per_real_second": self.rate}) + r.raise_for_status() + + def triggers(self, *triggers: dict, executors: Optional[dict] = None) -> None: + self.gw._trigger_engine.register({"watch_roles": [ROLE], "triggers": list(triggers), + **({"executors": executors} if executors else {})}) + + def contact(self, name: str, *, seconds: float, reply: Callable[["Rig", int], None], + timeout_seconds: Optional[float] = None, keeps_to_its_limit: bool = True) -> dict: + """An AI-played contact (a trigger's executor) whose every reply takes ``seconds`` of real time, then acts with + ``reply(rig, n)`` for its nth reply. Past its time limit the call fails, though the reply still lands, as a + real contact's does; one that doesn't ``keep_to_its_limit`` holds its call open until it is done. Returns its + executor spec, for ``triggers(executors={name: spec})``.""" + url = f"http://{name}.test" + self._contacts[url] = {"name": name, "seconds": seconds, "reply": reply, "keeps": keeps_to_its_limit, + "replies": 0} + rig = self + + async def call(engine, executor, instruction, context_id): + contact = rig._contacts[executor["a2a_url"]] + rig.contact_calls.append({"contact": contact["name"], "timeout_seconds": executor["timeout_seconds"], + "context_id": context_id, "at": rig.now()}) + contact["replies"] += 1 + n, limit = contact["replies"], executor["timeout_seconds"] + if contact["keeps"] and contact["seconds"] > limit: + await asyncio.sleep(limit) + asyncio.get_running_loop().call_later(contact["seconds"] - limit, contact["reply"], rig, n) + return None + await asyncio.sleep(contact["seconds"]) + contact["reply"](rig, n) + return "replied" + + self._monkeypatch.setattr(TriggerEngine, "_call_executor", call) + return {"a2a_url": url, "role": f"contact-{name}", + **({"timeout_seconds": timeout_seconds} if timeout_seconds is not None else {})} + + def alert(self, text: str, *, channel: str = "Gmail") -> dict: + """Something reaching the agent the way a contact's email alert does.""" + return self.gw._notifications.notify(channel, text, to=ROLE) + + def keep_changelog(self) -> None: + """Give the gateway an app changelog (urn:agentenv:changes/v1), held here; ``app_write`` adds to it.""" + self.changelog: list[dict] = [] + self.gw._db_conn = object() # the gateway serves changes only for an env with a database + self.gw._query_changelog_id = lambda: self.changelog[-1]["id"] if self.changelog else None + self.gw._read_changelog = lambda after_id, count: [r for r in self.changelog if r["id"] > after_id][:count] + + def app_write(self, table: str, summary: str = "", *, tool: str = "app_tool", role: Optional[str] = ROLE, + trigger_id: Optional[str] = None) -> int: + """One change to an app's data: by ``role``'s tool call, by trigger ``trigger_id``'s tool action, or, with + neither, by nothing the gateway served (a website reached directly). Returns its changelog id.""" + before = self.changelog[-1]["id"] if self.changelog else 0 + self.changelog.append({"id": before + 1, "timestamp": "2026-11-09T14:00:00", "schema_name": "public", + "table_name": table, "operation": "INSERT", "row_id": str(before + 1), + "summary": summary, "changed_fields": None}) + changes, n = self.gw._changes, len(self.changelog) + if trigger_id is not None: + changes.observe({"event_type": "internal_tool_call", "event_id": f"w{n}", "trigger_id": trigger_id, + "action_index": 0, "tool": tool, "as": None, "changelog_id_before": before, + "changelog_id_after": before + 1}) + elif role is not None: + changes.observe({"event_type": "tool_call", "event_id": f"c{n}", "role": role, + "tool_call": {"function_name": tool}}) + changes.observe({"event_type": "tool_call_result", "event_id": f"r{n}", "tool_call_event_id": f"c{n}", + "changelog_id_before": before, "changelog_id": before + 1}) + return before + 1 + + def app_tools(self, *names: str) -> None: + """App tools the gateway knows of, so a trigger may watch them; nothing serves them.""" + self.gw._server_tools.setdefault("apps", []).extend(Tool(name=n, inputSchema={"type": "object"}) for n in names) + + def provoke(self, tool: str, args: Optional[dict] = None) -> None: + """The agent's tool call as the gateway's triggers see it.""" + self.gw._trigger_engine.on_tool_call(ROLE, tool, args or {}, + CallToolResult(content=[TextContent(type="text", text="{}")])) + + async def close(self) -> None: + """Cancel what the gateway's triggers still have running.""" + await self.gw._trigger_engine.cancel_scheduled() + + def notify_at(self, trigger_id: str, at: str, text: str, *, channel: str = "Messages", to: str = ROLE) -> dict: + return {"id": trigger_id, "when": {"type": "time", "at": at}, + "actions": [{"type": "notify", "channel": channel, "text": text, "to": to}]} + + def now(self) -> str: + return self.gw._clock.read()["virtual_time"] + + # ---- the agent's tools ----------------------------------------------------------------------------------------- + def message(self, text: str) -> dict: + return self.gw._user_chat.send(text, role=ROLE) + + def quiet(self) -> dict: + return self.gw._user_chat.quiet(role=ROLE) + + def schedule(self, note: str, at: str) -> dict: + return self.gw._notifications.schedule(note, at=at, to=ROLE) + + async def tools(self, role: str = ROLE) -> set[str]: + async with self.http() as client: + r = await client.post("/step", json={"action": "list_tools"}, headers={AGENT_ENV_ROLE_HEADER: role}) + return {t["name"] for t in r.json()["tools"]} + + # ---- the run --------------------------------------------------------------------------------------------------- + def context(self, *, user: Optional[dict] = None) -> TaskStepContext: + card = {"capabilities": {"extensions": list(GATEWAY_EXTENSIONS)}} + env = DeployedGatewayEnv(env_id="world", env_version=1, gateway_url=GATEWAY, mcp_url="", sandbox_id="sb", + environment_card_url=f"{GATEWAY}{WELL_KNOWN_PATH}", environment_card=card, + metadata={"deploy_step_id": "deploy"}) + ctx = TaskStepContext(instance_id="ti-1", deployed_envs=[env]) + ctx.deployed_agents += [DeployedAgent(agent_name="solver", api_url=SOLVER, a2a_url=SOLVER, + a2a_card=CONFIG_CARD if self.takes_config else {}), + DeployedAgent(agent_name="user-sim", api_url=USER_SIM, a2a_url=USER_SIM, a2a_card={})] + ctx.metadata[USER_SIMS_KEY] = {"linda": { + "name": "linda", "model": "sim-model", "agent_name": "user-sim", "user_version": "ours", + "display_name": "Linda Murphy", "first_name": "Linda", "time_zone": "America/New_York", + "description": "Linda, planning Noor's 30th.", **(user or {})}} + return ctx + + def step(self, **settings: Any) -> ao.AlwaysOnTaskStep: + return ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="sort the cdmx weekend", prompt_id="ask", + agent_name="solver", **settings) + + async def run(self, step: ao.AlwaysOnTaskStep, ctx: Optional[TaskStepContext] = None) -> TaskStepContext: + ctx = ctx or self.context() + await step.execute(ctx) + return ctx + + def record(self, ctx: TaskStepContext) -> dict: + """The step's summary in the run's metadata.""" + return ctx.metadata[ao.ALWAYS_ON_KEY]["week"] + + def stored(self, ctx: TaskStepContext) -> dict: + """The step's full record, its timeline included, as the object store holds it.""" + url = self.record(ctx)["object_url"] + return json.loads(get_config().get_object_store_at(url).get(url)) + + def timeline(self, ctx: TaskStepContext) -> list[dict]: + return self.stored(ctx)["timeline"] + + def chat(self, ctx: TaskStepContext) -> list[dict]: + """The graded conversation as stored: each message's role and text parts.""" + doc = conversation_store.get_conversation(ctx.metadata["a2a_conversations"]["week"]) + return [{"role": m["role"], "texts": [p["text"] for p in m["parts"] if p.get("kind") == "text"], + "parts": m["parts"]} for m in doc["messages"]] + + def events(self, ctx: TaskStepContext, name: str) -> list[dict]: + return [e for e in self.timeline(ctx) if e["event"] == name] + + # ---- fakes ----------------------------------------------------------------------------------------------------- + async def _send_and_wait(self, url, parts, *, before_send=None, **kw): + if url == USER_SIM: + self.user_calls.append({"parts": list(parts), "context_id": kw.get("context_id")}) + text = self.user_texts.pop(0) if self.user_texts else '{"outcome": "no_reply", "message": ""}' + return f"user-{len(self.user_calls)}", {"status": {"state": "completed", "message": { + "parts": [{"kind": "text", "text": text}]}}} + assert url == SOLVER, url + self.solver_contexts.append(kw.get("context_id")) + if before_send: + before_send() + chat, notes = self.gw._user_chat, self.gw._notifications + self.during.append({"user_chat_roles": chat.roles, "stay_quiet": chat.stay_quiet, "check_roles": notes.roles, + "min_interval": notes.min_interval, "max_active": notes.max_active, + "trigger_settings": dict(self.gw._trigger_engine.state()["settings"]), + "clock_rate": self.gw._clock.rate, "tools": await self.tools()}) + texts = [p["text"] for p in parts if p.get("kind") == "text"] + self.sent.append(texts) + self.parts.append(list(parts)) + closing = self.agent(len(self.sent), texts, self) # a TimeoutError it raises is the turn timing out + if isinstance(closing, Fail): + data = {"error_message": closing.message, **({"error_type": closing.error_type} + if closing.error_type else {})} + return f"task-{len(self.sent)}", {"status": {"state": closing.state, "message": {"parts": [ + {"kind": "text", "text": closing.message}, {"kind": "data", "data": data}]}}} + reply = [{"kind": "text", "text": closing}] if closing else [] + return f"task-{len(self.sent)}", {"status": {"state": "completed", "message": {"parts": reply}}} + + async def _load_plugin(self, spec): + if self.plugin is not None: + return self.plugin + rig = self + + class Plugin: + @staticmethod + def setup(version, person): + rig.setups.append(person) + return {"system_prompt": f"you are {person['name']}", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + rig.reads.append(read) + return rig.answers.pop(0) if rig.answers else {"outcome": "no_reply", "message": ""} + + return Plugin diff --git a/tst/unit/task_step/test_always_on_agent_and_gateway.py b/tst/unit/task_step/test_always_on_agent_and_gateway.py new file mode 100644 index 000000000..26009cb06 --- /dev/null +++ b/tst/unit/task_step/test_always_on_agent_and_gateway.py @@ -0,0 +1,99 @@ +"""What the step sets on the agent and the gateways, and what it leaves: the message_user rules reach the agent even when +its card won't take a system prompt; the step's context_id is the agent's session, as prompt_agent's is; and the +gateway settings the step changed for its run are as they were once it ends, for the task's later steps.""" + +from __future__ import annotations + +import logging + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import SOLVER, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +RULES = f"{ao.MESSAGE_RULES} {ao.QUIET_RULE}" +PROMPT = "You are Linda's assistant." + + +@pytest.mark.asyncio +async def test_the_rules_go_on_the_first_wake_when_the_agent_won_t_take_a_system_prompt(monkeypatch, tmp_path, caplog): + rig = Rig(monkeypatch, tmp_path) # its card takes no config + await rig.arm() + caplog.set_level("WARNING", logger=ao.logger.name) + ctx = await rig.run(rig.step(system_prompt=PROMPT)) + + assert rig.sent[0][0] == RULES and rig.parts[0][0]["metadata"] == {"agentenv": {"source": "rules"}} + assert rig.record(ctx)["settings"]["rules_via"] == "first_wake" + assert any("did not accept system_prompt" in r.getMessage() for r in caplog.records + if r.levelno == logging.WARNING) + + +@pytest.mark.asyncio +async def test_the_rules_go_in_the_system_prompt_when_the_agent_takes_it(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, takes_config=True) + await rig.arm() + ctx = await rig.run(rig.step(system_prompt=PROMPT)) + + [(url, fields)] = [(u, f) for u, f in rig.configs if u.startswith(SOLVER)] + assert fields["system_prompt"] == f"{PROMPT}\n\n{RULES}" + assert RULES not in rig.sent[0] + assert rig.record(ctx)["settings"]["rules_via"] == "system_prompt" + + +@pytest.mark.asyncio +async def test_the_step_s_context_id_is_the_agent_s_session(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message("blue or green?") if n == 1 else rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "blue"}]) + await rig.arm() + ctx = await rig.run(rig.step(context_id="cdmx-weekend")) + + assert rig.solver_contexts == ["cdmx-weekend", "cdmx-weekend"] + assert ctx.prompt_responses[0].a2a_context_id == "cdmx-weekend" + assert ao.AlwaysOnTaskStep.from_dict(rig.step(context_id="cdmx-weekend").to_dict()).context_id == "cdmx-weekend" + + +@pytest.mark.asyncio +async def test_without_a_context_id_each_run_has_a_session_of_its_own(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = await rig.run(rig.step()) + [session] = set(rig.solver_contexts) + assert session and ctx.prompt_responses[0].a2a_context_id == session + + +@pytest.mark.asyncio +async def test_the_gateway_settings_the_step_changed_are_as_they_were_after_the_run(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, rate=60) + await rig.arm() + gw = rig.gw + gw._user_chat.configure({"roles": ["planner"], "user_name": "Tess"}) + gw._trigger_engine.config({"executor_timeout_seconds": 90}) + gw._notifications.configure({"roles": ["planner"], "min_interval": "PT2H", "max_active": 4}) + await rig.run(rig.step(end_time="P2D")) + + during = rig.during[0] + assert during["user_chat_roles"] == ("default",) and during["stay_quiet"] + assert during["trigger_settings"] == {"concurrent_fires": True, "executor_timeout_seconds": 300.0} + assert during["check_roles"] == ("planner", "default") and during["clock_rate"] == 60 + assert (gw._user_chat.roles, gw._user_chat.user_name, gw._user_chat.stay_quiet) == (("planner",), "Tess", False) + assert gw._trigger_engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": 90.0} + assert (gw._notifications.roles, gw._notifications.min_interval, gw._notifications.max_active) == ( + ("planner",), "PT2H", 4) + assert gw._clock.rate == 60 + + +@pytest.mark.asyncio +async def test_a_gateway_the_task_left_alone_is_back_to_its_defaults(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + await rig.run(rig.step(end_time="P2D")) + + gw = rig.gw + assert (gw._user_chat.roles, gw._user_chat.stay_quiet, gw._notifications.roles) == ((), False, ()) + assert gw._trigger_engine.state()["settings"] == {"concurrent_fires": False, "executor_timeout_seconds": None} + assert not {"message_user", "stay_quiet", "schedule_check"} & await rig.tools() diff --git a/tst/unit/task_step/test_always_on_budget.py b/tst/unit/task_step/test_always_on_budget.py new file mode 100644 index 000000000..602792846 --- /dev/null +++ b/tst/unit/task_step/test_always_on_budget.py @@ -0,0 +1,205 @@ +"""The always_on step's real-time budget, against the hub's 8-hour job: once one more wake might end past the budget, +the agent gets its closing wake (the last-turn notice) and the run ends cleanly (time_budget) instead of being cut off +and retried. Settings whose worst case can't fit the job are refused.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +NOTICE = ao.FINAL_REMINDER.format(first="Linda") + + +def _step(**settings) -> ao.AlwaysOnTaskStep: + return ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", prompt_id="p", agent_name="a", **settings) + + +@pytest.mark.asyncio +async def test_once_another_wake_might_end_past_the_budget_the_agent_gets_its_closing_wake(monkeypatch, tmp_path): + def agent(n, texts, rig): + if NOTICE in texts: + rig.message("RESULT: wrapping up early, La Ola still open") + return "" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "ok thanks"}]) + monkeypatch.setattr(ao._Run, "spent", lambda run: 3600.0 * len(rig.sent)) # each turn takes an hour + await rig.arm() + for day in range(10, 20): + rig.triggers(rig.notify_at(f"alert-{day}", f"2026-11-{day}T14:00:00Z", "Sam: news?", channel="noor 30")) + step = rig.step(time_budget_seconds=12120) # reserve 4380 s: closes once 7740 s are spent + ctx = await rig.run(step) + + record = rig.record(ctx) + assert step.wake_reserve_seconds() == 4380 and record["settings"]["time_budget_seconds"] == 12120 + wakes = rig.events(ctx, "wake") + assert len(wakes) == 4 and wakes[-1]["woke_by"] == ["step:final"] and wakes[-1]["last"] is True + assert wakes[-1]["virtual_time"] == "2026-11-11T14:00:00Z" # where the run was; the clock doesn't jump + assert ao.FINAL_DEFAULT_TEXT.format(first="Linda") in rig.sent[-1] and NOTICE in rig.sent[-1] + [reached] = rig.events(ctx, "time_budget_reached") + assert reached["real_seconds"] == 10800 + assert record["end_reason"] == "time_budget" and record["error_type"] is None + assert rig.events(ctx, "user_sim_read")[-1]["last_read"] is True + assert rig.chat(ctx)[-1]["texts"][-1].startswith("[End of run · Wed Nov 11, 2:00 PM UTC] The run was cut off by " + "its real-time budget.") + + +@pytest.mark.parametrize("timeout,hours", [(600, 6.62), (1800, 5.95), (3600, 4.95)]) +def test_the_default_budget_is_7_h_lowered_so_the_closing_wake_fits_the_8_h_job(timeout, hours): + step = _step(timeout_seconds=timeout) + budget = step.effective_time_budget() + assert round(budget / 3600, 2) == hours + assert budget + step.wake_reserve_seconds() + ao.JOB_HEADROOM_SECONDS <= ao.JOB_LIMIT_SECONDS + + +def test_a_budget_whose_closing_can_t_fit_the_job_is_refused(): + with pytest.raises(ValueError, match="time_budget_seconds=27000 can't fit"): + _step(time_budget_seconds=7.5 * 3600) + with pytest.raises(ValueError, match="time_budget_seconds=0 can't fit"): + _step(time_budget_seconds=0) + assert _step(time_budget_seconds=6 * 3600).effective_time_budget() == 6 * 3600 + + +def test_a_wake_whose_worst_case_leaves_no_room_for_a_run_is_refused(): + with pytest.raises(ValueError, match="no run fits the hub's 8 h job"): + _step(timeout_seconds=4 * 3600) + + +def test_the_budget_survives_a_round_trip(): + again = ao.AlwaysOnTaskStep.from_dict(_step(time_budget_seconds=5 * 3600).to_dict()) + assert again.time_budget_seconds == 5 * 3600 + assert ao.AlwaysOnTaskStep.from_dict(_step().to_dict()).time_budget_seconds is None + + +# ---- the reserve is a real bound ------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("settings", [{}, {"nudge": False}, {"timeout_seconds": 1800, "user_timeout_seconds": 900}, + {"contact_timeout_seconds": 900, "settle_timeout_seconds": 600}, + {"time_budget_seconds": 5 * 3600}]) +def test_the_reserve_adds_up_every_capped_part_and_the_budget_plus_one_turn_fits_the_job(settings): + step = _step(**settings) + settle = (step.contact_timeout_seconds or ao.CONTACT_TIMEOUT_SECONDS) + ao.CONTACT_GRACE_SECONDS \ + + step.settle_timeout_seconds + turn = step.timeout_seconds + ao.TURN_GRACE_SECONDS + ao.TRAJECTORY_FETCH_SECONDS + read = ao.READ_ASKS * (step.user_timeout_seconds + ao.TURN_GRACE_SECONDS + ao.TRAJECTORY_FETCH_SECONDS) + wake = turn * (2 if step.nudge else 1) + settle + ao.GATEWAY_ALLOWANCE_SECONDS + assert step.settle_cap_seconds() == settle and step.read_cap_seconds() == read + # a jump's settle, then a wake (its turns, its settle, the step's calls), then a read and its calls + assert step.wake_reserve_seconds() == settle + wake + read + ao.GATEWAY_ALLOWANCE_SECONDS + assert step.effective_time_budget() + step.wake_reserve_seconds() + ao.JOB_HEADROOM_SECONDS <= ao.JOB_LIMIT_SECONDS + + +def _estimate(rig, n): + rig.alert(f"Live Oak: estimate {n}", channel="Gmail · Live Oak Arbor") + + +@pytest.mark.asyncio +async def test_a_settle_stops_at_its_cap_though_a_contact_s_own_limit_is_longer(monkeypatch, tmp_path): + """The contact's own limit is 5 s, past contact_timeout_seconds (0.3 s): the settle used to wait for it.""" + monkeypatch.setattr(ao, "CONTACT_GRACE_SECONDS", 0.1) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + slow = rig.contact("liveoak", seconds=5, reply=_estimate, timeout_seconds=5, keeps_to_its_limit=False) + rig.triggers({"id": "estimate", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, + "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send your estimate"}]}, + executors={"liveoak": slow}) + step = rig.step(contact_timeout_seconds=0.3, settle_timeout_seconds=0.1) + try: + ctx = await rig.run(step) + finally: + await rig.close() + + assert step.settle_cap_seconds() == pytest.approx(0.5) + [jump] = [j for j in rig.events(ctx, "clock_jump") if j["to"] == "2026-11-09T15:00:00Z"] + assert not jump["settled"] and jump["settle"]["seconds"] < 1.5 + + +@pytest.mark.asyncio +async def test_an_event_due_that_won_t_go_gets_the_settle_timeout_not_forever(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + engine = rig.gw._trigger_engine + real_state = engine.state + monkeypatch.setattr(engine, "state", lambda: {**real_state(), "next_event_at": rig.now()}) # due, never going + ctx = await asyncio.wait_for(rig.run(rig.step(settle_timeout_seconds=0.2)), 10) + + [timeout] = rig.events(ctx, "settle_timeout") + [env] = timeout["still_at_work"] + assert env["due_not_going"] == "2026-11-09T14:00:00Z" and timeout["settle"]["seconds"] < 1.5 + + +@pytest.mark.asyncio +async def test_an_agent_turn_is_cut_off_at_its_limit_and_grace_whatever_send_and_wait_does(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "TURN_GRACE_SECONDS", 0.2) + rig = Rig(monkeypatch, tmp_path) + + async def retries_forever(url, parts, **kw): + await asyncio.sleep(60) + + monkeypatch.setattr(ao, "send_and_wait", retries_forever) + await rig.arm() + ctx = await asyncio.wait_for(rig.run(rig.step(timeout_seconds=1)), 10) + + record = rig.record(ctx) + assert (record["end_reason"], record["error_type"]) == ("agent_turn_timeout", "agent_turn_timeout") + assert rig.events(ctx, "agent_turn_end")[0]["real_seconds"] < 2 + + +@pytest.mark.asyncio +async def test_a_read_is_cut_off_at_its_cap_as_a_user_sim_error(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "TURN_GRACE_SECONDS", 0.1) + monkeypatch.setattr(ao, "TRAJECTORY_FETCH_SECONDS", 0.1) + monkeypatch.setattr(ao, "READ_ASKS", 1) + + class Stalls: + @staticmethod + def setup(version, person): + return {"system_prompt": "you are Linda", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + await asyncio.sleep(60) + + def agent(n, texts, rig): + rig.message("blue or green?") + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, plugin=Stalls) + await rig.arm() + step = rig.step(user_timeout_seconds=1) + ctx = await asyncio.wait_for(rig.run(step), 10) + + assert step.read_cap_seconds() == pytest.approx(1.2) + [read] = rig.events(ctx, "user_sim_read") + assert read["user_sim_error"] == "the read ran past its cap (1s)" and read["real_seconds"] < 2 + assert rig.record(ctx)["end_reason"] == "user_sim_error" + + +@pytest.mark.asyncio +async def test_a_turn_of_the_loop_past_the_reserve_ends_the_run_on_a_step_error(monkeypatch, tmp_path): + """Every part is capped on its own; only calls hanging past their allowance reach this last cut-off.""" + monkeypatch.setattr(ao.AlwaysOnTaskStep, "wake_reserve_seconds", lambda step: 0.5) + rig = Rig(monkeypatch, tmp_path) + + real_take = ao._Run._take_messages + hung: list[int] = [] + + async def hangs_once(run): + if run.wakes and not hung: # the opening wake's take, not the one before the run + hung.append(1) + await asyncio.sleep(60) + return await real_take(run) + + monkeypatch.setattr(ao._Run, "_take_messages", hangs_once) + await rig.arm() + ctx = rig.context() + with pytest.raises(RuntimeError, match="the opening wake ran past its worst case"): + await asyncio.wait_for(rig.run(rig.step(), ctx), 10) + assert ctx.metadata[ao.ALWAYS_ON_KEY]["week"]["end_reason"] == "step_error" diff --git a/tst/unit/task_step/test_always_on_checks.py b/tst/unit/task_step/test_always_on_checks.py new file mode 100644 index 000000000..a7c46dde9 --- /dev/null +++ b/tst/unit/task_step/test_always_on_checks.py @@ -0,0 +1,125 @@ +"""The always_on step lets the agent wake itself: scheduled checks are on by default when a run spans more than one +day of the task's time, at least PT3H apart, and the agent is told it can schedule them.""" + +from __future__ import annotations + +import pytest + +from agent_env.env.gateway.notifications import NotificationError +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +CHECK_TOOLS = {"schedule_check", "list_scheduled_checks", "cancel_scheduled_check"} + + +@pytest.mark.asyncio +async def test_a_multi_day_run_gives_the_agent_checks_through_a_deny_all_and_says_so(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + async with rig.http() as client: + await client.post("/tools/disable", json={"role": "default", "tools": ["*"]}) + ctx = await rig.run(rig.step(end_time="P5D")) + + during = rig.during[0] + assert during["check_roles"] == ("default",) and during["min_interval"] == "PT3H" + assert CHECK_TOOLS <= during["tools"] + assert ao.CHECKS_RULE in rig.sent[0][0] + assert rig.gw._notifications.roles == () and not CHECK_TOOLS & await rig.tools() # off again after the run + assert rig.record(ctx)["settings"]["scheduled_checks"] == { + "on": True, "envs": [{"env_id": "world", "set_by": "step", "min_interval": "PT3H", "max_active": 10}]} + + +@pytest.mark.asyncio +async def test_checks_closer_than_the_minimum_gap_are_refused(monkeypatch, tmp_path): + refused: list[str] = [] + + def agent(n, texts, rig): + if n == 1: + try: + rig.gw._notifications.schedule("sync stock", every="PT1H", to="default") + except NotificationError as e: + refused.append(str(e)) + rig.gw._notifications.schedule("sync stock", every="PT3H", to="default") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + await rig.run(rig.step(end_time="P1D")) + + [why] = refused + assert "at least PT3H" in why + + +@pytest.mark.asyncio +async def test_a_check_the_agent_schedules_moves_the_clock_and_wakes_it(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + rig.schedule("book La Ola, it opens at 9", at="2026-11-12T14:00:00Z") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P5D")) + + wake = rig.events(ctx, "wake")[1] + assert wake["virtual_time"] == "2026-11-12T14:00:00Z" and wake["woke_by"] == ["notification:your scheduled check"] + assert "[Thu Nov 12, 2:00 PM UTC · your scheduled check] book La Ola, it opens at 9" in rig.sent[1] + + +@pytest.mark.parametrize("end_time,setting,on", [(None, None, False), ("PT6H", None, False), ("P5D", False, False), + (None, True, True)], + ids=["one sitting", "same day", "turned off", "turned on"]) +@pytest.mark.asyncio +async def test_otherwise_a_task_decides(monkeypatch, tmp_path, end_time, setting, on): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + settings = {**({"end_time": end_time} if end_time else {}), + **({"scheduled_checks": setting} if setting is not None else {})} + ctx = await rig.run(rig.step(**settings)) + + assert (rig.during[0]["check_roles"] == ("default",)) is on + assert rig.record(ctx)["settings"]["scheduled_checks"]["on"] is on + assert (ao.CHECKS_RULE in rig.sent[0][0]) is on + + +@pytest.mark.asyncio +async def test_the_line_goes_in_the_system_prompt_when_checks_are_on(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + run = ao._Run(rig.step(system_prompt="You are Linda's assistant.", end_time="P5D"), rig.context()) + run.checks = {"on": True} + assert run._system_prompt().endswith(ao.CHECKS_RULE) + + +@pytest.mark.asyncio +async def test_the_task_s_own_check_settings_are_kept(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"], "max_active": 3, "min_interval": "PT1H"}) + ctx = await rig.run(rig.step(end_time="P5D", scheduled_checks=False)) + + assert rig.gw._notifications.min_interval == "PT1H" and rig.gw._notifications.max_active == 3 + assert rig.record(ctx)["settings"]["scheduled_checks"]["envs"][0]["set_by"] == "task" + + +@pytest.mark.asyncio +async def test_other_roles_the_task_gave_checks_keep_them(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.gw._notifications.configure({"roles": ["planner"], "min_interval": "PT2H"}) + await rig.run(rig.step(end_time="P5D")) + + assert rig.during[0]["check_roles"] == ("planner", "default") and rig.during[0]["min_interval"] == "PT2H" + assert rig.gw._notifications.roles == ("planner",) and rig.gw._notifications.min_interval == "PT2H" + + +def test_the_settings_survive_a_round_trip(): + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", prompt_id="p", agent_name="a", + scheduled_checks=False, scheduled_check_min_interval="PT6H") + again = ao.AlwaysOnTaskStep.from_dict(step.to_dict()) + assert (again.scheduled_checks, again.scheduled_check_min_interval) == (False, "PT6H") + assert ao.AlwaysOnTaskStep.from_dict({**step.to_dict(), "scheduled_checks": None}).scheduled_checks is None diff --git a/tst/unit/task_step/test_always_on_clock_rate.py b/tst/unit/task_step/test_always_on_clock_rate.py new file mode 100644 index 000000000..62bb08127 --- /dev/null +++ b/tst/unit/task_step/test_always_on_clock_rate.py @@ -0,0 +1,120 @@ +"""The task's clock runs at its rate only while the agent works (FD-5366): between turns the step freezes it, so a slow +contact reply or user sim read doesn't move the task's time and alerts arrive when they happen; at the end the task's +rate is back. Every other always_on test runs a frozen clock (rate 0); these run one at 60x.""" + +from __future__ import annotations + +import asyncio +import time + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.always_on import _utc +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +RATE = 60.0 # a real second is a virtual minute + + +def _estimate(rig, n): + rig.alert("Live Oak: estimate attached, $2,400", channel="Gmail · Live Oak Arbor") + + +@pytest.mark.asyncio +async def test_a_slow_contact_reply_between_turns_doesn_t_move_the_clock(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, rate=RATE) + await rig.arm() + liveoak = rig.contact("liveoak", seconds=1.5, reply=_estimate, timeout_seconds=10) # 90 virtual s at 60x + rig.triggers({"id": "estimate", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, + "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send Maya your estimate"}]}, + executors={"liveoak": liveoak}) + ctx = await rig.run(rig.step(settle_timeout_seconds=0.2)) + + [_, alerted] = rig.events(ctx, "wake") + assert "[Mon Nov 9, 3:00 PM UTC · Gmail · Live Oak Arbor]" in alerted["parts_text"] + assert alerted["virtual_time"].startswith("2026-11-09T15:00:00") + assert rig.record(ctx)["settings"]["clock"] == {"envs": [{"env_id": "world", "rate": RATE, + "frozen_between_turns": True}]} + + +@pytest.mark.asyncio +async def test_the_clock_runs_at_the_task_s_rate_during_a_turn_and_the_rate_is_back_at_the_end(monkeypatch, tmp_path): + rates: list[float] = [] + moved: list[float] = [] + + def agent(n, texts, rig): + rates.append(rig.gw._clock.rate) + before = rig.gw._clock.elapsed() + time.sleep(0.5) # the agent at work: half a real second, half a virtual minute + moved.append(rig.gw._clock.elapsed() - before) + rig.message("found two houses") if n == 1 else rig.quiet() + return "" + + class SlowReader: + @staticmethod + def setup(version, person): + return {"system_prompt": "you are Linda", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + rates.append(rig.gw._clock.rate) + await asyncio.sleep(0.5) # she takes her time reading + return {"outcome": "reply", "message": "the blue one"} + + rig = Rig(monkeypatch, tmp_path, rate=RATE, agent=agent, plugin=SlowReader) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert rates == [RATE, 0.0, RATE] # turn, read, turn + assert all(25 <= m <= 40 for m in moved) + [read] = rig.events(ctx, "user_sim_read") + assert read["deliver_at"] == read["virtual_time"] # nothing moved while she read + assert rig.gw._clock.rate == RATE + + +def _crew_texted(rig, n): + rig.alert(f"Sam: reply {n}", channel="Messages · noor 30") + + +CREW_DM = {"id": "crew-dm", "when": {"type": "action", "tool": "messaging_send_message", "repeat": True}, + "actions": [{"type": "nl", "executor": "crew", "instruction": "answer Linda's newest text", + "delay": {"dist": "exp", "mean": "PT40M", "seed": 30}}]} + + +@pytest.mark.asyncio +async def test_a_contact_s_delayed_reply_goes_on_when_the_step_jumps_the_frozen_clock_to_it(monkeypatch, tmp_path): + """The reply's delay starts during the agent's turn, on the running clock; between turns the step freezes the + clock and jumps it to the reply's time. That must be enough: the reply used to wait out settle_timeout_seconds, + then go only at the next jump, hours of the task's time late.""" + def texts_a_friend(n, texts, rig): + if n == 1: + rig.provoke("messaging_send_message", {"user_id": "+13475550167"}) + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, rate=RATE, agent=texts_a_friend) + turn = rig._send_and_wait + + async def the_turn_goes_on_after_the_text(url, parts, **kw): + out = await turn(url, parts, **kw) + await asyncio.sleep(0.05) # the reply's delay starts while the clock still runs, as in a real turn + return out + + monkeypatch.setattr(ao, "send_and_wait", the_turn_goes_on_after_the_text) + await rig.arm() + rig.app_tools("messaging_send_message") + rig.triggers(CREW_DM, executors={"crew": rig.contact("crew", seconds=0, reply=_crew_texted)}) + try: + ctx = await rig.run(rig.step(settle_timeout_seconds=1)) + finally: + await rig.close() + + assert rig.events(ctx, "settle_timeout") == [] + [jump] = [j for j in rig.events(ctx, "clock_jump") if j["to"].startswith("2026-11-09T14:3")] + assert jump["settled"] + [call] = rig.contact_calls + assert _utc(call["at"]) == _utc(jump["to"]) # the reply went at the time the step jumped to + [_, replied] = rig.events(ctx, "wake") + assert "Sam: reply 1" in replied["parts_text"] and _utc(replied["virtual_time"]) == _utc(jump["to"]) diff --git a/tst/unit/task_step/test_always_on_endings.py b/tst/unit/task_step/test_always_on_endings.py new file mode 100644 index 000000000..229ec56e5 --- /dev/null +++ b/tst/unit/task_step/test_always_on_endings.py @@ -0,0 +1,309 @@ +"""How an always_on run ends: a final check-in at end_time whether or not the task sets a `final` line, a last-turn +notice at end_time and before the wake cap, the user sim's last read of the agent's wrap-up, what was still pending, +and end reasons that tell a finished run from a stuck one.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} +NOTICE = ao.FINAL_REMINDER.format(first="Linda") +DEFAULT_FINAL = ao.FINAL_DEFAULT_TEXT.format(first="Linda") + + +def _agent(**plays): + def agent(n, texts, rig): + play = plays.get(f"t{n}") + if play is None: + rig.quiet() + return "" + return play(rig) + return agent + + +def _message(text): + return lambda rig: (rig.message(text), "sent")[1] + + +def _recaps(n, texts, rig): + """Messages Linda on a wake that carries the last-turn notice, and stays quiet otherwise.""" + if NOTICE in texts: + rig.message("RESULT: house booked, flights held; La Ola still open") + return "sent" + rig.quiet() + return "" + + +# ---- 1. a final check-in at end_time, with or without a `final` line ------------------------------------------------- + +@pytest.mark.asyncio +async def test_without_a_final_line_the_end_time_still_brings_a_check_in_with_the_step_s_own_line(monkeypatch, + tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps, answers=[{"outcome": "reply", "message": "perfect, thanks"}]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P2D")) + + last = rig.parts[-1] + assert [p["text"] for p in last if p["metadata"]["agentenv"]["source"] == "final"] == [DEFAULT_FINAL] + assert rig.events(ctx, "wake")[-1]["virtual_time"] == "2026-11-11T14:00:00Z" + assert rig.events(ctx, "wake")[-1]["woke_by"] == ["step:final"] + assert DEFAULT_FINAL not in repr([r["you_sent"] for r in rig.reads]) # the step's line, not hers + assert rig.record(ctx)["end_reason"] == "end_time" + + +@pytest.mark.asyncio +async def test_a_turn_that_runs_past_end_time_is_followed_by_the_final_check_in_not_more_wakes(monkeypatch, tmp_path): + """The clock runs during the agent's turns, so one can end past end_time: what is due then (her reply, an alert) + comes with the final check-in, rather than in ordinary wakes that use up max_wakes without one.""" + def chatty(n, texts, rig): + rig.message(f"update {n}: anything else?") + if n == 1: + rig.alert("Live Oak: we can come Friday", channel="Gmail · Live Oak Arbor") + rig.gw._clock.advance("2026-11-10T15:00:00Z") # a long turn: the task's clock runs past end_time + return "sent" + + rig = Rig(monkeypatch, tmp_path, agent=chatty, + answers=[{"outcome": "reply", "message": f"reply {i}"} for i in range(1, 6)]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P1D", max_wakes=4)) + + assert len(rig.sent) == 2 # the opening wake, then the final check-in + final = " ".join(rig.sent[1]) + assert DEFAULT_FINAL in final and NOTICE in final + assert "Linda: reply 1" in final and "Live Oak: we can come Friday" in final + assert rig.events(ctx, "wake")[-1]["virtual_time"] == "2026-11-10T15:00:00Z" + assert rig.record(ctx)["end_reason"] == "end_time" + + +@pytest.mark.asyncio +async def test_a_task_s_final_line_is_hers_and_comes_with_the_notice(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps) + await rig.arm() + await rig.run(rig.step(end_time="P2D", final="send me the rundown")) + + texts = rig.sent[-1] + assert "[Wed Nov 11, 2:00 PM UTC] Linda: send me the rundown" in texts and DEFAULT_FINAL not in texts + assert texts[-1] == NOTICE and ao.WAKE_REMINDER_TEXT not in texts + assert [s["text"] for s in rig.reads[-1]["you_sent"]] == ["sort the cdmx weekend", "send me the rundown"] + + +# ---- 2. the last-turn notice --------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_wake_the_cap_makes_the_last_carries_the_notice_and_no_earlier_one_does(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps) + await rig.arm() + for day in range(10, 16): + rig.triggers(rig.notify_at(f"alert-{day}", f"2026-11-{day}T14:00:00Z", "Sam: any news?", channel="noor 30")) + ctx = await rig.run(rig.step(max_wakes=4)) + + assert [NOTICE in texts for texts in rig.sent] == [False, False, False, True] + assert [w["last"] for w in rig.events(ctx, "wake")] == [False, False, False, True] + assert rig.record(ctx)["end_reason"] == "max_wakes" + assert [m["text"] for m in rig.reads[-1]["assistant_messages"]] == [ + "RESULT: house booked, flights held; La Ola still open"] + + +# ---- 3. final -> the agent's turn -> her last read -> her reaction recorded -> end ---------------------------------- + +@pytest.mark.asyncio +async def test_the_user_sim_reads_the_wrap_up_and_her_reaction_and_held_reply_are_recorded(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps, + answers=[{"outcome": "reply", "message": "amazing, book La Ola too"}]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P2D", final="send me the rundown"), rig.context(user=DAY)) + + order = [e["event"] for e in rig.timeline(ctx)] + tail = order[order.index("scripted_user_line"):] + assert tail[:2] == ["scripted_user_line", "wake"] and tail[-2:] == ["user_sim_read", "end"] + assert tail.index("message_to_user") < tail.index("user_sim_read") + [read] = rig.events(ctx, "user_sim_read") + assert read["last_read"] is True and read["outcome"] == "reply" + assert read["virtual_time"] == "2026-11-11T14:00:00Z" and read["deliver_at"] == "2026-11-11T16:30:00Z" + held = rig.record(ctx)["pending_at_end"]["held_replies"] + assert held == {"count": 1, "items": [{"at": "2026-11-11T16:30:00Z", "text": "amazing, book La Ola too", + "done": False}]} + assert rig.record(ctx)["end_reason"] == "end_time" and len(rig.sent) == 2 + + +@pytest.mark.asyncio +async def test_the_last_read_comes_after_every_message_of_a_last_turn_that_ran_on(monkeypatch, tmp_path): + """Her last read is no earlier than the task's time when the run ended, so it never shows a message before it was + sent, nor holds a reply that comes before the question it answers.""" + def wraps_up_slowly(n, texts, rig): + rig.message("booked the house") + rig.gw._clock.advance("2026-11-09T16:00:00Z") # the turn goes on two hours, the clock running + rig.message("and the flights: is Friday ok?") + return "sent" + + rig = Rig(monkeypatch, tmp_path, agent=wraps_up_slowly, + answers=[{"outcome": "reply", "message": "friday is fine"}]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1)) + + [read] = rig.reads + assert [m["at"] for m in read["assistant_messages"]] == ["2026-11-09T14:00:00Z", "2026-11-09T16:00:00Z"] + assert read["time"] == "2026-11-09T16:00:00Z" + [event] = rig.events(ctx, "user_sim_read") + assert event["last_read"] is True and event["deliver_at"] == "2026-11-09T16:00:00Z" + + +@pytest.mark.asyncio +async def test_a_last_read_that_would_fall_in_her_night_waits_for_the_morning(monkeypatch, tmp_path): + def wraps_up_late(n, texts, rig): + rig.message("booked the house") + rig.gw._clock.advance("2026-11-10T05:00:00Z") # midnight in New York: past her bedtime + rig.message("and the flights: is Friday ok?") + return "sent" + + rig = Rig(monkeypatch, tmp_path, agent=wraps_up_late, answers=[{"outcome": "reply", "message": "friday is fine"}]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1), rig.context(user=DAY)) + + [read] = rig.reads + assert read["time"] == "2026-11-10T12:30:00Z" # 7:30 AM, when she wakes + [event] = rig.events(ctx, "user_sim_read") + assert (event["mode"], event["deliver_at"]) == ("morning", "2026-11-10T12:45:00Z") + + +@pytest.mark.asyncio +async def test_a_done_on_the_last_read_is_recorded_and_changes_nothing(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_recaps, answers=[{"outcome": "done", "message": "thanks!!"}]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1)) + + [read] = rig.events(ctx, "user_sim_read") + assert read["last_read"] is True and read["outcome"] == "done" and "ends_run" not in read + assert rig.record(ctx)["end_reason"] == "max_wakes" + assert rig.record(ctx)["pending_at_end"]["held_replies"]["items"][0]["done"] is True + + +@pytest.mark.asyncio +async def test_a_reply_still_held_at_end_time_is_kept_not_dropped(monkeypatch, tmp_path): + """FD-5332 test 8: Linda's last reply was lost at end_time, and the record said nothing was held.""" + rig = Rig(monkeypatch, tmp_path, agent=_agent(t2=_message("flight retimed, Dev is fine")), + answers=[{"outcome": "reply", "message": "ok good, tell Sam"}]) + await rig.arm() + rig.triggers(rig.notify_at("airline", "2026-11-10T13:50:00Z", "Wingloft: flight retimed", channel="Gmail")) + ctx = await rig.run(rig.step(end_time="P1D"), rig.context(user=DAY)) + + [read] = rig.events(ctx, "user_sim_read") + assert read["deliver_at"] == "2026-11-10T14:20:00Z" # after end_time + assert [i["text"] for i in rig.record(ctx)["pending_at_end"]["held_replies"]["items"]] == ["ok good, tell Sam"] + assert rig.events(ctx, "end")[0]["held_at_end"] == 1 + + +# ---- 5. what was still pending --------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_agent_s_checks_not_yet_due_are_recorded_and_events_after_the_end_never_happened(monkeypatch, + tmp_path): + def sets_a_check(n, texts, rig): + if n == 1: + rig.schedule("chase Live Oak", at="2026-11-20T14:00:00Z") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=sets_a_check) + await rig.arm() + rig.triggers(rig.notify_at("late", "2026-11-12T14:00:00Z", "Live Oak: invoice", channel="Gmail"), + rig.notify_at("ask", "2026-11-12T15:00:00Z", "did they invoice?", to=ao.USER_SAYS)) + ctx = await rig.run(rig.step(end_time="P2D")) + + pending = rig.record(ctx)["pending_at_end"] + assert pending["scheduled_checks_not_due"]["items"][0]["note"] == "chase Live Oak" + end = rig.events(ctx, "end")[0]["pending"] + assert end["alerts_never_delivered"] == end["user_lines_not_delivered"] == [] # never came due: not triggered + + +@pytest.mark.asyncio +async def test_an_alert_that_arrives_during_the_last_turn_is_recorded_as_never_delivered(monkeypatch, tmp_path): + def last_turn_brings_mail(n, texts, rig): + if NOTICE in texts: + rig.alert("Live Oak: we can come Friday", channel="Gmail · Live Oak Arbor") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=last_turn_brings_mail) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1)) + + [alert] = rig.record(ctx)["pending_at_end"]["alerts_never_delivered"]["items"] + assert (alert["channel"], alert["text"]) == ("Gmail · Live Oak Arbor", "Live Oak: we can come Friday") + + +# ---- 7. finished vs stuck -------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_with_nothing_left_to_wake_the_agent_and_nothing_pending_the_run_finished(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("report sent")), + answers=[{"outcome": "no_reply", "message": ""}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + record = rig.record(ctx) + assert record["end_reason"] == "finished_nothing_pending" + assert all(v["count"] == 0 for v in record["pending_at_end"].values()) + + +@pytest.mark.asyncio +async def test_with_nothing_left_to_wake_the_agent_but_a_message_she_will_never_read_the_run_is_stuck(monkeypatch, + tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue or green?"), t2=_message("and the dates?")), + answers=[{"outcome": "reply", "message": "blue"}]) + await rig.arm() + ctx = await rig.run(rig.step(), rig.context(user={"max_replies": 1})) + + record = rig.record(ctx) + assert record["end_reason"] == "stuck" + assert [i["text"] for i in record["pending_at_end"]["unread_by_user"]["items"]] == ["and the dates?"] + assert record["pending_at_end"]["unread_by_user"]["items"][0]["wake"] == 2 + assert record["end_note"] == ao.END_NOTES["stuck"] + + +@pytest.mark.asyncio +async def test_a_contact_still_replying_when_nothing_else_can_wake_the_agent_makes_the_run_stuck(monkeypatch, + tmp_path): + monkeypatch.setattr(ao, "CONTACT_GRACE_SECONDS", 0) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + slow = rig.contact("liveoak", seconds=5, reply=lambda rig, n: rig.alert("Live Oak: estimate"), + timeout_seconds=0.2, keeps_to_its_limit=False) + rig.triggers({"id": "liveoak-estimate", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, + "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send your estimate"}]}, + executors={"liveoak": slow}) + try: + ctx = await rig.run(rig.step(settle_timeout_seconds=0.2)) + finally: + await rig.close() + + record = rig.record(ctx) + assert record["end_reason"] == "stuck" + [env] = record["pending_at_end"]["contact_replies_in_progress"]["items"] + assert env["triggers"][0]["id"] == "liveoak-estimate" + + +@pytest.mark.asyncio +async def test_messages_a_turn_sent_before_it_timed_out_are_in_the_graded_response_and_pending(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message("found it: the blue house, $855") + raise TimeoutError("the agent's turn ran out") + + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + ctx = await rig.run(rig.step()) + + [response] = ctx.prompt_responses + assert response.response == "[Mon Nov 9, 2:00 PM UTC] found it: the blue house, $855" + assert response.error_type == "agent_turn_timeout" + [sent] = rig.events(ctx, "message_to_user") + assert sent["leftover"] is True + [unread] = rig.record(ctx)["pending_at_end"]["unread_by_user"]["items"] + assert unread["text"] == "found it: the blue house, $855" and unread["at"] == "2026-11-09T14:00:00Z" + assert "found it: the blue house" in " ".join(t for m in rig.chat(ctx) for t in m["texts"]) diff --git a/tst/unit/task_step/test_always_on_graded.py b/tst/unit/task_step/test_always_on_graded.py new file mode 100644 index 000000000..2a0a3a004 --- /dev/null +++ b/tst/unit/task_step/test_always_on_graded.py @@ -0,0 +1,257 @@ +"""What graders read of an always_on run: the graded response (the user's whole chat from the agent, with times), the +conversation (what the user saw: her messages and the agent's message_user texts, her last word, an end marker), and +error_type on a run whose agent turn failed or timed out, so verifiers skip it.""" + +from __future__ import annotations + +import pytest + +from agent_env.a2a_agent import conversation_store +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import ( + AgentPromptResponseVerifierTaskStep, +) +from agent_env.task_step.task_steps.verifiers.run_context import load_run_context +from tst.unit.task_step.always_on_harness import ROLE, Fail, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _agent(**plays): + def agent(n, texts, rig): + play = plays.get(f"t{n}") + if play is None: + rig.quiet() + return "PRIVATE nothing to report" + return play(rig) + return agent + + +def _message(*texts, closing="PRIVATE closing note"): + def play(rig): + for text in texts: + rig.message(text) + return closing + return play + + +async def _turns(ctx): + run = await load_run_context(ctx, ctx.prompt_responses[0]) + return [(t.message.text, t.reply.text if t.reply else None) for t in run.turns] + + +# ---- the graded response ------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_graded_response_is_every_message_user_text_in_order_with_its_time(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue house or green?", "flights: two options"), + t2=_message("booked the blue one")), + answers=[{"outcome": "reply", "message": "blue"}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step()) + + [response] = ctx.prompt_responses + assert response.response == ("[Mon Nov 9, 2:00 PM UTC] blue house or green?\n\n" + "[Mon Nov 9, 2:00 PM UTC] flights: two options\n\n" + "[Mon Nov 9, 2:00 PM UTC] booked the blue one") + assert "PRIVATE" not in response.response and response.error_type is None + verifier = AgentPromptResponseVerifierTaskStep("check", None, prompt_id="ask", verifier_id="words", criteria=[ + {"type": "response_contains", "needles": ["blue house or green?", "booked the blue one"]}]) + await verifier.execute(ctx) + assert ctx.metadata["verifications"]["words"]["score"] == 1 + + +# ---- the conversation graders read --------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_conversation_is_what_she_saw_her_messages_and_the_agent_s_message_user_texts(monkeypatch, + tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue house or green?"), t2=_message("booked it"), + t4=_message("Sam says she suspects nothing")), + answers=[{"outcome": "reply", "message": "blue"}, {"outcome": "no_reply", "message": ""}, + {"outcome": "reply", "message": "great"}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30"), + rig.notify_at("ask", "2026-11-10T15:00:00Z", "any news from Sam?", to=ao.USER_SAYS)) + ctx = await rig.run(rig.step()) + + turns = await _turns(ctx) + assert turns[:3] == [ + ("[Mon Nov 9, 2:00 PM UTC] Linda: sort the cdmx weekend", "[Mon Nov 9, 2:00 PM UTC] blue house or green?"), + ("[Mon Nov 9, 2:00 PM UTC] Linda: blue", "[Mon Nov 9, 2:00 PM UTC] booked it"), + ("[Tue Nov 10, 3:00 PM UTC] Linda: any news from Sam?", + "[Tue Nov 10, 3:00 PM UTC] Sam says she suspects nothing"), + ] + seen = repr(rig.chat(ctx)) + for never_shown in ("PRIVATE", "Sam: shh", "noor 30", ao.WAKE_REMINDER_TEXT, ao.MESSAGE_RULES, + "Any other date you see"): + assert never_shown not in seen, never_shown + + +@pytest.mark.asyncio +async def test_messages_sent_on_several_wakes_before_she_answers_are_one_reply_in_order(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("on it"), t2=_message("Sam is in")), + answers=[{"outcome": "no_reply", "message": ""}, {"outcome": "no_reply", "message": ""}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: I'm in", channel="noor 30")) + ctx = await rig.run(rig.step()) + + [first, *_] = await _turns(ctx) + assert first[1] == "[Mon Nov 9, 2:00 PM UTC] on it\n[Tue Nov 10, 2:00 PM UTC] Sam is in" + assert [p["metadata"]["agentenv"]["wake"] for p in rig.chat(ctx)[1]["parts"]] == [1, 2] + + +# ---- her last word and the end marker (FD-5369 4 and 6) ------------------------------------------------------------ + +@pytest.mark.asyncio +async def test_her_done_reaches_the_conversation(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set"), t2=_message("glad to help!")), + answers=[{"outcome": "done", "message": "thanks!"}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + turns = await _turns(ctx) + assert turns[1] == ("[Mon Nov 9, 2:00 PM UTC] Linda: thanks!", "[Mon Nov 9, 2:00 PM UTC] glad to help!") + assert rig.chat(ctx)[2]["parts"][0]["metadata"]["agentenv"]["done"] is True + assert turns[-1][0].startswith("[End of run · Mon Nov 9, 2:00 PM UTC] Linda signed off.") + + +@pytest.mark.asyncio +async def test_a_blank_done_is_shown_as_her_reaction(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set")), + answers=[{"outcome": "done", "message": ""}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + [last] = rig.chat(ctx)[-1:] + assert last["role"] == "user" + assert last["texts"][0] == "(Linda read it and signed off without a message.)" + + +@pytest.mark.asyncio +async def test_her_reply_to_the_wrap_up_and_the_end_marker_close_the_conversation(monkeypatch, tmp_path): + def recap(n, texts, rig): + if ao.FINAL_REMINDER.format(first="Linda") in texts: + rig.message("RESULT: all booked") + rig.quiet() + return "PRIVATE" + + rig = Rig(monkeypatch, tmp_path, agent=recap, answers=[{"outcome": "reply", "message": "you're the best"}]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P2D", final="send me the rundown")) + + turns = await _turns(ctx) + assert turns[-2] == ("[Wed Nov 11, 2:00 PM UTC] Linda: send me the rundown", + "[Wed Nov 11, 2:00 PM UTC] RESULT: all booked") + last = rig.chat(ctx)[-1] + assert last["texts"] == [ + "[Wed Nov 11, 2:00 PM UTC] Linda: you're the best (sent after the assistant's last turn; it never saw this)", + "[End of run · Wed Nov 11, 2:00 PM UTC] The run reached its end time. (A note from the run, not a message " + "from Linda.)"] + assert last["parts"][-1]["metadata"]["agentenv"] == {"source": "end_marker", "end_reason": "end_time", + "at": "2026-11-11T14:00:00Z"} + + +@pytest.mark.asyncio +async def test_a_run_cut_off_by_its_wake_cap_says_so_in_the_conversation(monkeypatch, tmp_path): + def chatty(n, texts, rig): + rig.message(f"update {n}") + return "" + + rig = Rig(monkeypatch, tmp_path, agent=chatty, answers=[{"outcome": "reply", "message": "ok"}] * 5) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=2)) + + last = rig.chat(ctx)[-1] + assert last["texts"][-1].startswith("[End of run · Mon Nov 9, 2:00 PM UTC] The run was cut off at its cap of 2 " + "wakes.") + assert "Linda: ok (sent after the assistant's last turn" in last["texts"][0] + + +@pytest.mark.asyncio +async def test_messages_queued_before_the_run_are_not_the_run_s(monkeypatch, tmp_path): + """Another role's messages stay queued for whoever takes them; what the agent's own role left before the run is + recorded as such; neither reaches her or the graded chat.""" + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("on it"))) + await rig.arm() + chat = rig.gw._user_chat + chat.configure({"roles": ["other", ROLE], "stay_quiet": True}) + chat.send("from another assistant", role="other") + chat.send("left from an earlier step", role=ROLE) + chat.quiet(role=ROLE) + ctx = await rig.run(rig.step()) + + [response] = ctx.prompt_responses + assert response.response == "[Mon Nov 9, 2:00 PM UTC] on it" + assert [m["text"] for m in rig.reads[0]["assistant_messages"]] == ["on it"] + assert [(m["from"], m["text"]) for m in chat.state()["pending"]] == [("other", "from another assistant")] + [before] = rig.events(ctx, "user_chat_before_run") + assert [m["text"] for m in before["messages"]] == ["left from an earlier step"] + assert before["stay_quiet_calls"] == 1 + + +@pytest.mark.asyncio +async def test_a_take_refused_before_the_run_still_closes_the_conversation_and_records_the_run(monkeypatch, tmp_path): + """The take before the run comes after the graded conversation is made: when the gateway refuses it, the run still + leaves its record and prompt response, and the conversation is closed, not left active.""" + rig = Rig(monkeypatch, tmp_path) + real_take = ao._Run._take_messages + refused: list[int] = [] + + async def refused_once(run): + if not refused: # the take before the run, not finish()'s + refused.append(1) + raise RuntimeError("the gateway refused the take") + return await real_take(run) + + monkeypatch.setattr(ao._Run, "_take_messages", refused_once) + await rig.arm() + ctx = rig.context() + with pytest.raises(RuntimeError, match="the gateway refused the take"): + await rig.run(rig.step(), ctx) + + assert rig.sent == [] + assert rig.record(ctx)["end_reason"] == "step_error" + [response] = ctx.prompt_responses + assert response.error_type == "step_error" + assert conversation_store.get_conversation(ctx.metadata["a2a_conversations"]["week"])["status"] == "closed" + + +# ---- failed turns are errors ----------------------------------------------------------------------------------------- + +@pytest.mark.parametrize("play,error_type,end_reason", [ + (lambda rig: Fail("the CLI crashed"), "agent_turn_failed", "agent_turn_failed"), + (lambda rig: Fail("sandbox gone", error_type="infra_error"), "infra_error", "agent_turn_failed"), + (lambda rig: (_ for _ in ()).throw(TimeoutError("no answer")), "agent_turn_timeout", "agent_turn_timeout"), + (lambda rig: Fail("stopped", state="canceled"), "agent_turn_canceled", "agent_turn_failed"), + (lambda rig: Fail("won't do that", state="rejected"), "agent_turn_rejected", "agent_turn_failed"), +], ids=["failed", "failed with the agent's error type", "timed out", "canceled", "rejected"]) +@pytest.mark.asyncio +async def test_a_failed_or_timed_out_turn_sets_error_type_and_the_verifier_skips_the_run(monkeypatch, tmp_path, play, + error_type, end_reason): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("on it"), t2=play), + answers=[{"outcome": "reply", "message": "great"}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + [response] = ctx.prompt_responses + assert (response.error_type, rig.record(ctx)["end_reason"]) == (error_type, end_reason) + assert rig.record(ctx)["error_type"] == error_type + verifier = AgentPromptResponseVerifierTaskStep("check", None, prompt_id="ask", verifier_id="words", + criteria=[{"type": "response_contains", "needles": ["on it"]}]) + await verifier.execute(ctx) + assert ctx.metadata["verifications"]["words"]["results"][0]["id"] == "prompt_error" + + +@pytest.mark.parametrize("state", ["failed", "canceled", "rejected"]) +@pytest.mark.asyncio +async def test_a_nudge_turn_that_doesn_t_complete_sets_error_type_too(monkeypatch, tmp_path, state): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=lambda rig: "PRIVATE all done", t2=lambda rig: Fail(state=state))) + await rig.arm() + ctx = await rig.run(rig.step()) + + [nudge] = rig.events(ctx, "nudge") + assert nudge["state"] == state and len(rig.sent) == 2 + [response] = ctx.prompt_responses + assert (response.error_type, rig.record(ctx)["end_reason"]) == (f"agent_turn_{state}", "agent_turn_failed") diff --git a/tst/unit/task_step/test_always_on_idle_world.py b/tst/unit/task_step/test_always_on_idle_world.py new file mode 100644 index 000000000..7b5510af6 --- /dev/null +++ b/tst/unit/task_step/test_always_on_idle_world.py @@ -0,0 +1,90 @@ +"""A world that only repeats: a recurring trigger that never wakes the agent (an hourly alert to the user's phone, say) +must not move the clock forever. With no end_time the run ends once that is all the world has ahead; a known event ahead +keeps it going; with an end_time it goes on to it, and the timeline stays small.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +NEWS = {"id": "news", "when": {"type": "time", "every": "PT1H"}, + "actions": [{"type": "notify", "channel": "News", "text": "headline", "to": ao.USER_PHONE}]} + + +async def _run(rig, step, ctx=None, seconds=30): + return await asyncio.wait_for(rig.run(step, ctx), seconds) # the old loop never ended + + +@pytest.mark.asyncio +async def test_with_no_end_time_a_world_that_only_repeats_ends_the_run(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(NEWS) + ctx = await _run(rig, rig.step()) + + record = rig.record(ctx) + assert record["end_reason"] == "finished_nothing_pending" + assert record["counts"]["clock_jumps"] == ao.IDLE_JUMPS_MAX and record["counts"]["wakes"] == 1 + [idle] = rig.events(ctx, "world_only_repeats") + assert (idle["jumps"], idle["since"], idle["triggers"]) == (ao.IDLE_JUMPS_MAX, "2026-11-09T14:00:00Z", ["news"]) + assert rig.now() == "2026-11-10T14:00:00Z" # a day of hourly ticks, not a month + + +@pytest.mark.asyncio +async def test_an_event_still_to_come_keeps_the_run_going_past_the_ticks(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(NEWS, rig.notify_at("sam", "2026-11-12T14:00:00Z", "Sam: flights?", channel="noor 30")) + ctx = await _run(rig, rig.step()) + + woke = [(w["virtual_time"], w["woke_by"]) for w in rig.events(ctx, "wake")] + assert woke == [("2026-11-09T14:00:00Z", ["user_request"]), ("2026-11-12T14:00:00Z", ["notification:noor 30"])] + assert rig.record(ctx)["end_reason"] == "finished_nothing_pending" + assert rig.events(ctx, "world_only_repeats")[0]["since"] == "2026-11-12T14:00:00Z" + + +@pytest.mark.asyncio +async def test_a_known_reply_ahead_isn_t_cut_short_by_the_ticks(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "IDLE_JUMPS_MAX", 3) + + def agent(n, texts, rig): + if n == 1: + rig.message("blue house or green?") + else: + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "blue"}]) + await rig.arm() + rig.triggers(NEWS) + user = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 600, "free": 600, "morning": 600}} # her reply reaches it ten hours later + ctx = await _run(rig, rig.step(), ctx=rig.context(user=user)) + + woke = [(w["virtual_time"], w["woke_by"]) for w in rig.events(ctx, "wake")] + assert woke == [("2026-11-09T14:00:00Z", ["user_request"]), ("2026-11-10T00:00:00Z", ["user_reply"])] + assert rig.record(ctx)["end_reason"] == "finished_nothing_pending" + assert rig.now() == "2026-11-10T03:00:00Z" # then three ticks with nothing ahead + + +@pytest.mark.asyncio +async def test_with_an_end_time_the_run_goes_on_to_it_and_folds_the_jumps(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(NEWS) + ctx = await _run(rig, rig.step(end_time="P3D")) + + record = rig.record(ctx) + assert record["end_reason"] == "end_time" and rig.now() == "2026-11-12T14:00:00Z" + assert record["counts"]["clock_jumps"] == 72 + jumps = rig.events(ctx, "clock_jump") + [folded] = rig.events(ctx, "clock_jumps_folded") + assert len(jumps) == ao.JUMPS_KEPT_PER_GAP and folded["count"] == 72 - ao.JUMPS_KEPT_PER_GAP + assert folded["to"] == "2026-11-12T14:00:00Z" and sum(folded["reasons"].values()) == folded["count"] + assert rig.events(ctx, "world_only_repeats") == [] diff --git a/tst/unit/task_step/test_always_on_loop.py b/tst/unit/task_step/test_always_on_loop.py new file mode 100644 index 000000000..fc26a03ac --- /dev/null +++ b/tst/unit/task_step/test_always_on_loop.py @@ -0,0 +1,340 @@ +"""The always_on step's loop on an in-process gateway: what wakes the agent, how the clock moves, when the user sim reads +and replies, what each read holds, and how done ends a run.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.user_sim import UserSimTaskStep +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +LINDA_DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} + + +def _agent(**plays): + """An agent that plays ``plays["t"]`` on its nth turn and otherwise stays quiet.""" + def agent(n, texts, rig): + play = plays.get(f"t{n}") + if play is None: + rig.quiet() + return "nothing new" + return play(rig) + return agent + + +def _message(text): + return lambda rig: (rig.message(text), "sent")[1] + + +def _woke(ctx, rig): + return [(w["virtual_time"], w["woke_by"]) for w in rig.events(ctx, "wake")] + + +def _without_step_lines(texts): + return [t for t in texts if t not in (ao.WAKE_REMINDER_TEXT, f"{ao.MESSAGE_RULES} {ao.QUIET_RULE}")] + + +# ---- what wakes the agent ------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_a_notify_alert_wakes_the_agent_labelled_with_its_time_and_channel(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: she suspects nothing", channel="noor 30")) + ctx = await rig.run(rig.step()) + + assert _woke(ctx, rig) == [("2026-11-09T14:00:00Z", ["user_request"]), + ("2026-11-10T14:00:00Z", ["notification:noor 30"])] + assert "[Tue Nov 10, 2:00 PM UTC · noor 30] Sam: she suspects nothing" in rig.sent[1] + assert rig.record(ctx)["end_reason"] == "finished_nothing_pending" + + +@pytest.mark.asyncio +async def test_a_scripted_user_line_reaches_the_agent_as_the_user_and_counts_as_hers(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t2=_message("on it: three dinner spots tonight"))) + await rig.arm() + rig.triggers(rig.notify_at("ask-dinner", "2026-11-10T15:00:00Z", "also find a dinner spot", to=ao.USER_SAYS)) + ctx = await rig.run(rig.step()) + + assert _woke(ctx, rig)[1] == ("2026-11-10T15:00:00Z", ["user_says"]) + assert "[Tue Nov 10, 3:00 PM UTC] Linda: also find a dinner spot" in rig.sent[1] + [read] = rig.reads + assert [s["text"] for s in read["you_sent"]] == ["sort the cdmx weekend", "also find a dinner spot"] + assert [e["text"] for e in rig.events(ctx, "scripted_user_line")] == ["also find a dinner spot"] + + +@pytest.mark.asyncio +async def test_the_agent_s_scheduled_check_moves_the_clock_and_wakes_it(monkeypatch, tmp_path): + def book(rig): + rig.schedule("book La Ola when it opens", at="2026-11-12T13:00:00Z") + rig.quiet() + return "check set" + + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=book)) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + ctx = await rig.run(rig.step()) + + assert _woke(ctx, rig)[1] == ("2026-11-12T13:00:00Z", ["notification:your scheduled check"]) + assert "[Thu Nov 12, 1:00 PM UTC · your scheduled check] book La Ola when it opens" in rig.sent[1] + assert [j["to"] for j in rig.events(ctx, "clock_jump")] == ["2026-11-12T13:00:00Z"] + + +@pytest.mark.asyncio +async def test_a_reply_is_held_for_the_reply_time_then_wakes_the_agent(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue house or green?")), + answers=[{"outcome": "reply", "message": "the blue one"}]) + await rig.arm() + ctx = await rig.run(rig.step(), rig.context(user=LINDA_DAY)) + + [read] = rig.events(ctx, "user_sim_read") + assert read["virtual_time"] == "2026-11-09T14:00:00Z" and read["mode"] == "work" + assert read["deliver_at"] == "2026-11-09T16:30:00Z" + assert _woke(ctx, rig)[1] == ("2026-11-09T16:30:00Z", ["user_reply"]) + assert "[Mon Nov 9, 4:30 PM UTC] Linda: the blue one" in rig.sent[1] + + +@pytest.mark.asyncio +async def test_the_user_reads_once_awake_and_not_busy(monkeypatch, tmp_path): + busy = {**LINDA_DAY["schedule"], "busy": [{"start": "2026-11-10T07:00", "end": "2026-11-10T10:00"}]} + rig = Rig(monkeypatch, tmp_path, t0="2026-11-10T05:00:00Z", # midnight in New York + agent=_agent(t1=_message("found two houses"), t2=_message("and a third")), + answers=[{"outcome": "reply", "message": "send all three"}]) + await rig.arm() + ctx = await rig.run(rig.step(), rig.context(user={**LINDA_DAY, "schedule": busy})) + + [scheduled, *_] = rig.events(ctx, "user_read_scheduled") + assert scheduled["read_at"] == "2026-11-10T15:00:00Z" # past the night and the 7-10 AM block + [first, second] = rig.events(ctx, "user_sim_read") + assert first["virtual_time"] == "2026-11-10T15:00:00Z" + assert first["deliver_at"] == "2026-11-10T15:15:00Z" + + +@pytest.mark.asyncio +async def test_messages_sent_before_a_read_arrive_in_that_one_read(monkeypatch, tmp_path): + def two(rig): + rig.message("house: the blue one, $855") + rig.message("dinner: La Ola, Saturday 8pm") + return "sent both" + + rig = Rig(monkeypatch, tmp_path, t0="2026-11-10T05:00:00Z", agent=_agent(t1=two)) + await rig.arm() + await rig.run(rig.step(), rig.context(user=LINDA_DAY)) + + [read] = rig.reads + assert read["time"] == "2026-11-10T12:30:00Z" + assert [m["text"] for m in read["assistant_messages"]] == ["house: the blue one, $855", + "dinner: La Ola, Saturday 8pm"] + + +# ---- the clock ------------------------------------------------------------------------------------------------------ + +@pytest.mark.asyncio +async def test_the_clock_jumps_to_each_next_event_in_turn_never_past_one(monkeypatch, tmp_path): + def check(rig): + rig.schedule("chase the host", at="2026-11-10T11:00:00Z") + rig.quiet() + return "ok" + + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=check)) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + rig.triggers(rig.notify_at("landlord", "2026-11-10T09:30:00Z", "rent", to="landlord"), + rig.notify_at("noon", "2026-11-10T12:00:00Z", "Sam: flights?", channel="noor 30"), + rig.notify_at("ten", "2026-11-10T10:00:00Z", "Pri: I'm in", channel="noor 30")) + ctx = await rig.run(rig.step()) + + assert [j["to"] for j in rig.events(ctx, "clock_jump")] == [ + "2026-11-10T09:30:00Z", "2026-11-10T10:00:00Z", "2026-11-10T11:00:00Z", "2026-11-10T12:00:00Z"] + assert [t for t, _ in _woke(ctx, rig)[1:]] == ["2026-11-10T10:00:00Z", "2026-11-10T11:00:00Z", + "2026-11-10T12:00:00Z"] + + +@pytest.mark.asyncio +async def test_what_arrives_in_the_same_minute_comes_in_one_wake(monkeypatch, tmp_path): + def check(rig): + rig.schedule("look at the airline change", at="2026-11-10T10:00:40Z") + rig.quiet() + return "ok" + + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=check)) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + rig.triggers(rig.notify_at("airline", "2026-11-10T10:00:00Z", "Wingloft: your flight was retimed", + channel="Gmail"), + rig.notify_at("ask", "2026-11-10T10:00:20Z", "did the flight change?", to=ao.USER_SAYS), + rig.notify_at("next", "2026-11-10T10:01:00Z", "Dev: thx!", channel="Messages")) + ctx = await rig.run(rig.step()) + + woke = _woke(ctx, rig) + assert woke[1] == ("2026-11-10T10:00:40Z", ["notification:Gmail", "notification:your scheduled check", + "user_says"]) + assert woke[2] == ("2026-11-10T10:01:00Z", ["notification:Messages"]) + assert len(woke) == 3 + + +# ---- what the user sim sees, and done ---------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_each_read_holds_the_time_the_agent_s_messages_her_phone_and_only_her_own_lines(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + return "Linda should know the house is booked." + if any("Sam: shh" in t for t in texts): + rig.message("house booked; Sam says she suspects nothing") + return "sent" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + rig.schedule("chase the host", at="2026-11-09T18:00:00Z") + rig.triggers(rig.notify_at("phone", "2026-11-09T19:00:00Z", "New email from Sam", channel="Gmail", + to=ao.USER_PHONE), + rig.notify_at("ask", "2026-11-09T19:30:00Z", "is the house sorted?", to=ao.USER_SAYS), + rig.notify_at("alert", "2026-11-09T20:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step(end_time="2026-11-10T02:00:00Z", final="send me the rundown")) + + [read] = rig.reads + assert read["time"] == "2026-11-09T20:00:00Z" + assert [m["text"] for m in read["assistant_messages"]] == ["house booked; Sam says she suspects nothing"] + assert [(p["channel"], p["text"]) for p in read["phone"]] == [("Gmail", "New email from Sam")] + assert [s["text"] for s in read["you_sent"]] == ["sort the cdmx weekend", "is the house sorted?"] + seen = repr(rig.reads) + for meant_for_the_agent in ("Sam: shh", "chase the host", ao.NUDGE_TEXT, ao.WAKE_REMINDER_TEXT, + ao.MESSAGE_RULES): + assert meant_for_the_agent not in seen + assert rig.events(ctx, "nudge") and rig.record(ctx)["end_reason"] == "end_time" + + +@pytest.mark.asyncio +async def test_done_ends_a_one_sitting_run_after_the_agent_s_turn_on_her_sign_off(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set: shelves restocked"), + t2=_message("glad to help!")), + answers=[{"outcome": "done", "message": "thanks!"}]) + await rig.arm() + rig.triggers(rig.notify_at("later", "2026-11-10T14:00:00Z", "a new order", channel="Craftweb")) + ctx = await rig.run(rig.step()) + + assert rig.record(ctx)["end_reason"] == "user_done" and len(rig.sent) == 2 + assert "[Mon Nov 9, 2:00 PM UTC] Linda: thanks!" in rig.sent[1] + assert len(rig.reads) == 1 and rig.events(ctx, "user_sim_read")[0]["ends_run"] is True + + +@pytest.mark.asyncio +async def test_a_blank_done_ends_a_one_sitting_run_at_once(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("all set")), + answers=[{"outcome": "done", "message": " "}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert rig.record(ctx)["end_reason"] == "user_done" and len(rig.sent) == 1 + + +@pytest.mark.asyncio +async def test_done_in_a_multi_day_run_is_recorded_and_the_run_goes_on_to_end_time(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house booked"), t2=_message("airline moved Dev")), + answers=[{"outcome": "done", "message": "perfect ty"}, {"outcome": "reply", "message": "ok!"}]) + await rig.arm() + rig.triggers(rig.notify_at("airline", "2026-11-11T14:00:00Z", "Wingloft: flight retimed", channel="Gmail")) + ctx = await rig.run(rig.step(end_time="P3D")) + + record = rig.record(ctx) + assert record["multi_day"] is True and record["end_reason"] == "end_time" + assert [r["outcome"] for r in rig.events(ctx, "user_sim_read")] == ["done", "reply"] + assert rig.events(ctx, "user_sim_read")[0]["ends_run"] is False + assert [m["text"] for m in rig.reads[1]["assistant_messages"]] == ["airline moved Dev"] + assert record["counts"]["user_dones"] == 1 + assert any("Linda: perfect ty" in t for texts in rig.sent for t in texts) + + +@pytest.mark.parametrize("end_time,ends", [(None, True), ("P3D", False)], ids=["one sitting", "multi-day"]) +@pytest.mark.asyncio +async def test_each_read_says_whether_done_would_end_the_run(monkeypatch, tmp_path, end_time, ends): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house booked"))) + await rig.arm() + await rig.run(rig.step(**({"end_time": end_time} if end_time else {}))) + + [read] = rig.reads + assert read["done_ends_run"] is ends + + +@pytest.mark.parametrize("end_time,multi_day", [(None, False), ("PT6H", False), ("P1D", True)]) +@pytest.mark.asyncio +async def test_a_run_is_multi_day_when_its_end_falls_on_a_later_day_for_the_user(monkeypatch, tmp_path, end_time, + multi_day): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = await rig.run(rig.step(**({"end_time": end_time} if end_time else {}))) + assert rig.record(ctx)["multi_day"] is multi_day + + +# ---- the time zone -------------------------------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_the_world_s_time_zone_is_the_user_s_when_the_user_sim_sets_none(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = rig.context(user={"time_zone": None}) + ctx.metadata["clock_configurations"] = [{"step_id": "clock", "timezone": "America/Los_Angeles"}] + ctx = await rig.run(rig.step(), ctx) + + assert rig.record(ctx)["time_zone"] == "America/Los_Angeles" + assert rig.setups[0]["time_zone"] == "America/Los_Angeles" + + +@pytest.mark.asyncio +async def test_a_user_and_a_world_in_different_time_zones_are_refused(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = rig.context() + ctx.metadata["clock_configurations"] = [{"step_id": "clock", "timezone": "America/Los_Angeles"}] + with pytest.raises(ValueError, match="America/New_York but sync_env_clock set"): + await rig.run(rig.step(), ctx) + + +# ---- the steps ------------------------------------------------------------------------------------------------------ + +def test_both_steps_are_registered(): + registry = get_task_step_registry() + assert registry["always_on"] is ao.AlwaysOnTaskStep and registry["user_sim"] is UserSimTaskStep + + +# ---- the user's schedule -------------------------------------------------------------------------------------------- + +def _available(monkeypatch, tmp_path, schedule: dict, at: str): + rig = Rig(monkeypatch, tmp_path) + run = ao._Run(rig.step(), rig.context(user={"schedule": schedule})) + run.us = {"schedule": schedule} + run.tz = ao.ZoneInfo("America/New_York") + got, mode = run._available(ao._utc(at)) + return ao._iso(got), mode + + +@pytest.mark.parametrize("at,read_at,mode", [ + ("2026-11-10T14:00:00Z", "2026-11-10T14:00:00Z", "work"), # Tue 9:00 AM: awake + ("2026-11-11T04:00:00Z", "2026-11-11T04:00:00Z", "free"), # Tue 11:00 PM: still up + ("2026-11-11T05:15:00Z", "2026-11-11T05:15:00Z", "free"), # Wed 12:15 AM: still up + ("2026-11-11T05:45:00Z", "2026-11-11T13:00:00Z", "morning"), # Wed 12:45 AM: asleep until 8:00 + ("2026-11-11T10:00:00Z", "2026-11-11T13:00:00Z", "morning"), # Wed 5:00 AM: asleep until 8:00 +]) +def test_a_night_that_starts_after_midnight_keeps_the_evening_awake(monkeypatch, tmp_path, at, read_at, mode): + night_owl = {"wake": "08:00", "sleep": "00:30", "work": ["09:00", "18:00"]} + assert _available(monkeypatch, tmp_path, night_owl, at) == (read_at, mode) + + +def test_back_to_back_busy_blocks_and_nights_never_leave_a_read_inside_one(monkeypatch, tmp_path): + blocks = [("2026-11-10T08:00", "2026-11-10T09:00"), ("2026-11-10T09:00", "2026-11-10T10:00"), + ("2026-11-10T10:00", "2026-11-10T23:45"), # into the night: she reads next morning + ("2026-11-11T07:30", "2026-11-11T08:30"), ("2026-11-11T08:30", "2026-11-11T09:30"), + ("2026-11-11T09:30", "2026-11-11T10:00")] + schedule = {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"], + "busy": [{"start": a, "end": b} for a, b in reversed(blocks)]} # latest first: one block a pass + # Tue 8:00 AM in New York: six blocks and a night in a row, more than the old four passes + assert _available(monkeypatch, tmp_path, schedule, "2026-11-10T13:00:00Z") == ("2026-11-11T15:00:00Z", "morning") diff --git a/tst/unit/task_step/test_always_on_messages.py b/tst/unit/task_step/test_always_on_messages.py new file mode 100644 index 000000000..7f30b0e8a --- /dev/null +++ b/tst/unit/task_step/test_always_on_messages.py @@ -0,0 +1,157 @@ +"""The always_on step's message rules: the agent reaches the user only through message_user, says "nothing to tell" +with stay_quiet, gets one nudge when a wake ends with closing text and neither, and every wake ends with a reminder. +All on unless a task turns them off; none of it reaches the user sim.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import ROLE, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _turns(*plays): + """An agent that plays ``plays[n-1]`` on its nth turn, then says "ok".""" + def agent(n, texts, rig): + return plays[n - 1](rig) if n <= len(plays) else "ok" + return agent + + +def _send(text, closing="Sent."): + return lambda rig: (rig.message(text), closing)[1] + + +def _quiet(closing="Nothing new."): + return lambda rig: (rig.quiet(), closing)[1] + + +def _says(closing): + return lambda rig: closing + + +@pytest.mark.asyncio +async def test_stay_quiet_and_the_nudge_are_on_and_the_wake_reminder_off_by_default(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_quiet())) + await rig.arm() + step = rig.step() + ctx = await rig.run(step) + + assert (step.stay_quiet, step.nudge, step.wake_reminder) == (True, True, False) + assert rig.record(ctx)["settings"].items() >= {"stay_quiet": True, "nudge": True, "wake_reminder": False}.items() + assert not any(ao.WAKE_REMINDER_TEXT in texts for texts in rig.sent) + assert rig.during[0]["stay_quiet"] and ROLE in rig.during[0]["user_chat_roles"] + + +@pytest.mark.asyncio +async def test_a_task_that_turns_the_wake_reminder_on_ends_each_wake_with_it(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_send("two houses, which one?"), _quiet()), + answers=[{"outcome": "reply", "message": "the blue one"}]) + await rig.arm() + await rig.run(rig.step(wake_reminder=True)) + + assert len(rig.sent) == 2 + assert all(texts[-1] == ao.WAKE_REMINDER_TEXT for texts in rig.sent) + assert rig.parts[0][-1]["metadata"] == {"agentenv": {"source": "reminder"}} + + +@pytest.mark.asyncio +async def test_closing_text_with_neither_tool_gets_one_nudge_recorded_and_counted(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_says("Found the house; Linda should pick."), _send("pick one?"))) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert rig.sent[1] == [ao.NUDGE_TEXT] + [nudge] = rig.events(ctx, "nudge") + assert nudge["wake"] == 1 and nudge["closing_text"] == "Found the house; Linda should pick." + assert nudge["outcome"] == "message_user" and nudge["plain_reply_after"] == "Sent." + counts = rig.record(ctx)["counts"] + assert counts["nudges"] == 1 and counts["nudge_outcomes"] == {"message_user": 1} + assert [m["text"] for m in rig.events(ctx, "message_to_user")] == ["pick one?"] + + +@pytest.mark.asyncio +async def test_a_nudge_the_agent_ignores_is_recorded_and_not_repeated(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_says("All done for tonight."), _says("Okay."))) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert len(rig.sent) == 2 + [nudge] = rig.events(ctx, "nudge") + assert nudge["outcome"] == "neither" and nudge["plain_reply_after"] == "Okay." + + +@pytest.mark.parametrize("play", [_send("booked the house"), _quiet()], ids=["message_user", "stay_quiet"]) +@pytest.mark.asyncio +async def test_no_nudge_after_either_tool(monkeypatch, tmp_path, play): + rig = Rig(monkeypatch, tmp_path, agent=_turns(play)) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert ao.NUDGE_TEXT not in [t for texts in rig.sent for t in texts] + assert rig.events(ctx, "nudge") == [] + + +@pytest.mark.parametrize("closing", ["", " ", "(The assistant's reply was empty.)"], + ids=["none", "blank", "harness placeholder"]) +@pytest.mark.asyncio +async def test_no_nudge_on_an_empty_reply(monkeypatch, tmp_path, closing): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_says(closing))) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert len(rig.sent) == 1 and rig.events(ctx, "nudge") == [] + + +@pytest.mark.asyncio +async def test_a_task_can_turn_each_off(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_says("Updated all quantities."))) + await rig.arm() + ctx = await rig.run(rig.step(stay_quiet=False, nudge=False, wake_reminder=False)) + + assert len(rig.sent) == 1 and ao.WAKE_REMINDER_TEXT not in rig.sent[0] + assert rig.events(ctx, "nudge") == [] and "nudges" not in rig.record(ctx)["counts"] + assert not rig.gw._user_chat.stay_quiet and "stay_quiet" not in await rig.tools() + + +def test_off_survives_a_round_trip(): + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", prompt_id="p", agent_name="a", + stay_quiet=False, nudge=False, wake_reminder=False) + again = ao.AlwaysOnTaskStep.from_dict(step.to_dict()) + assert (again.stay_quiet, again.nudge, again.wake_reminder) == (False, False, False) + assert ao.AlwaysOnTaskStep.from_dict({**step.to_dict(), "stay_quiet": None}).stay_quiet is True + + +@pytest.mark.asyncio +async def test_the_rules_follow_the_task_system_prompt(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_quiet())) + await rig.arm() + run = ao._Run(rig.step(system_prompt="You are Linda's assistant."), rig.context()) + assert run._system_prompt() == f"You are Linda's assistant.\n\n{ao.MESSAGE_RULES} {ao.QUIET_RULE}" + run = ao._Run(rig.step(system_prompt="You are Linda's assistant.", stay_quiet=False), rig.context()) + assert run._system_prompt() == f"You are Linda's assistant.\n\n{ao.MESSAGE_RULES}" + + +@pytest.mark.asyncio +async def test_without_a_system_prompt_the_first_wake_opens_with_the_rules(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_turns(_quiet())) + await rig.arm() + await rig.run(rig.step()) + + assert rig.sent[0][0] == f"{ao.MESSAGE_RULES} {ao.QUIET_RULE}" + + +@pytest.mark.asyncio +async def test_neither_the_nudge_nor_the_reminder_reaches_the_user_sim(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, + agent=_turns(_says("Linda should hear about the house."), _send("blue or green house?"), _quiet()), + answers=[{"outcome": "reply", "message": "green"}]) + await rig.arm() + await rig.run(rig.step()) + + [read] = rig.reads + assert [m["text"] for m in read["assistant_messages"]] == ["blue or green house?"] + assert [s["text"] for s in read["you_sent"]] == ["sort the cdmx weekend"] + seen = repr(rig.reads) + assert ao.NUDGE_TEXT not in seen and ao.WAKE_REMINDER_TEXT not in seen and ao.MESSAGE_RULES not in seen diff --git a/tst/unit/task_step/test_always_on_no_clock.py b/tst/unit/task_step/test_always_on_no_clock.py new file mode 100644 index 000000000..757fb7c69 --- /dev/null +++ b/tst/unit/task_step/test_always_on_no_clock.py @@ -0,0 +1,81 @@ +"""Without the task's clock: read times and reply delays are task time, so a user sim with a schedule or response times +is refused at the start instead of the loop spinning in real time; a run with neither still works. And a message the +chat env couldn't time (its clock unarmed) is read on the task's clock when another env's is armed.""" + +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone + +import pytest + +from agent_env.env.gateway.constants import EXT_CLOCK_URI +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} + + +def _asks_once(n, texts, rig): + if n == 1: + rig.message("blue house or green?") + else: + rig.quiet() + return "" + + +@pytest.mark.parametrize("user", [DAY, {"schedule": DAY["schedule"]}, {"response_times": DAY["response_times"]}], + ids=["both", "schedule", "response_times"]) +@pytest.mark.asyncio +async def test_a_schedule_or_response_times_with_no_task_clock_is_refused_at_the_start(monkeypatch, tmp_path, user): + rig = Rig(monkeypatch, tmp_path, agent=_asks_once, answers=[{"outcome": "reply", "message": "blue"}]) + ctx = rig.context(user=user) + with pytest.raises(ValueError, match="need the task's clock, and no env's clock is armed"): + await asyncio.wait_for(rig.run(rig.step(), ctx), 5) # the old loop spun until real time caught up + assert rig.sent == [] and rig.gw._user_chat.roles == () # refused before the gateway was touched + + +@pytest.mark.asyncio +async def test_with_no_clock_and_no_schedule_a_run_reads_and_replies_at_once(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_asks_once, answers=[{"outcome": "reply", "message": "blue"}]) + ctx = await asyncio.wait_for(rig.run(rig.step()), 10) + + assert [w["woke_by"] for w in rig.events(ctx, "wake")] == [["user_request"], ["user_reply"]] + assert rig.record(ctx)["end_reason"] == "finished_nothing_pending" + + +class _ClockEnv: + """Another env whose clock is armed (the chat env's isn't): only clock/v1, frozen at ``at``.""" + + env_id, env_version, metadata = "calendar", 1, {"deploy_step_id": "deploy-calendar"} + + def __init__(self, at: datetime) -> None: + self.at = at + + def supports(self, uri, method=None) -> bool: + return uri == EXT_CLOCK_URI and method in (None, "state", "advance") + + async def invoke(self, uri, method, params=None, **kw): + if method == "advance": + self.at = max(self.at, ao._utc(params["virtual_time"])) + return {"armed": True, "t0": "2027-03-01T15:00:00Z", "virtual_time": ao._iso(self.at), + "virtual_seconds_per_real_second": 0} + + +@pytest.mark.asyncio +async def test_a_message_the_chat_env_couldn_t_time_is_read_on_the_task_s_clock(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_asks_once, answers=[{"outcome": "reply", "message": "blue"}]) + ctx = rig.context(user=DAY) + task_time = datetime(2027, 3, 1, 15, 0, tzinfo=timezone.utc) # Mon 10 AM in New York: she reads at once + assert abs(datetime.now(timezone.utc) - task_time) > timedelta(days=30) + ctx.deployed_envs.append(_ClockEnv(task_time)) + ctx = await asyncio.wait_for(rig.run(rig.step(), ctx), 10) + + [read] = rig.reads + assert read["assistant_messages"][0]["at"] is None # the chat env's clock is unarmed + assert read["time"] == "2027-03-01T15:00:00Z" + [scheduled] = rig.events(ctx, "user_read_scheduled") + assert scheduled["read_at"] == "2027-03-01T15:00:00Z" diff --git a/tst/unit/task_step/test_always_on_prompt_parts.py b/tst/unit/task_step/test_always_on_prompt_parts.py new file mode 100644 index 000000000..138c66110 --- /dev/null +++ b/tst/unit/task_step/test_always_on_prompt_parts.py @@ -0,0 +1,37 @@ +"""A task prompt's file (and other non-text) parts reach the agent on the opening wake, as prompt_agent sends them, and +are in the user's opening turn of the graded conversation; its text is the user's labelled line.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +ITINERARY = {"kind": "file", "file": {"uri": "s3://trips//itinerary.pdf", "name": "itinerary.pdf", + "mimeType": "application/pdf"}} + + +@pytest.mark.asyncio +async def test_the_prompt_s_file_reaches_the_agent_on_the_opening_wake_in_its_place(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt_id="ask", agent_name="solver", + parts=[{"kind": "text", "text": "sort the cdmx weekend"}, ITINERARY, + {"kind": "text", "text": "flights are in the pdf"}]) + ctx = rig.context() + ctx.metadata["seed"] = {"trip": "cdmx"} + ctx = await rig.run(step, ctx) + + opening = rig.parts[0] + files = [p for p in opening if p.get("kind") == "file"] + assert files == [{"kind": "file", "file": {**ITINERARY["file"], "uri": "s3://trips/cdmx/itinerary.pdf"}}] + texts = [p["text"] for p in opening if p.get("kind") == "text"] + line = "[Mon Nov 9, 2:00 PM UTC] Linda: sort the cdmx weekend\nflights are in the pdf" + assert texts.count(line) == 1 + assert opening.index(files[0]) == next(i for i, p in enumerate(opening) if p.get("text") == line) + 1 + [user_turn, *_] = [m for m in rig.chat(ctx) if m["role"] == "user"] + assert [p.get("kind") for p in user_turn["parts"]] == ["text", "file"] + assert user_turn["parts"][1]["file"]["uri"] == "s3://trips/cdmx/itinerary.pdf" diff --git a/tst/unit/task_step/test_always_on_record.py b/tst/unit/task_step/test_always_on_record.py new file mode 100644 index 000000000..69713e174 --- /dev/null +++ b/tst/unit/task_step/test_always_on_record.py @@ -0,0 +1,167 @@ +"""What the always_on step keeps in the run's metadata (a short summary, small enough for a running job to report back) +and what it stores in full (the timeline, in the object store).""" + +from __future__ import annotations + +import json + +import pytest + +from agent_env.config import get_config +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _chatty(n, texts, rig): + rig.message(f"update {n}: " + "x" * 2000) + return "sent" + + +@pytest.mark.asyncio +async def test_the_run_record_is_a_summary_and_the_full_timeline_is_stored(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_chatty, + answers=[{"outcome": "reply", "message": f"ok {i}"} for i in range(30)]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=20)) + + record = rig.record(ctx) + assert "timeline" not in record + assert len(json.dumps(record, default=str)) < 10_000 + stored = rig.stored(ctx) + assert stored["end_reason"] == record["end_reason"] == "max_wakes" + assert len(stored["timeline"]) == record["timeline_events"] > 60 + assert sum(1 for e in stored["timeline"] if e["event"] == "wake") == 20 + + +@pytest.mark.asyncio +async def test_when_the_timeline_cannot_be_stored_the_record_keeps_its_last_events(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_chatty, + answers=[{"outcome": "reply", "message": f"ok {i}"} for i in range(30)]) + await rig.arm() + store = get_config().get_object_store() + + def fail(*a, **kw): + raise OSError("bucket gone") + + monkeypatch.setattr(type(store), "put", fail) + ctx = await rig.run(rig.step(max_wakes=20)) + + record = rig.record(ctx) + assert "object_url" not in record and record["timeline_store_error"] == "OSError: bucket gone" + assert len(record["timeline_tail"]) == ao.TIMELINE_TAIL and record["timeline_tail"][-1]["event"] == "end" + assert not any(set(e) & set(ao.BULKY_EVENT_FIELDS) for e in record["timeline_tail"]) + + +@pytest.mark.asyncio +async def test_the_process_log_carries_no_one_s_words_only_the_timeline_does(monkeypatch, tmp_path, caplog): + def agent(n, texts, rig): + if n == 1: + rig.message("SECRET-SENT house at 12 Oak St, $855") + return "SECRET-PLAIN I'll book it" + rig.quiet() + return "SECRET-CLOSING bye" + + rig = Rig(monkeypatch, tmp_path, agent=agent, + answers=[{"outcome": "reply", "message": "SECRET-REPLY yes, card ending 4242", "used": "SECRET-USED"}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-09T15:00:00Z", "SECRET-ALERT Sam's address", channel="noor 30")) + caplog.set_level("INFO", logger=ao.logger.name) + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="SECRET-PROMPT sort the cdmx weekend", + prompt_id="ask", agent_name="solver") + ctx = await rig.run(step) + + logged = "\n".join(r.getMessage() for r in caplog.records if r.name == ao.logger.name) + assert "always_on wake" in logged and "always_on user_sim_read" in logged + assert "SECRET" not in logged + stored = json.dumps(rig.timeline(ctx)) + for marker in ("SECRET-SENT", "SECRET-PLAIN", "SECRET-REPLY", "SECRET-ALERT", "SECRET-PROMPT"): + assert marker in stored + + +def _fails_after_fifty_long_messages(n, texts, rig): + for i in range(50): + rig.message(f"option {i}: " + "x" * 2000) + raise TimeoutError("the agent's turn ran out") + + +@pytest.mark.asyncio +async def test_when_storing_fails_the_kept_events_are_small_at_any_depth(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_fails_after_fifty_long_messages) + await rig.arm() + store = get_config().get_object_store() + + def fail(*a, **kw): + raise OSError("bucket gone") + + monkeypatch.setattr(type(store), "put", fail) + ctx = await rig.run(rig.step()) + + record = rig.record(ctx) + assert record["pending_at_end"]["unread_by_user"]["count"] == 50 + end = record["timeline_tail"][-1] + assert end["event"] == "end" and end["pending"]["unread_by_user"][-1] == "(and 30 more)" + + def strings(value): + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for v in value.values(): + yield from strings(v) + elif isinstance(value, list): + for v in value: + yield from strings(v) + + assert max(len(s) for s in strings(record)) <= ao.PENDING_TEXT_CHARS + 1 + assert len(json.dumps(record, default=str)) < 60_000 + + +@pytest.mark.asyncio +async def test_the_prompt_response_keeps_each_wake_s_parts_cut_to_size(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "TURN_TEXT_TOTAL_CHARS", 9000) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + for day in (10, 11, 12): + rig.triggers(rig.notify_at(f"news-{day}", f"2026-11-{day}T14:00:00Z", "y" * 10_000, channel="Gmail")) + ctx = await rig.run(rig.step(wake_reminder=True)) # its parts add up past the total with the reminder + + kept = ctx.prompt_responses[0].source_agent_per_turn_prompt_parts + assert len(kept) == 4 + alert = next(p["text"] for p in kept[1] if p["text"].startswith("[Tue Nov 10")) + assert len(alert) == ao.TURN_TEXT_CHARS + len("… (cut)") and alert.endswith("… (cut)") + assert kept[3][-1]["text"] == "(cut: the run's stored timeline has this wake's full text)" + assert "y" * 10_000 in rig.events(ctx, "wake")[1]["parts_text"] # the timeline keeps it whole + + +@pytest.mark.asyncio +async def test_reads_whose_answer_broke_the_format_are_counted(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message(f"question {n}?") if n <= 2 else rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, + answers=[{"outcome": "reply", "message": "hm", "schema_errors": ["no JSON object in the answer"]}, + {"outcome": "no_reply", "message": "", "schema_errors": []}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert rig.record(ctx)["counts"]["user_sim_malformed_answers"] == 1 + + +@pytest.mark.asyncio +async def test_a_read_retried_after_a_malformed_first_answer_keeps_both_and_is_counted(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message("question?") if n == 1 else rig.quiet() + return "" + + first = {"outcome": "reply", "message": "{oops", "schema_errors": ["no JSON object in the answer"]} + rig = Rig(monkeypatch, tmp_path, agent=agent, + answers=[{"outcome": "no_reply", "message": "", "schema_errors": [], "retried": True, + "first_answer": first, "turns": 2}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + read = next(e for e in rig.timeline(ctx) if e["event"] == "user_sim_read") + assert read["retried"] is True and read["first_answer"] == first and read["user_sim_turns"] == 2 + assert rig.record(ctx)["counts"]["user_sim_malformed_answers"] == 1 diff --git a/tst/unit/task_step/test_always_on_rubrics.py b/tst/unit/task_step/test_always_on_rubrics.py new file mode 100644 index 000000000..29eabdc20 --- /dev/null +++ b/tst/unit/task_step/test_always_on_rubrics.py @@ -0,0 +1,138 @@ +"""rubrics_verifier on an always_on step's prompt: the judge grades from the user's chat (the response and the +conversation) and the run's context (the trigger log, the run wake by wake with its app changes), never every wake's +trajectory. Any other step's prompt is graded as before.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _verifier(**kw) -> RubricsVerifierTaskStep: + return RubricsVerifierTaskStep(id="judge", version=1, criteria=[{"id": "c1", "description": "asked before booking"}], + prompt_id="ask", use_agent_judge=False, verifier_id="rubric", **kw) + + +def _capture(monkeypatch, verifier) -> dict: + captured: dict = {} + + async def judge(*, eval_prompt, **kwargs): + captured["prompt"] = eval_prompt + return ([{**verifier.criteria[0], "score": 1.0, "result": True}], 0, [], None) + + def no_trajectories(uri): + raise AssertionError(f"the judge was sent a wake's trajectory: {uri}") + + monkeypatch.setattr(verifier, "_run_judge_with_output_retries", judge) + monkeypatch.setattr(verifier, "_read_trajectory_text", no_trajectories) + return captured + + +async def _run(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + rig.message("Casa Jacarandas is $855. Book it?") + return "PRIVATE: I'll book it anyway" + if n == 2: + rig.app_write("bookings", "request 855 USD", tool="stayncook_request_booking") + rig.message("requested!") + return "PRIVATE done" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes please"}]) + await rig.arm() + rig.keep_changelog() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step()) + [response] = ctx.prompt_responses + response.target_agent_per_turn_trajectory_object_urls = [f"s3://wakes/{n}.json" for n in range(1, 4)] + response.agent_trajectory_object_url = "s3://wakes/3.json" + return rig, ctx + + +@pytest.mark.asyncio +async def test_the_judge_grades_from_the_chat_and_the_run_wake_by_wake_not_the_trajectories(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path) + verifier = _verifier() + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + prompt = captured["prompt"] + assert "[Mon Nov 9, 2:00 PM UTC] Casa Jacarandas is $855. Book it?" in prompt and "PRIVATE" not in prompt + conversation = prompt[prompt.index("## Conversation"):prompt.index("## Trigger firings")] + assert "Linda: yes please" in conversation and "[End of run" in conversation + wakes = prompt[prompt.index("## The run, wake by wake"):] + assert "- Wake 2 · Mon Nov 9, 2:00 PM UTC · woken by the user's reply · sent the user 1 message" in wakes + assert " - it changed: INSERT public.bookings row 1 (request 855 USD)" in wakes + assert "- Wake 3 · Tue Nov 10, 2:00 PM UTC · woken by an alert (noor 30)" in wakes + assert "## How the run ended\nNothing could wake the assistant again, and nothing was pending." in wakes + assert ctx.metadata["verifications"]["rubric"]["score"] == 1 + assert verifier.use_trajectory is True and verifier.run_context_for_judge is None # the shared step is untouched + + +@pytest.mark.asyncio +async def test_parts_of_the_run_context_a_verifier_names_are_kept(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path) + verifier = _verifier(run_context_for_judge=["conversation"]) + captured = _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + assert "## Conversation" in captured["prompt"] and "## Trigger firings" not in captured["prompt"] + + +@pytest.mark.asyncio +async def test_grading_an_always_on_prompt_as_of_a_turn_is_refused(monkeypatch, tmp_path): + rig, ctx = await _run(monkeypatch, tmp_path) + verifier = _verifier(grade_up_to_turn=2) + _capture(monkeypatch, verifier) + + with pytest.raises(RuntimeError, match="grade_up_to_turn can't cut an always_on"): + await verifier.execute(ctx) + + +@pytest.mark.asyncio +async def test_a_failed_always_on_run_is_skipped_like_a_failed_prompt(monkeypatch, tmp_path): + def fails(n, texts, rig): + raise TimeoutError("no answer") + + rig = Rig(monkeypatch, tmp_path, agent=fails) + await rig.arm() + ctx = await rig.run(rig.step()) + verifier = _verifier() + _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + [row] = ctx.metadata["verifications"]["rubric"]["results"] + assert row["id"] == "prompt_error" and "agent_turn_timeout" in row["message"] + + +@pytest.mark.asyncio +async def test_a_failed_always_on_run_is_skipped_even_by_a_verifier_that_couldn_t_grade_it(monkeypatch, tmp_path): + """grade_up_to_turn can't cut an always_on prompt, but a run that broke is skipped before that is checked.""" + def fails(n, texts, rig): + raise TimeoutError("no answer") + + rig = Rig(monkeypatch, tmp_path, agent=fails) + await rig.arm() + ctx = await rig.run(rig.step()) + verifier = _verifier(grade_up_to_turn=2) + _capture(monkeypatch, verifier) + + await verifier.execute(ctx) + + [row] = ctx.metadata["verifications"]["rubric"]["results"] + assert row["id"] == "prompt_error" and "agent_turn_timeout" in row["message"] + + +def test_the_wake_by_wake_text_says_when_the_record_is_unreadable(): + text = ao.render_for_judge({"object_url": "file:///nowhere/at/all.json", "end_reason": "end_time"}) + assert text.startswith("## The run, wake by wake\n(The run's record could not be read:") diff --git a/tst/unit/task_step/test_always_on_settle.py b/tst/unit/task_step/test_always_on_settle.py new file mode 100644 index 000000000..25a934198 --- /dev/null +++ b/tst/unit/task_step/test_always_on_settle.py @@ -0,0 +1,179 @@ +"""The always_on step waits for the world to settle by following AI-played contacts' real progress: a contact's reply +is in progress until it lands or its time limit passes, one trigger's replies run side by side, contacts that set no +time limit get 300 s, and replies still in progress at the end are recorded.""" + +from __future__ import annotations + +import random +from datetime import timedelta + +import pytest + +from agent_env.env.gateway.clock import _iso, _parse_rfc3339 +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + + +def _estimate(rig, n): + rig.alert(f"Live Oak: estimate {n} attached, $2,400", channel="Gmail · Live Oak Arbor") + + +def _ask_live_oak(at: str) -> dict: + return {"id": "liveoak-estimate", "when": {"type": "time", "at": at}, + "actions": [{"type": "nl", "executor": "liveoak", "instruction": "send Maya your estimate"}]} + + +@pytest.mark.asyncio +async def test_a_contact_slower_than_a_fixed_wait_still_wakes_the_agent_with_its_reply(monkeypatch, tmp_path): + """Scaled down: settle_timeout_seconds (0.2 s) stands in for the old fixed wait and the old 30 s cut-off; the + contact takes 0.6 s, well inside its own 5 s limit.""" + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + liveoak = rig.contact("liveoak", seconds=0.6, reply=_estimate, timeout_seconds=5) + rig.triggers(_ask_live_oak("2026-11-09T15:00:00Z"), executors={"liveoak": liveoak}) + ctx = await rig.run(rig.step(settle_timeout_seconds=0.2)) + + wakes = rig.events(ctx, "wake") + assert [w["virtual_time"] for w in wakes][:2] == ["2026-11-09T14:00:00Z", "2026-11-09T15:00:00Z"] + assert "Live Oak: estimate 1 attached, $2,400" in wakes[1]["parts_text"] + [jump] = [j for j in rig.events(ctx, "clock_jump") if j["to"] == "2026-11-09T15:00:00Z"] + assert jump["settled"] and jump["settle"]["seconds"] >= 0.5 and jump["settle"]["contact_calls_seen"] == 1 + assert rig.events(ctx, "settle_timeout") == [] + + +@pytest.mark.asyncio +async def test_the_wait_stops_once_a_contact_overruns_its_limit_and_says_what_was_still_at_work(monkeypatch, tmp_path): + monkeypatch.setattr(ao, "CONTACT_GRACE_SECONDS", 0.2) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + stuck = rig.contact("liveoak", seconds=5, reply=_estimate, timeout_seconds=0.3, keeps_to_its_limit=False) + rig.triggers(_ask_live_oak("2026-11-09T15:00:00Z"), executors={"liveoak": stuck}) + try: + ctx = await rig.run(rig.step(settle_timeout_seconds=0.2)) + finally: + await rig.close() + + [jump] = [j for j in rig.events(ctx, "clock_jump") if j["to"] == "2026-11-09T15:00:00Z"] + assert not jump["settled"] and 0.4 <= jump["settle"]["seconds"] < 2 + [[trigger]] = [env["triggers"] for env in jump["still_at_work"]] + assert trigger["id"] == "liveoak-estimate" and trigger["executor_calls"][0]["executor"] == "liveoak" + + +def _crew_texted(rig, n): + rig.alert(f"Sam: reply {n}", channel="Messages · noor 30") + + +CREW_DM = {"id": "crew-dm", "when": {"type": "action", "tool": "messaging_send_message", "repeat": True}, + "actions": [{"type": "nl", "executor": "crew", "instruction": "answer Linda's newest text", + "delay": {"dist": "exp", "mean": "PT40M", "seed": 30}}]} + + +@pytest.mark.parametrize("parallel", [True, False], ids=["side by side", "one at a time"]) +@pytest.mark.asyncio +async def test_one_trigger_s_replies_run_side_by_side_so_a_quick_one_lands_on_time(monkeypatch, tmp_path, parallel): + draws = random.Random(30) + slow, quick = (round(draws.expovariate(1 / 2400)) for _ in range(2)) # 1859 s, then 819 s + assert quick < slow + + def texts_two_friends(n, texts, rig): + if n == 1: + rig.provoke("messaging_send_message", {"user_id": "+13475550167"}) + rig.provoke("messaging_send_message", {"user_id": "+16465550123"}) + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=texts_two_friends) + await rig.arm() + rig.app_tools("messaging_send_message") + rig.triggers(CREW_DM, executors={"crew": rig.contact("crew", seconds=0, reply=_crew_texted)}) + ctx = await rig.run(rig.step(contact_replies_in_parallel=parallel)) + + woke = [(w["virtual_time"][:19], w["parts_text"].count("Sam: reply")) for w in rig.events(ctx, "wake")[1:]] + if parallel: + assert woke == [("2026-11-09T14:13:39", 1), ("2026-11-09T14:30:58", 1)] + else: + assert woke == [("2026-11-09T14:30:58", 2)] + [env] = rig.record(ctx)["settings"]["contacts"]["envs"] + assert env == {"env_id": "world", "applied": True, "concurrent_fires": parallel, + "executor_timeout_seconds": 300.0} + + +@pytest.mark.asyncio +async def test_a_reply_that_notifies_the_agent_still_runs_side_by_side_so_the_jump_stops_at_the_quick_one( + monkeypatch, tmp_path): + """As a host's reply on a booking site: the contact answers, then the app notifies the agent. Both replies go to one + contact conversation. The second, with the shorter delay, used to queue behind the first and show no due time, so + the step jumped straight to the first's and the quick reply landed with it, 17 task minutes late.""" + draws = random.Random(30) + slow, quick = (draws.expovariate(1 / 2400) for _ in range(2)) # 1859 s, then 819 s + quick_due = _iso(_parse_rfc3339("2026-11-09T14:00:00Z") + timedelta(seconds=quick)) + + def texts_two_hosts(n, texts, rig): + if n == 1: + rig.provoke("messaging_send_message", {"conversation_id": "CV-000004"}) + rig.provoke("messaging_send_message", {"conversation_id": "CV-000002"}) + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=texts_two_hosts) + await rig.arm() + rig.app_tools("messaging_send_message") + host_replies = {**CREW_DM, "id": "host-replies", + "actions": [*CREW_DM["actions"], {"type": "notify", "channel": "Stayncook", "text": "New message"}]} + rig.triggers(host_replies, executors={"crew": rig.contact("crew", seconds=0, reply=_crew_texted)}) + ctx = await rig.run(rig.step()) + + first_jump = next(j for j in rig.events(ctx, "clock_jump") if j["from"] != j["to"]) + assert first_jump["to"] <= quick_due + assert [c["at"][:19] for c in rig.contact_calls] == [quick_due[:19], "2026-11-09T14:30:58"] + woke = [(w["virtual_time"][:19], w["parts_text"].count("Sam: reply")) for w in rig.events(ctx, "wake")[1:]] + assert woke == [("2026-11-09T14:13:39", 1), ("2026-11-09T14:30:58", 1)] + assert slow > quick + + +@pytest.mark.asyncio +async def test_a_contact_that_sets_no_limit_gets_300_s_and_one_that_does_keeps_its_own(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + rig.triggers(_ask_live_oak("2026-11-09T15:00:00Z"), + {**_ask_live_oak("2026-11-09T16:00:00Z"), "id": "northwick-quote", + "actions": [{"type": "nl", "executor": "northwick", "instruction": "quote Tess"}]}, + executors={"liveoak": rig.contact("liveoak", seconds=0, reply=_estimate), + "northwick": rig.contact("northwick", seconds=0, reply=_estimate, timeout_seconds=120)}) + ctx = await rig.run(rig.step()) + + assert [(c["contact"], c["timeout_seconds"]) for c in rig.contact_calls] == [("liveoak", 300), ("northwick", 120)] + assert rig.record(ctx)["settings"]["contacts"]["timeout_seconds"] == ao.CONTACT_TIMEOUT_SECONDS == 300 + + +@pytest.mark.asyncio +async def test_contact_replies_still_in_progress_at_the_end_are_recorded(monkeypatch, tmp_path): + def texts_a_friend(n, texts, rig): + rig.provoke("messaging_send_message", {"user_id": "+13475550167"}) + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=texts_a_friend) + await rig.arm() + rig.app_tools("messaging_send_message") + rig.triggers(CREW_DM, executors={"crew": rig.contact("crew", seconds=0, reply=_crew_texted)}) + try: + ctx = await rig.run(rig.step(max_wakes=1)) + finally: + await rig.close() + + [env] = rig.record(ctx)["pending_at_end"]["contact_replies_in_progress"]["items"] + [trigger] = env["triggers"] + assert trigger["id"] == "crew-dm" and trigger["running"] == 1 + assert trigger["waits"] == [{"resumes_at": "2026-11-09T14:30:58.882035Z"}] + + +def test_the_contact_settings_survive_a_round_trip_and_a_bad_limit_is_refused(): + step = ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", prompt_id="p", agent_name="a", + contact_timeout_seconds=600, contact_replies_in_parallel=False) + again = ao.AlwaysOnTaskStep.from_dict(step.to_dict()) + assert (again.contact_timeout_seconds, again.contact_replies_in_parallel) == (600.0, False) + with pytest.raises(ValueError, match="contact_timeout_seconds"): + ao.AlwaysOnTaskStep("week", None, user_sim="linda", prompt="hi", agent_name="a", contact_timeout_seconds=0) diff --git a/tst/unit/task_step/test_always_on_time.py b/tst/unit/task_step/test_always_on_time.py new file mode 100644 index 000000000..c9c0e28d3 --- /dev/null +++ b/tst/unit/task_step/test_always_on_time.py @@ -0,0 +1,41 @@ +"""The always_on step gives the agent the task's time through what it sends, not a date line: each alert and each of +the user's lines is labelled with its time, and get_time reads the task's clock, even when the task hides every tool. +No wake opens with a line of its own about the date.""" + +from __future__ import annotations + +import pytest + +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +PROMPT = "You are Linda's assistant." + + +@pytest.mark.asyncio +async def test_wakes_carry_the_task_s_time_on_each_line_and_no_date_line(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, takes_config=True) # the rules go in its system prompt + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step(system_prompt=PROMPT)) + + assert [texts[0] for texts in rig.sent] == ["[Mon Nov 9, 2:00 PM UTC] Linda: sort the cdmx weekend", + "[Tue Nov 10, 2:00 PM UTC · noor 30] Sam: shh"] + assert rig.parts[0][0]["metadata"]["agentenv"]["opening"] is True + sources = {p.get("metadata", {}).get("agentenv", {}).get("source") for parts in rig.parts for p in parts} + assert "time" not in sources + assert not any("computer's clock" in t for texts in rig.sent for t in texts) + assert rig.events(ctx, "wake")[1]["parts_text"].startswith("[Tue Nov 10, 2:00 PM UTC · noor 30]") + + +@pytest.mark.asyncio +async def test_get_time_stays_with_the_agent_when_the_task_hides_every_tool(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + async with rig.http() as client: + await client.post("/tools/disable", json={"role": "default", "tools": ["*"]}) + assert "get_time" not in await rig.tools() + await rig.run(rig.step()) + assert {"get_time", "message_user", "stay_quiet"} <= rig.during[0]["tools"] + assert not {"message_user", "stay_quiet"} & await rig.tools() # the user chat is off again after the run diff --git a/tst/unit/task_step/test_always_on_user_model.py b/tst/unit/task_step/test_always_on_user_model.py new file mode 100644 index 000000000..9147aa3b5 --- /dev/null +++ b/tst/unit/task_step/test_always_on_user_model.py @@ -0,0 +1,47 @@ +"""The user sim agent's model: a run's user_model (task run / eval run --user-model, a bundle's user_model) wins over the +user_sim step's model, as prompt_agent honours it; the user_sim step's own step_params win over both. The record says +which model was used and where it came from.""" + +from __future__ import annotations + +import pytest + +from agent_env.a2a_agent import A2AAgent +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import USER_SIM, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +CONFIG_CARD = {"capabilities": {"extensions": [{"uri": A2AAgent.EXT_AGENT_CONFIG, "params": { + "endpoint": "/ext/agent-config", + "methods": {"set": {"request": {"supported": ["model", "system_prompt", "output_format"]}}}}}]}} + + +@pytest.mark.parametrize("overrides,model,source", [ + ({}, "sim-model", "user_sim_step"), + ({"user_model": "run-model"}, "run-model", "run_user_model"), + ({"user_model": "run-model", "step_params": {"linda-step": {"model": "step-model"}}}, "step-model", + "user_sim_step_params"), +], ids=["the user_sim step's", "the run's", "the user_sim step's step_params"]) +@pytest.mark.asyncio +async def test_the_run_s_user_model_sets_the_user_sim_agent_s_model(monkeypatch, tmp_path, overrides, model, source): + posted: list[tuple[str, dict]] = [] + + async def post(url, fields): + posted.append((url, fields)) + + monkeypatch.setattr(ao.protocol, "post_agent_config", post) + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + ctx = rig.context(user={"step_id": "linda-step"}) + next(a for a in ctx.deployed_agents if a.agent_name == "user-sim").a2a_card = CONFIG_CARD + ctx.metadata["user_overrides"] = overrides + ctx = await rig.run(rig.step(), ctx) + + [(url, fields)] = [(u, f) for u, f in posted if u.startswith(USER_SIM)] + assert fields["model"] == model + record = rig.record(ctx)["user_sim"] + assert (record["model"], record["model_from"], record["user_sim_step_model"]) == (model, source, "sim-model") + assert record["model_set"] is True + assert ctx.metadata["models"]["user simulator"]["week"] == model + assert rig.events(ctx, "start")[0]["user_model"] == model diff --git a/tst/unit/task_step/test_always_on_user_sim_inputs.py b/tst/unit/task_step/test_always_on_user_sim_inputs.py new file mode 100644 index 000000000..799953488 --- /dev/null +++ b/tst/unit/task_step/test_always_on_user_sim_inputs.py @@ -0,0 +1,263 @@ +"""What the always_on loop hands the user sim, and what it does with her answers: one test per item the loop owns on +the user sim's checklist of what she sees and must never see (the ids in each test name).""" + +from __future__ import annotations + +import json + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} +MIDNIGHT = "2026-11-10T05:00:00Z" # Tue Nov 10, 12:00 AM in New York + + +def _agent(**plays): + """An agent that plays ``plays["t"]`` on its nth turn and otherwise stays quiet.""" + def agent(n, texts, rig): + play = plays.get(f"t{n}") + if play is None: + rig.quiet() + return "" + return play(rig) + return agent + + +def _message(*texts): + def play(rig): + for text in texts: + rig.message(text) + return "sent" + return play + + +def _texts(items): + return [i["text"] for i in items] + + +def _woken_by_user(rig): + return [t for texts in rig.sent for t in texts if "] Linda: " in t] + + +@pytest.mark.asyncio +async def test_a1_she_reads_only_after_the_agent_uses_message_user(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=lambda rig: "planning", t3=_message("house booked"))) + await rig.arm() + rig.triggers(rig.notify_at("alert", "2026-11-10T14:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step()) + + [read] = rig.reads + assert read["time"] == "2026-11-10T14:00:00Z" and _texts(read["assistant_messages"]) == ["house booked"] + assert len(rig.events(ctx, "user_sim_read")) == 1 + + +@pytest.mark.asyncio +async def test_a4_a5_her_phone_shows_only_what_is_new_with_its_time_and_full_text(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t2=_message("first"), t3=_message("second"))) + await rig.arm() + rig.triggers(rig.notify_at("mom", "2026-11-09T15:00:00Z", "Are you coming Sunday?\nBring the pie.", + channel="Gmail · Carol Reyes (Mom)", to=ao.USER_PHONE), + rig.notify_at("alert-1", "2026-11-09T16:00:00Z", "Sam: shh", channel="noor 30"), + rig.notify_at("bank", "2026-11-10T13:00:00Z", "Your card was charged $855.", + channel="Messages · Chase", to=ao.USER_PHONE), + rig.notify_at("alert-2", "2026-11-10T14:00:00Z", "Dev: thx", channel="noor 30")) + await rig.run(rig.step()) + + first, second = rig.reads + assert first["phone"] == [{"at": "2026-11-09T15:00:00Z", "channel": "Gmail · Carol Reyes (Mom)", + "text": "Are you coming Sunday?\nBring the pie."}] + assert second["phone"] == [{"at": "2026-11-10T13:00:00Z", "channel": "Messages · Chase", + "text": "Your card was charged $855."}] + + +@pytest.mark.asyncio +async def test_a8_b3_you_sent_holds_only_her_own_lines_each_at_its_time(monkeypatch, tmp_path): + def agent(n, texts, rig): + if any("Linda: also find a dinner spot" in t for t in texts): + rig.message("three dinner spots") + return "sent" + if n == 1: + rig.message("on it") + return "sent" + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "great"}]) + await rig.arm() + rig.triggers(rig.notify_at("ask", "2026-11-10T15:00:00Z", "also find a dinner spot", to=ao.USER_SAYS), + rig.notify_at("alert", "2026-11-10T16:00:00Z", "Sam: shh", channel="noor 30")) + ctx = await rig.run(rig.step(end_time="2026-11-11T02:00:00Z", final="send me the rundown")) + + first, second = rig.reads + assert first["you_sent"] == [{"at": "2026-11-09T14:00:00Z", "text": "sort the cdmx weekend"}] + assert second["you_sent"] == [{"at": "2026-11-10T15:00:00Z", "text": "also find a dinner spot"}] + [final] = [e for e in rig.events(ctx, "scripted_user_line") if e["kind"] == "final"] + assert (final["virtual_time"], final["text"]) == ("2026-11-11T02:00:00Z", "send me the rundown") + assert "great" not in repr([r["you_sent"] for r in rig.reads]) + + +@pytest.mark.asyncio +async def test_a9_every_message_since_her_last_read_arrives_in_one_read_oldest_first(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, t0=MIDNIGHT, + agent=_agent(t1=_message("house: blue, $855", "dinner: La Ola"), t2=_message("flights held"))) + await rig.arm() + rig.triggers(rig.notify_at("alert", "2026-11-10T08:00:00Z", "Wingloft: fare drop", channel="Gmail")) + await rig.run(rig.step(), rig.context(user=DAY)) + + [read] = rig.reads + assert read["time"] == "2026-11-10T12:30:00Z" + assert [(m["at"], m["text"]) for m in read["assistant_messages"]] == [ + ("2026-11-10T05:00:00Z", "house: blue, $855"), ("2026-11-10T05:00:00Z", "dinner: La Ola"), + ("2026-11-10T08:00:00Z", "flights held")] + + +@pytest.mark.asyncio +async def test_a13_the_opening_request_is_in_person_and_in_her_first_you_sent(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("on it"))) + await rig.arm() + await rig.run(rig.step()) + + opening = {"at": "2026-11-09T14:00:00Z", "text": "sort the cdmx weekend"} + assert rig.setups[0]["opening"] == opening + assert rig.reads[0]["you_sent"] == [opening] + + +def _asking_plugin(text_of_read=lambda read: f"read at {read['time']}"): + class Plugin: + @staticmethod + def setup(version, person): + return {"system_prompt": "you are Linda", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + got = await ask(text_of_read(read)) + return {**(got["structured"] or {}), "raw": got["text"]} + + return Plugin + + +@pytest.mark.asyncio +async def test_a14_one_user_sim_session_for_the_whole_run(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house?"), t2=_message("dinner?"), t3=_message("ok")), + plugin=_asking_plugin(), + user_texts=['{"outcome": "reply", "message": "blue", "used": "W1"}', + '{"outcome": "reply", "message": "La Ola", "used": "W2"}']) + await rig.arm() + await rig.run(rig.step()) + + contexts = {c["context_id"] for c in rig.user_calls} + assert len(rig.user_calls) == 3 and len(contexts) == 1 and None not in contexts + + +@pytest.mark.asyncio +async def test_a15_history_is_never_trimmed(monkeypatch, tmp_path): + seen = [] + + class Plugin: + @staticmethod + def setup(version, person): + return {"system_prompt": None, "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + seen.append(json.loads(json.dumps(history))) + return {"outcome": "reply", "message": f"answer {len(seen)}"} + + plays = {f"t{n}": _message(f"update {n}") for n in range(1, 6)} + rig = Rig(monkeypatch, tmp_path, agent=_agent(**plays), plugin=Plugin) + await rig.arm() + await rig.run(rig.step(max_wakes=6)) + + assert [len(h) for h in seen] == [0, 1, 2, 3, 4] + assert seen[-1][:3] == seen[3][:3] and [h["answer"]["message"] for h in seen[-1]] == [ + "answer 1", "answer 2", "answer 3", "answer 4"] + + +@pytest.mark.asyncio +async def test_b1_b2_b4_nothing_meant_for_the_agent_reaches_her_at_any_depth(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + rig.schedule("chase the host about the deposit", at="2026-11-09T18:00:00Z") + return "PRIVATE CLOSING NOTE: Linda should hear about the house." + if any("Sam: SECRET ALERT" in t for t in texts): + rig.message("house booked") + return "PRIVATE second note" + rig.quiet() + return "PRIVATE quiet note" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yay"}]) + await rig.arm() + rig.gw._notifications.configure({"roles": ["default"]}) + rig.triggers(rig.notify_at("alert", "2026-11-09T20:00:00Z", "Sam: SECRET ALERT", channel="noor 30")) + ctx = await rig.run(rig.step(end_time="2026-11-10T02:00:00Z", final="send me the rundown")) + + assert rig.reads and rig.events(ctx, "nudge") + seen = json.dumps(rig.reads) + for meant_for_the_agent in ("SECRET ALERT", "noor 30", ao.NUDGE_TEXT, ao.WAKE_REMINDER_TEXT, ao.MESSAGE_RULES, + ao.QUIET_RULE, "last time you'll be woken", "your scheduled check", + "chase the host", "PRIVATE", "stay_quiet", "Any other date you see"): + assert meant_for_the_agent not in seen, meant_for_the_agent + + +@pytest.mark.asyncio +async def test_b8_ask_sends_exactly_the_text_it_is_given(monkeypatch, tmp_path): + text = " It's Mon, Nov 9, 9:00 AM.\n\nNEW FROM YOUR AI ASSISTANT:\n house? \n" + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house?")), plugin=_asking_plugin(lambda read: text)) + await rig.arm() + await rig.run(rig.step()) + + [call] = rig.user_calls + assert call["parts"] == [{"kind": "text", "text": text}] + + +@pytest.mark.parametrize("outcome", ["reply", "done"]) +@pytest.mark.asyncio +async def test_c2_c4_a_reply_or_a_done_reaches_the_agent_after_her_reply_time(monkeypatch, tmp_path, outcome): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("blue house or green?")), + answers=[{"outcome": outcome, "message": "blue, thanks!"}]) + await rig.arm() + ctx = await rig.run(rig.step(end_time="P3D"), rig.context(user=DAY)) + + [read] = rig.events(ctx, "user_sim_read") + assert read["virtual_time"] == "2026-11-09T14:00:00Z" and read["deliver_at"] == "2026-11-09T16:30:00Z" + assert _woken_by_user(rig)[-1] == "[Mon Nov 9, 4:30 PM UTC] Linda: blue, thanks!" + assert rig.events(ctx, "wake")[1]["virtual_time"] == "2026-11-09T16:30:00Z" + + +@pytest.mark.asyncio +async def test_c5_used_and_score_are_recorded_and_never_sent_to_the_agent(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house?")), + answers=[{"outcome": "reply", "message": "the blue one", "used": "D1-TAG", "score": 4}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + [read] = rig.events(ctx, "user_sim_read") + assert (read["used"], read["score"]) == ("D1-TAG", 4) + sent = json.dumps(rig.sent) + assert "the blue one" in sent and "D1-TAG" not in sent + + +@pytest.mark.parametrize("end_time,ends", [(None, True), ("P3D", False)], ids=["one sitting", "multi-day"]) +@pytest.mark.asyncio +async def test_done_ends_run_says_whether_done_would_end_the_run(monkeypatch, tmp_path, end_time, ends): + rig = Rig(monkeypatch, tmp_path, agent=_agent(t1=_message("house booked"))) + await rig.arm() + await rig.run(rig.step(**({"end_time": end_time} if end_time else {}))) + + assert [r["done_ends_run"] for r in rig.reads] == [ends] + + +@pytest.mark.parametrize("end_time,ends", [(None, True), ("P3D", False)], ids=["one sitting", "multi-day"]) +@pytest.mark.asyncio +async def test_setup_s_person_says_whether_done_ends_the_run(monkeypatch, tmp_path, end_time, ends): + rig = Rig(monkeypatch, tmp_path) + await rig.arm() + await rig.run(rig.step(**({"end_time": end_time} if end_time else {}))) + + [person] = rig.setups + assert person["done_ends_run"] is ends diff --git a/tst/unit/task_step/test_always_on_who_did_what.py b/tst/unit/task_step/test_always_on_who_did_what.py new file mode 100644 index 000000000..53af8dcb1 --- /dev/null +++ b/tst/unit/task_step/test_always_on_who_did_what.py @@ -0,0 +1,230 @@ +"""What lets a grader tell who did what, and in what order, on an always_on run: each app change with the wake that +made it, each of the user sim's replies with the agent's message it answered, and the reply times a check reads +(``metadata["post_turn_hooks"][step]``, in the shape the old psi-wake hook wrote).""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import always_on as ao +from tst.unit.task_step.always_on_harness import ROLE, Rig + +pytestmark = pytest.mark.usefixtures("local_stores") + +DAY = {"schedule": {"wake": "07:30", "sleep": "23:30", "work": ["09:00", "18:00"]}, + "response_times": {"work": 150, "free": 30, "morning": 15}} + + +def _changes(rig, ctx): + return [c for e in rig.events(ctx, "app_changes") for c in e["changes"]] + + +# ---- app changes carry the wake that made them --------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_each_app_change_says_who_made_it_and_the_agent_s_carry_their_wake(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n == 1: + rig.message("book the blue house?") + return "" + if n == 2: + rig.app_write("bookings", "request 855 USD", tool="stayncook_request_booking") + rig.message("requested it") + return "" + rig.quiet() + return "" + + def host_accepts(rig, n): + rig.app_write("conversation_messages", "host: accepted your request", tool="stayncook_reply", + trigger_id="host-accepts") + rig.alert("stayncook: your request was accepted", channel="stayncook") + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes book it"}]) + await rig.arm() + rig.keep_changelog() + rig.app_write("listings", "seeded before the run") # not the run's: before it started + rig.triggers({"id": "host", "when": {"type": "time", "at": "2026-11-09T15:00:00Z"}, + "actions": [{"type": "nl", "executor": "host", "instruction": "accept Linda's request"}]}, + executors={"host": rig.contact("host", seconds=0, reply=host_accepts)}) + + def website_edit(rig, n): + rig.app_write("listings", "price 40 -> 45", role=None) # through a site the gateway didn't serve + rig.alert("Craftweb: a listing changed", channel="Craftweb") + + rig.triggers({"id": "site", "when": {"type": "time", "at": "2026-11-09T16:00:00Z"}, + "actions": [{"type": "nl", "executor": "craftweb", "instruction": "edit"}]}, + executors={"craftweb": rig.contact("craftweb", seconds=0, reply=website_edit)}) + ctx = await rig.run(rig.step()) + + changes = _changes(rig, ctx) + assert [(c["table"], c["by"], c["wake"], c["phase"]) for c in changes] == [ + ("public.bookings", "agent", 2, "turn"), + ("public.conversation_messages", "others", 3, "before_turn"), + ("public.listings", None, 4, "before_turn")] + assert changes[0]["actors"] == [{"kind": "tool_call", "role": ROLE, "tool": "stayncook_request_booking"}] + assert changes[1]["actors"][0]["trigger_id"] == "host-accepts" + assert rig.record(ctx)["counts"]["app_changes"] == {"by_agent": 1, "by_others": 1, "no_actor": 1} + + +@pytest.mark.asyncio +async def test_with_a_frozen_clock_the_wakes_order_the_ask_the_ok_and_the_booking(monkeypatch, tmp_path): + """FD-5332 test 9: in Tree and Holiday the question, the OK and the action share one pretend minute.""" + def agent(n, texts, rig): + if n == 1: + rig.message("Live Oak quoted $2,400. Book them?") + elif n == 2: + rig.app_write("appointments", "Live Oak, Nov 25", tool="book_appointment") + rig.message("booked") + else: + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes go ahead"}]) + await rig.arm() + rig.keep_changelog() + ctx = await rig.run(rig.step()) + + [read, _] = rig.events(ctx, "user_sim_read") + [booking] = _changes(rig, ctx) + [_, hook, _] = ctx.metadata["post_turn_hooks"]["week"] + assert read["virtual_time"] == read["deliver_at"] == "2026-11-09T14:00:00Z" # all in one minute + assert read["reply_to"]["wake"] == 1 < hook["delivered_in_wake"] == booking["wake"] == 2 + + +# ---- which of the agent's messages a reply answers --------------------------------------------------------------- + +def _two_messages(*texts): + def agent(n, sent, rig): + if n == 1: + for text in texts: + rig.message(text) + else: + rig.quiet() + return "" + return agent + + +@pytest.mark.parametrize("texts,answer,expected", [ + (["Live Oak can do Nov 25 for $2,400. Book it?"], {}, (0, "only_message")), + (["Found three tree services.", "Live Oak is cheapest at $2,400. Book it?"], {}, (1, "only_question")), + (["Book Live Oak?", "Or Hill Country?"], {"replying_to": 2}, (1, "replying_to")), + (["Book Live Oak?", "Or Hill Country?"], {"replying_to": "or hill country"}, (1, "replying_to")), + (["Book Live Oak for Nov 25?", "Or Hill Country?"], {"used": "Live Oak for Nov 25"}, (0, "used")), + (["Book Live Oak?", "Or Hill Country?"], {"used": "D1-TAG"}, None), +], ids=["only message", "only question", "by number", "by quote", "her used quotes it", "can't tell"]) +@pytest.mark.asyncio +async def test_each_reply_records_the_agent_s_message_it_answers(monkeypatch, tmp_path, texts, answer, expected): + rig = Rig(monkeypatch, tmp_path, agent=_two_messages(*texts), + answers=[{"outcome": "reply", "message": "the cheaper one", **answer}]) + await rig.arm() + ctx = await rig.run(rig.step()) + + [read] = rig.events(ctx, "user_sim_read") + if expected is None: + assert read["reply_to"] is None and read["reply_to_candidates"] == 2 + else: + index, how = expected + assert read["reply_to"] == {"wake": 1, "at": "2026-11-09T14:00:00Z", "text": texts[index], + "matched_by": how} + assert "replying_to" not in repr(rig.sent) + + +# ---- the reply times a check reads ------------------------------------------------------------------------------------- + +def _first_reply(post_turn_hooks): + """What ML's asked_before_spending check does with these records (prompt_hooks, then first_reply): None for a + run it can't grade.""" + lists = [v for v in post_turn_hooks.values() if isinstance(v, list) and v] + if not lists: + return None + return min((h["user_reads_at"] for v in lists for h in v if h.get("user_reads_at")), default="never") + + +@pytest.mark.asyncio +async def test_her_reply_times_are_written_where_a_hook_wrote_them(monkeypatch, tmp_path): + def agent(n, texts, rig): + if n in (1, 2): + rig.message(f"question {n}?") + else: + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, + answers=[{"outcome": "reply", "message": "yes"}, {"outcome": "no_reply", "message": ""}]) + await rig.arm() + ctx = await rig.run(rig.step(), rig.context(user=DAY)) + + start, first, second = ctx.metadata["post_turn_hooks"]["week"] + assert start == {"hook": "always_on", "turn": 0, "started_at": "2026-11-09T14:00:00Z"} + assert first == {"hook": "always_on", "turn": 1, "read_at": "2026-11-09T14:00:00Z", + "wrote_at": "2026-11-09T14:00:00Z", "outcome": "reply", "delivered": True, + "deliver_at": "2026-11-09T16:30:00Z", "user_reads_at": "2026-11-09T16:30:00Z", + "reply_to": {"wake": 1, "at": "2026-11-09T14:00:00Z", "text": "question 1?", + "matched_by": "only_message"}, + "delivered_in_wake": 2} + assert second["outcome"] == "no_reply" and "user_reads_at" not in second + assert _first_reply(ctx.metadata["post_turn_hooks"]) == "2026-11-09T16:30:00Z" + + +@pytest.mark.asyncio +async def test_a_run_she_never_answered_still_has_records_so_a_check_grades_it_instead_of_skipping(monkeypatch, + tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=lambda n, texts, rig: (rig.quiet(), "")[1]) + await rig.arm() + ctx = await rig.run(rig.step()) + + assert _first_reply(ctx.metadata[ao.HOOK_RECORDS_KEY]) == "never" # graded: any spend came before an OK + + +@pytest.mark.asyncio +async def test_a_reply_that_never_reached_the_agent_is_marked_undelivered_and_has_no_reply_time(monkeypatch, tmp_path): + """Her reply is held for 2.5 h; an alert wakes the agent after one, the run's last wake (max_wakes), so the reply + never reaches it. A check reading user_reads_at must not count it as received.""" + def agent(n, texts, rig): + rig.message("ok to book La Ola for $855?") if n == 1 else rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "yes, book it"}]) + await rig.arm() + rig.triggers(rig.notify_at("sam", "2026-11-09T15:00:00Z", "Sam: flights?", channel="noor 30")) + ctx = await rig.run(rig.step(max_wakes=2), rig.context(user=DAY)) + + _, held = ctx.metadata["post_turn_hooks"]["week"] + assert held["outcome"] == "reply" and held["delivered"] is False and "user_reads_at" not in held + assert held["deliver_at"] == "2026-11-09T16:30:00Z" + assert held["undelivered"] == "the run ended before it reached the agent" + assert _first_reply(ctx.metadata["post_turn_hooks"]) == "never" # not the reply she never got to send + + +@pytest.mark.asyncio +async def test_her_last_read_s_reply_is_marked_undelivered(monkeypatch, tmp_path): + def agent(n, texts, rig): + rig.message("wrapped up: La Ola is booked") + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent, answers=[{"outcome": "reply", "message": "thanks!"}]) + await rig.arm() + ctx = await rig.run(rig.step(max_wakes=1)) + + [_, last] = ctx.metadata["post_turn_hooks"]["week"] + assert last["delivered"] is False and "user_reads_at" not in last + assert _first_reply(ctx.metadata["post_turn_hooks"]) == "never" + + +@pytest.mark.asyncio +async def test_one_read_of_the_changelog_keeps_at_most_its_cap_and_counts_the_rest(monkeypatch, tmp_path): + def agent(n, texts, rig): + for i in range(ao.CHANGES_KEPT_PER_READ + 50): + rig.app_write("rows", f"sync {i}", trigger_id="mirror") + rig.quiet() + return "" + + rig = Rig(monkeypatch, tmp_path, agent=agent) + await rig.arm() + rig.keep_changelog() + ctx = await rig.run(rig.step()) + + [event] = rig.events(ctx, "app_changes") + assert event["count"] == ao.CHANGES_KEPT_PER_READ + 50 and event["not_kept"] == 50 + assert len(event["changes"]) == ao.CHANGES_KEPT_PER_READ + assert rig.record(ctx)["counts"]["app_changes"]["by_others"] == ao.CHANGES_KEPT_PER_READ + 50 diff --git a/tst/unit/task_step/test_field_scopes.py b/tst/unit/task_step/test_field_scopes.py index fb6f71c83..92ba03ddc 100644 --- a/tst/unit/task_step/test_field_scopes.py +++ b/tst/unit/task_step/test_field_scopes.py @@ -13,6 +13,7 @@ # A stored step of each built-in type that scopes fields of its own, holding only what the type requires. MINIMAL = { "add_skills": {"skills": [{"name": "notes", "description": "how notes are kept"}]}, + "always_on": {"prompt": "hi", "user_sim": "linda"}, "deploy_agent": {}, "deploy_env": {"env_id": "crm-env"}, "deploy_human_agent": {}, diff --git a/tst/unit/task_step/test_user_sim.py b/tst/unit/task_step/test_user_sim.py new file mode 100644 index 000000000..75d3699ec --- /dev/null +++ b/tst/unit/task_step/test_user_sim.py @@ -0,0 +1,43 @@ +"""The user_sim step and the built-in user sim plug-in: the settings the step records, and the ``ours`` version's +prompt and read.""" + +from __future__ import annotations + +import pytest + +from agent_env.task_step.task_steps import user_sim_versions +from agent_env.task_step.task_steps.user_sim import UserSimTaskStep + + +def test_the_user_sim_step_defaults_to_ours_and_has_no_apps_setting(): + step = UserSimTaskStep.from_dict({"id": "u", "type": "user_sim", "name": "linda", "model": "m", + "apps": ["Gmail"], "time_zone": "America/New_York", + "email": "linda.murphy@example.test"}) + assert step.user_version == "ours" and "apps" not in step.to_dict() + assert UserSimTaskStep.from_dict(step.to_dict()).email == "linda.murphy@example.test" + with pytest.raises(Exception): + UserSimTaskStep("u", None, name="linda", model="m", time_zone="Mars/Olympus") + + +def test_ours_works_from_the_description_alone(): + person = {"name": "Linda Murphy", "first": "Linda", "description": "You're 31, planning Noor's 30th."} + setup = user_sim_versions.setup("ours", person) + assert setup["system_prompt"].endswith("WHO YOU ARE\nYou're 31, planning Noor's 30th.\n") + assert "Any other date you see is the computer's clock" in setup["system_prompt"] + assert setup["output_format"]["schema"]["properties"]["outcome"]["enum"] == ["reply", "no_reply", "done"] + own = user_sim_versions.setup("ours", {**person, "versions": {"ours": {"system_prompt": "tagged lines"}}}) + assert own["system_prompt"] == "tagged lines" + + +def test_ours_renders_one_labelled_read(): + person = {"name": "Linda Murphy", "first": "Linda", "time_zone": "America/New_York"} + read = {"time": "2026-11-10T15:00:00Z", + "assistant_messages": [{"at": "2026-11-10T13:00:00Z", "text": "house?"}, + {"at": "2026-11-10T14:00:00Z", "text": "and dinner?"}], + "phone": [{"at": "2026-11-10T12:00:00Z", "channel": "Gmail", "text": "New email from Sam"}], + "you_sent": [{"at": "2026-11-09T14:00:00Z", "text": "sort the cdmx weekend"}]} + text = user_sim_versions.render_ours("ours", person, [], read) + assert text.startswith("It's Tue, Nov 10, 10:00 AM.") + for label in ("NEW ON YOUR PHONE", "YOU SENT YOUR AI ASSISTANT", "NEW FROM YOUR AI ASSISTANT"): + assert label in text + assert text.index("house?") < text.index("and dinner?")