From f5454b6089f7cf45818028bf765727078f366f90 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 13:34:07 -0800 Subject: [PATCH 1/6] feat(website): a session cookie per site, /go?sid=, the clock's rate, conformance levels and models= (FD-4683) This PR's change, rebased as one commit onto #443. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/README.md | 14 +- packages/agentenv-protocol/pyproject.toml | 7 + .../src/agentenv_protocol/website/clock.py | 26 +- .../agentenv_protocol/website/conformance.py | 128 +++++++--- .../src/agentenv_protocol/website/database.py | 228 ++++++++++++++++++ .../agentenv_protocol/website/environment.py | 173 +++++++++++-- .../src/agentenv_protocol/website/routes.py | 4 +- .../src/agentenv_protocol/website/session.py | 42 +++- .../src/agentenv_protocol/website/store.py | 93 ++++++- .../tests/test_website_conformance.py | 134 ++++++++-- .../tests/test_website_database.py | 213 ++++++++++++++++ .../tests/test_website_environment.py | 150 +++++++++++- .../tests/test_website_notes_example.py | 16 +- .../tests/test_website_state.py | 132 ++++++++++ src/agent_env/env/gateway/clock.py | 12 +- tst/unit/env/gateway/clock_advance_test.py | 19 ++ uv.lock | 92 ++++++- 17 files changed, 1359 insertions(+), 124 deletions(-) create mode 100644 packages/agentenv-protocol/src/agentenv_protocol/website/database.py create mode 100644 packages/agentenv-protocol/tests/test_website_database.py create mode 100644 packages/agentenv-protocol/tests/test_website_state.py diff --git a/packages/agentenv-protocol/examples/website/notes/README.md b/packages/agentenv-protocol/examples/website/notes/README.md index eade87bde..fc5cba314 100644 --- a/packages/agentenv-protocol/examples/website/notes/README.md +++ b/packages/agentenv-protocol/examples/website/notes/README.md @@ -24,12 +24,14 @@ them. The page shows the universe's time. on only beside a role it verified with the same key, so a page cannot claim `admin` through it. Any other deploy sets no key, and its gateway passes on whatever role a browser sends; with no `WEBSITE_ROLE_KEY` set, every caller is `default` and the route is closed. -- **Users.** `ApiContext.user` is the user in the signed session cookie. With no cookie, it is the acting user (the +- **Users.** `ApiContext.user` is the user in the signed session cookie, `agentenv_session_notes` (the site's own, so + it doesn't collide with another site's under the same gateway host; the card names it). With no cookie, it is the acting user (the store's `current_user`), and `GET /api/session` signs that user in. `POST /api/sign-in` switches to another seeded user. A user's notes are invisible to every other user, and another user's note answers 404. - **State.** The site writes no data hooks, so it uses the default store. `data/add` loads the seed (as a data part, or - as a file part by bytes or `file://`), `data/get` exports it, and `data/reset` empties it. `data/get` is how - agentenv reads what a task left the site with. + as a file part by bytes or `file://`), `data/get` exports it, `data/reset` empties it, and `/go` returns + `{initial_state, current_state, state_diff}`. A page opened at `?sid=` keeps a state of its own, started + from the seed, which `/go?sid=` reports; without `sid` it is the default session, the data plane's. - **Time.** Notes are stamped with `self.now()`, which follows the gateway's clock once `clock/v1`'s `sync_time` names it. The frontend reads `X-Universe-Now` from every API response into `universeNow()` and shows it in the header. It uses this helper rather than `clock_patch=True`, so the page works the same against a server written without the @@ -66,11 +68,13 @@ The build context is the repo root, because the image installs the protocol pack ```bash docker build -f packages/agentenv-protocol/examples/website/notes/Dockerfile -t notes-site . docker run -d --name notes-site -p 18080:8000 notes-site -python -m agentenv_protocol.website.conformance http://localhost:18080 --clock-host host.docker.internal +python -m agentenv_protocol.website.conformance http://localhost:18080 --clock-host host.docker.internal --strict curl -s http://localhost:18080/api/notes ``` -Use `--clock-host` so the container can reach the checker's clock. The clock check leaves the server following a +Use `--clock-host` so the container can reach the checker's clock. `--strict` fails on a recommended check's warning +too (the card, `X-Universe-Now`, the clock, forwarded headers, cookies); without it only the contract's required checks +fail the run, and Notes passes every check either way. The clock check leaves the server following a clock that is gone once the check ends: later timestamps start from 2031 until the server is synced again, so restart the container before using it. diff --git a/packages/agentenv-protocol/pyproject.toml b/packages/agentenv-protocol/pyproject.toml index 73559b1c2..17492f651 100644 --- a/packages/agentenv-protocol/pyproject.toml +++ b/packages/agentenv-protocol/pyproject.toml @@ -49,6 +49,12 @@ computer = [ "Pillow>=10.4", "uvicorn>=0.30", ] +# A website whose state is an MCP server's database: @website(models=) and self.db, on a +# postgresql:// DATABASE_URL through psycopg 3, SQLAlchemy 2.1's driver for it. +db = [ + "sqlalchemy>=2.1", + "psycopg[binary]>=3.1", +] dev = [ "a2a-sdk[http-server]>=0.3.26,<0.4", "python-multipart>=0.0.13", @@ -59,6 +65,7 @@ dev = [ "fastmcp>=3.4,<4", # the standalone-fastmcp mount path "Pillow>=10.4", # the computer extra's frame crops and encodings "av>=17", # the computer extra's encoder, and the tests' decoder + "sqlalchemy>=2.1", # the db extra's models= site, tested on sqlite ] [project.urls] Homepage = "https://www.agentenvframework.com" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py index 0d27d86a9..dec301780 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/clock.py @@ -1,8 +1,9 @@ """The universe's clock as a site reads it: ``clock/v1``'s ``sync_time`` hands the server the gateway's ``/clock/time`` URL, as it hands it to MCP servers, and the site's time then follows that clock's ``{"virtual_time": }``, -read at most once a second and extrapolated between reads. The rate it extrapolates at is one two successive intervals -between reads agreed on, so a clock set or advanced between two reads is taken for a jump, not a speed; until two agree -it is real time's. An unsynced site reads wall time. Every answer of a site carries the time it answered at as ``X-Universe-Now``, +read at most once a second and extrapolated between reads. The rate it extrapolates at is the one the clock answers +with, ``virtual_seconds_per_real_second``, as the gateway's does; for a clock that answers none, it is one two +successive intervals between reads agreed on, so a clock set or advanced between two reads is taken for a jump, not a +speed, and until two agree it is real time's. An unsynced site reads wall time. Every answer of a site carries the time it answered at as ``X-Universe-Now``, RFC3339 in UTC. ``sync_time`` may also name the universe's time zone, an IANA name, which the clock keeps for the site to show local times in; the time itself stays UTC.""" from __future__ import annotations @@ -11,6 +12,7 @@ import datetime import json import logging +import math import time import httpx @@ -60,8 +62,8 @@ async def sync(self, url: str, timezone: str | None = None) -> datetime.datetime can't be read, keeping the clock and zone it followed. It waits for a refresh reading the clock it followed, so that read never lands on the new clock.""" async with self._reading(): - virtual = await _read(url) - self.url, self._virtual, self._rate, self._measured = url, virtual, 1.0, None + virtual, rate = await _read(url) + self.url, self._virtual, self._rate, self._measured = url, virtual, 1.0 if rate is None else rate, None self.timezone = timezone self._read_at = self._tried_at = time.monotonic() return virtual @@ -76,12 +78,14 @@ async def refresh(self) -> None: return self._tried_at = started try: - virtual = await _read(self.url) + virtual, rate = await _read(self.url) except Exception as e: logger.debug("reading the universe's clock at %s failed: %r", self.url, e) return read_at = time.monotonic() - if self._virtual is not None and read_at > self._read_at: + if rate is not None: + self._rate = rate + elif self._virtual is not None and read_at > self._read_at: measured = max(0.0, (virtual - self._virtual).total_seconds() / (read_at - self._read_at)) if self._measured is not None and abs(measured - self._measured) <= RATE_AGREEMENT * max(measured, self._measured, 1.0): self._rate = measured @@ -94,11 +98,15 @@ def _reading(self) -> asyncio.Lock: return self._lock -async def _read(url: str) -> datetime.datetime: +async def _read(url: str) -> tuple[datetime.datetime, float | None]: + """The clock's time, and the rate it says it runs at: None when it says none, or none that is a finite rate.""" async with httpx.AsyncClient() as client: response = await client.get(url, timeout=READ_TIMEOUT_S) response.raise_for_status() - return parse_time(response.json()["virtual_time"]) + body = response.json() + rate = body.get("virtual_seconds_per_real_second") + ok = isinstance(rate, (int, float)) and not isinstance(rate, bool) and math.isfinite(rate) and rate >= 0 + return parse_time(body["virtual_time"]), float(rate) if ok else None def clock_patch_script(now: datetime.datetime) -> bytes: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py index 14e62c93b..67a8a114a 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py @@ -1,18 +1,27 @@ -"""Check a running site's server against ``urn:agentenv:website/v1``, whatever it's written in: -``python -m agentenv_protocol.website.conformance `` prints each check and exits 1 when one failed, 2 when the -server couldn't be reached. Everything goes over HTTP: the card, ``/`` under a made-up host, readiness by the card's -``website/v1`` ``ready``, whose answer must be a ``WebsiteReadiness`` (by the declared health path, deprecated, for a -card without it), ``X-Universe-Now`` on every answer the other checks got, a data-plane round trip at the card's ``url`` -(data/get, data/reset, data/get, which no longer returns the exported data, data/add of what the first data/get -returned, data/get again), ``clock/v1``'s -``sync_time`` against a clock the checker serves, and the forwarded headers wherever an answer shows them, on ``/`` and -on a GET route with its trailing slash toggled, which a server may redirect. +"""Check a running site's server against ``urn:agentenv:website/v1``, whatever it's written in: ``python -m +agentenv_protocol.website.conformance `` prints each check and exits 1 when a required one failed (or, with +``--strict``, when a recommended one warned), 2 when the server couldn't be reached. Everything goes over HTTP. + +The required checks are the contract, what the gateway and a browser rely on of any server: ``/`` serves HTML under a +made-up host, the site is ready, by the card's ``website/v1`` ``ready``, whose answer must be a +``WebsiteReadiness``, or, for a card without it, by the health path (the card's, else ``/api/health``) answering 200, which is deprecated, the data plane and card's paths are not +the site's pages, and, when the card advertises the data plane, a round trip at the card's ``url`` (data/get, +data/reset, data/get, which no longer returns the exported data, data/add of what the first data/get returned, data/get +again) returns what it was given. A failed one is ``failed``. The recommended checks are what an SDK site does and the +gateway uses when a server offers it: a card declaring the site's domains, health path and routes, ``X-Universe-Now`` +on every answer the other checks got, ``clock/v1``'s ``sync_time`` against a clock the checker serves, the forwarded +headers wherever an answer shows them (on ``/`` and on a GET route with its trailing slash toggled, which a server may +redirect), and cookies tied to no domain, the session path's the one the card names. A failed one is a ``warning``. The round trip leaves the server's data as it found it, adding the export back when a step after the reset fails too. -The clock check leaves the -server following a clock that is gone once the check ends, so a server it ran against keeps extrapolating from that -clock's time until it's synced again. The checker's clock starts at ``CLOCK_START``, far enough from wall time that a -synced server shows it, and a synced server's ``X-Universe-Now`` may be ``CLOCK_SLACK_S`` seconds from it.""" +data/add sets the state a task starts from, so a server whose state shows changes since its initial state, as an SDK +site's does, would have them folded into it, and data/reset drops a site's named sessions, which data/get doesn't +export: the round trip is skipped for a server whose state shows changes or ``changed_sessions``. Its state is its +card's ``website/v1`` ``state``, else, for a card without it, what the path its ``params.state.path`` names answers, +else ``/go``'s, deprecated. The clock check leaves the server following a clock that is gone once the check ends, so a +server it ran against keeps extrapolating from that clock's time until it's synced again. The checker's clock starts at +``CLOCK_START``, far enough from wall time that a synced server shows it, and a synced server's ``X-Universe-Now`` may +be ``CLOCK_SLACK_S`` seconds from it.""" from __future__ import annotations import argparse @@ -33,24 +42,34 @@ from ..client import find_extension, find_extension_method from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, iso, parse_time -from .environment import DEFAULT_HEALTH_PATH +from .environment import DEFAULT_HEALTH_PATH, STATE_PATH from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness +from .state import WEBSITE_STATE_OP -Status = Literal["passed", "failed", "skipped"] +Status = Literal["passed", "failed", "warning", "skipped"] CLOCK_START = datetime.datetime(2031, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) CLOCK_SLACK_S = 120 +REQUIRED = ("any_host", "health", "reserved_paths", "data_plane") @dataclasses.dataclass(frozen=True) class Result: + """One check's outcome; ``required`` for a check of the contract, whose failure is ``failed``, and not for a + recommended one, whose failure is a ``warning``.""" + name: str status: Status detail: str = "" + required: bool = True @property def passed(self) -> bool: return self.status != "failed" + def fails(self, strict: bool = False) -> bool: + """Whether the result fails the run: a failed check, or with ``strict`` a warning too.""" + return self.status == "failed" or (strict and self.status == "warning") + def check(base_url: str, *, client: httpx.Client | None = None, clock_host: str = "127.0.0.1", timeout: float = 30) -> list[Result]: @@ -84,20 +103,27 @@ def run(self) -> list[Result]: card = response.json() if response.status_code == 200 else {} except ValueError: card = {} + card = card if isinstance(card, dict) else {} self.rpc_url = _rpc_url(self.base_url, card) site = (find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {} health = site.get("health") if isinstance(site.get("health"), str) else DEFAULT_HEALTH_PATH - steps = (("any_host", self._any_host), ("health", lambda: self._health(card, health)), - ("universe_now", self._universe_now), ("data_plane", lambda: self._data_plane(card)), + steps = (("card", lambda: self._card(response, site)), ("any_host", self._any_host), + ("health", lambda: self._health(card, health)), ("universe_now", self._universe_now), + ("reserved_paths", self._reserved_paths), ("data_plane", lambda: self._data_plane(card)), ("clock", lambda: self._clock(card, health)), ("forwarded", lambda: self._forwarded(site)), ("cookies", lambda: self._cookies(site))) outcomes = {name: _outcome(step) for name, step in steps if name != "universe_now"} outcomes["universe_now"] = _outcome(self._universe_now) - return [self._card(response, site), *(Result(name, *outcomes[name]) for name, _step in steps)] - - def _card(self, response: httpx.Response, site: dict) -> Result: + results = [] + for name, _step in steps: + status, detail = outcomes[name] + required = name in REQUIRED + results.append(Result(name, "warning" if status == "failed" and not required else status, detail, required)) + return results + + def _card(self, response: httpx.Response, site: dict) -> tuple[Status, str]: if response.status_code != 200: - return Result("card", "failed", f"{WELL_KNOWN_PATH} answered {response.status_code}") + return "failed", f"{WELL_KNOWN_PATH} answered {response.status_code}" problems = [] if not site: problems.append(f"declares no {WEBSITE_EXTENSION_URI} with params") @@ -109,7 +135,7 @@ def _card(self, response: httpx.Response, site: dict) -> Result: if not isinstance(routes, list) or not all(isinstance(r, dict) and {"method", "path", "name"} <= set(r) for r in routes): problems.append("its params' routes aren't a list of {method, path, name}") - return Result("card", "failed" if problems else "passed", "; ".join(problems)) + return ("failed" if problems else "passed"), "; ".join(problems) def _any_host(self) -> tuple[Status, str]: host = f"conformance-{uuid.uuid4().hex[:8]}.invalid" @@ -151,6 +177,14 @@ def _universe_now(self) -> tuple[Status, str]: parse_time(response.headers[UNIVERSE_NOW_HEADER]) return "passed", f"on all {len(self.responses)} answers" + def _reserved_paths(self) -> tuple[Status, str]: + paths = [RPC_PATH, WELL_KNOWN_PATH, f"{RPC_PATH}/conformance-{uuid.uuid4().hex[:8]}", f"{RPC_PATH}/", f"{WELL_KNOWN_PATH}/"] + for path in paths: + response = self.get(path) + if response.is_success and response.headers.get("content-type", "").startswith("text/html"): + return "failed", f"{path} answered a page, but the gateway keeps {RPC_PATH} and {WELL_KNOWN_PATH}" + return "passed", f"no page at {', '.join(paths)}" + def _rpc(self, method: str, params: dict | None = None) -> dict: response = self._seen(self.client.post(self.rpc_url, json={"jsonrpc": "2.0", "id": method, "method": method, "params": params or {}})) @@ -163,10 +197,17 @@ def _rpc(self, method: str, params: dict | None = None) -> dict: def _data_plane(self, card: dict) -> tuple[Status, str]: operations = (card.get("capabilities") or {}).get("operations") - missing = [m for m in (METHOD_GET, METHOD_RESET, METHOD_ADD) if operations is not None and m not in operations] + operations = operations if isinstance(operations, list) else [] + missing = [m for m in (METHOD_GET, METHOD_RESET, METHOD_ADD) if m not in operations] + if len(missing) == 3: + return "skipped", "the card advertises no data plane" if missing: return "failed", f"the card's operations lack {', '.join(missing)}" exported = self._rpc(METHOD_GET).get("parts") or [] + if where := self._changed_since_initial(card): + return "skipped", (f"data/get answered; {where} shows changes since the initial state, or named sessions " + "with changes, which data/reset and data/add would fold in or drop, so the round trip " + "was not run") self._rpc(METHOD_RESET) added_back = False try: @@ -188,6 +229,26 @@ def _data_plane(self, card: dict) -> tuple[Status, str]: return "failed", "data/get after data/reset and data/add of its own export returned other data" return "passed", f"{len(exported)} part(s) round-tripped through reset and add" + def _changed_since_initial(self, card: dict) -> str | None: + """Where the server's state was read when its ``state_diff`` names changes or it names ``changed_sessions``, + None when it names neither or can't be read: the card's ``website/v1`` ``state``, else the path its + ``params.state.path`` names, else ``/go``. Read aside, as no check's answer, since a server without one needn't + stamp its 404.""" + op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) + if op is not None and op.get("endpoint"): + where, request = f"website/v1 {WEBSITE_STATE_OP}", (op["method"], self.base_url + op["endpoint"]) + else: + declared = ((find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {}).get("state") + path = declared.get("path") if isinstance(declared, dict) else None + where = path if isinstance(path, str) and path.startswith("/") else STATE_PATH + request = ("GET", self.base_url + where) + try: + state = self.client.request(*request).json() + except Exception: + return None + changed = isinstance(state, dict) and bool(state.get("state_diff") or state.get("changed_sessions")) + return where if changed else None + def _clock(self, card: dict, health: str) -> tuple[Status, str]: if find_extension(card, CLOCK_EXTENSION_URI) is None: return "failed", f"the card declares no {CLOCK_EXTENSION_URI}" @@ -224,14 +285,18 @@ def _forwarded(self, site: dict) -> tuple[Status, str]: return "passed", f"no answer to {', '.join(paths)} named the internal Host" def _cookies(self, site: dict) -> tuple[Status, str]: + session = [] if isinstance(site.get("session_path"), str): - self.get(site["session_path"]) + session = self.get(site["session_path"]).headers.get_list("set-cookie") cookies = [c for r in self.responses for c in r.headers.get_list("set-cookie")] if not cookies: return "skipped", "no answer set a cookie" scoped = [c.split("=", 1)[0] for c in cookies if any(a.strip().lower().startswith("domain=") for a in c.split(";"))] if scoped: return "failed", f"cookies tied to a domain: {', '.join(sorted(set(scoped)))}" + named = site.get("session_cookie") + if isinstance(named, str) and session and named not in {c.split("=", 1)[0].strip() for c in session}: + return "failed", f"{site['session_path']} set {', '.join(c.split('=', 1)[0] for c in session)}, not the card's {named}" return "passed", f"{len(cookies)} cookie(s), none tied to a domain" @@ -286,7 +351,7 @@ def virtual_now() -> datetime.datetime: class Clock(BaseHTTPRequestHandler): def do_GET(self) -> None: - body = json.dumps({"virtual_time": iso(virtual_now())}).encode() + body = json.dumps({"virtual_time": iso(virtual_now()), "virtual_seconds_per_real_second": 1.0}).encode() self.send_response(200) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) @@ -306,7 +371,7 @@ def log_message(self, *args) -> None: server.server_close() -_MARK = {"passed": "PASS", "failed": "FAIL", "skipped": "----"} +_MARK = {"passed": "PASS", "failed": "FAIL", "warning": "WARN", "skipped": "----"} def main(argv: list[str] | None = None) -> int: @@ -315,6 +380,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("url", help="where the server answers, such as http://localhost:8000") parser.add_argument("--clock-host", default="127.0.0.1", help="the address the server reaches this checker at") parser.add_argument("--timeout", type=float, default=30, help="seconds for each request") + parser.add_argument("--strict", action="store_true", help="fail on a recommended check's warning too") args = parser.parse_args(argv) try: results = check(args.url, clock_host=args.clock_host, timeout=args.timeout) @@ -322,10 +388,12 @@ def main(argv: list[str] | None = None) -> int: print(f"couldn't reach {args.url}: {type(e).__name__}: {e}", file=sys.stderr) return 2 for result in results: - print(f"{_MARK[result.status]} {result.name}{': ' + result.detail if result.detail else ''}") - failed = sum(not r.passed for r in results) - print(f"{args.url}: {len(results) - failed} passed or skipped, {failed} failed") - return 1 if failed else 0 + kind = "required" if result.required else "recommended" + print(f"{_MARK[result.status]} {result.name} ({kind}){': ' + result.detail if result.detail else ''}") + counts = {status: sum(r.status == status for r in results) for status in _MARK} + print(f"{args.url}: {counts['passed']} passed, {counts['failed']} failed, {counts['warning']} warning(s), " + f"{counts['skipped']} skipped{' (strict)' if args.strict else ''}") + return 1 if any(r.fails(args.strict) for r in results) else 0 if __name__ == "__main__": diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py new file mode 100644 index 000000000..34373ea47 --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -0,0 +1,228 @@ +"""A site whose state is a database's: ``@website(models=...)`` names the data an MCP server keeps, so the site's pages +and the server's tools read and write the same rows under the same constraints (and the same rules, where the site +calls the server's own service code). ``models`` is either an object with +``load(data)``, ``export()`` and ``reset()`` (each may be async), the data hooks themselves, or SQLAlchemy models: a +declarative ``Base``, a ``MetaData``, or a module holding one (its ``Base``, else its one metadata), as the +synthetic_servers kit's ``database.py`` declares them. For models, the site's ``self.db`` is a SQLAlchemy session +factory on ``DATABASE_URL``, which agent-env points at the env's own schema of the service database (its +``search_path``), creating the models' tables there on first use; its data is a JSON object of each table's rows by +table name, which data/add inserts, data/get exports, and data/reset deletes. SQLAlchemy is imported only for models, +from the ``db`` extra, which carries SQLAlchemy 2.1 and its ``postgresql://`` driver, ``psycopg``; a site whose +``DATABASE_URL`` names another driver installs that one, and an engine whose driver isn't installed raises ImportError +naming it. + +Beside the tables, the data's string ``current_user`` names the user a browser starts signed in as, as the default +store's does, and the state data/add last loaded is the initial one ``/go`` reports. A models site keeps one state, +the database's, so ``/go`` answers it for every ``sid``.""" +from __future__ import annotations + +import copy +import datetime +import decimal +import itertools +import json +import types +import uuid +from typing import Any, Callable, Iterable + +from pydantic_core import to_jsonable_python + +from ..agent_env_environment import _maybe_await +from .store import CURRENT_USER, part_data, verifier_state + +HOOKS = ("load", "export", "reset") + + +def is_hooks(models: Any) -> bool: + """Whether ``models`` is data hooks of its own: an object with callable ``load``, ``export`` and ``reset``.""" + return all(callable(getattr(models, name, None)) for name in HOOKS) + + +def metadata_of(models: Any) -> Any: + """The SQLAlchemy ``MetaData`` ``models`` names: itself, its ``metadata``, or a module's ``Base``'s or one + metadata; raises TypeError for anything else.""" + sqlalchemy = _sqlalchemy() + if isinstance(models, sqlalchemy.MetaData): + return models + if isinstance(getattr(models, "metadata", None), sqlalchemy.MetaData): + return models.metadata + if isinstance(models, types.ModuleType): + if isinstance(getattr(getattr(models, "Base", None), "metadata", None), sqlalchemy.MetaData): + return models.Base.metadata + found = {id(m): m for value in vars(models).values() if isinstance( + m := value if isinstance(value, sqlalchemy.MetaData) else getattr(value, "metadata", None), + sqlalchemy.MetaData)} + if len(found) == 1: + return next(iter(found.values())) + if found: + raise TypeError(f"module {models.__name__} holds {len(found)} MetaData; pass models= its Base") + raise TypeError(f"models= is an object with load/export/reset, or SQLAlchemy models (a declarative Base, a " + f"MetaData, or a module holding one), not {models!r}") + + +def _sqlalchemy() -> Any: + try: + import sqlalchemy + except ImportError as e: + raise ImportError("@website(models=...) of SQLAlchemy models needs sqlalchemy: " + "pip install 'agentenv-framework-protocol[db]'") from e + return sqlalchemy + + +class Tables: + """The data hooks of SQLAlchemy ``metadata`` on the database at ``url()``: each table's rows by table name.""" + + def __init__(self, metadata: Any, url: Callable[[], str | None]) -> None: + self.metadata, self._url = metadata, url + self._engine: Any = None + self._sessions: Any = None + + @property + def engine(self) -> Any: + """The engine on ``DATABASE_URL``, made on first use with the models' tables created there; raises + ImportError naming the driver when the URL's isn't installed.""" + if self._engine is None: + url = self._url() + if not url: + raise RuntimeError("a site of SQLAlchemy models keeps its rows in DATABASE_URL, which isn't set") + sqlalchemy = _sqlalchemy() + try: + engine = sqlalchemy.create_engine(url) + except ImportError as e: + scheme = sqlalchemy.engine.make_url(url).drivername + raise ImportError(f"DATABASE_URL's {scheme}:// needs the database driver {e.name or 'it names'}, which " + f"isn't installed; the db extra carries psycopg for postgresql://, and a site installs any " + f"other driver itself (pip install {e.name or 'the driver'})") from e + self.metadata.create_all(engine) + self._engine = engine + return self._engine + + @property + def sessions(self) -> Any: + if self._sessions is None: + from sqlalchemy.orm import sessionmaker + + self._sessions = sessionmaker(bind=self.engine) + return self._sessions + + def export(self) -> dict: + sqlalchemy = _sqlalchemy() + with self.engine.connect() as connection: + return {table.name: [to_jsonable_python(dict(row._mapping)) for row in connection.execute( + sqlalchemy.select(table).order_by(*table.primary_key.columns))] for table in self.metadata.sorted_tables} + + def load(self, data: dict) -> None: + """Insert ``data``'s rows, each table's after the tables it refers to, in their order, each run of rows that name + the same columns as one insert (one insert takes its columns from its first row, so a row that leaves an + optional column out goes in its own); refuses a name that is no table.""" + tables = {table.name: table for table in self.metadata.sorted_tables} + unknown = sorted(set(data) - set(tables)) + if unknown: + raise ValueError(f"no table {', '.join(map(repr, unknown))}; the tables are {sorted(tables)}") + with self.engine.begin() as connection: + for name, table in tables.items(): + rows = data.get(name) or [] + if not (isinstance(rows, list) and all(isinstance(row, dict) for row in rows)): + raise ValueError(f"table {name!r}'s data is not a list of rows") + for _, run in itertools.groupby(rows, key=lambda row: sorted(row)): + connection.execute(table.insert(), [_row(table, row) for row in run]) + if connection.dialect.name == "postgresql": + _advance_sequences(connection, tables.values()) + + def reset(self) -> None: + with self.engine.begin() as connection: + for table in reversed(self.metadata.sorted_tables): + connection.execute(table.delete()) + + def key(self, name: str, row: dict) -> str: + """A row's identity, its primary key's values, which ``/go``'s diff matches rows by.""" + table = next((t for t in self.metadata.sorted_tables if t.name == name), None) + columns = [c.name for c in table.primary_key.columns] if table is not None else [] + return json.dumps([row.get(c) for c in columns] if columns else row, sort_keys=True, default=str) + + +_PARSERS: dict = {datetime.datetime: datetime.datetime.fromisoformat, datetime.date: datetime.date.fromisoformat, + datetime.time: datetime.time.fromisoformat, decimal.Decimal: decimal.Decimal, uuid.UUID: uuid.UUID} + + +def _row(table: Any, row: dict) -> dict: + """``row`` with each JSON string a date, time, decimal or UUID column holds parsed to that type.""" + parsed = dict(row) + for column in table.columns: + value = parsed.get(column.name) + if isinstance(value, str): + kind = _python_type(column) + if kind in _PARSERS: + parsed[column.name] = _PARSERS[kind](value.replace("Z", "+00:00") if kind is datetime.datetime else value) + return parsed + + +def _python_type(column: Any) -> type | None: + try: + return column.type.python_type + except NotImplementedError: + return None + + +def _advance_sequences(connection: Any, tables: Iterable[Any]) -> None: + """Move each serial key's sequence past the rows just loaded, so the next row the site or server adds takes a key + of its own.""" + sqlalchemy = _sqlalchemy() + for table in tables: + keys = list(table.primary_key.columns) + if len(keys) == 1 and keys[0].autoincrement in (True, "auto") and _python_type(keys[0]) is int: + quote = connection.dialect.identifier_preparer + name = quote.format_table(table) + connection.execute(sqlalchemy.text( + f"SELECT setval(pg_get_serial_sequence(:table, :column), COALESCE(MAX({quote.quote(keys[0].name)}), 0) + 1, " + f"false) FROM {name}"), {"table": name, "column": keys[0].name}) + + +class Sessions: + """A site's ``self.db``: ``self.db()`` is a SQLAlchemy ``Session`` on the site's database, and ``self.db.begin()`` + one in a transaction that commits as its ``with`` ends; the engine is made on first use, not at mount.""" + + def __init__(self, tables: Tables) -> None: + self._tables = tables + + def __call__(self, **options: Any) -> Any: + return self._tables.sessions(**options) + + def begin(self) -> Any: + return self._tables.sessions.begin() + + @property + def engine(self) -> Any: + return self._tables.engine + + +class ModelState: + """A models site's data plane and ``/go``: ``hooks`` loads, exports and resets its data, the state data/add last + loaded is the initial one, and the data's ``current_user`` is kept beside it.""" + + def __init__(self, hooks: Any) -> None: + self.hooks = hooks + self.initial: dict = {} + self.current_user: str | None = None + + async def reset(self) -> None: + await _maybe_await(self.hooks.reset()) + self.initial, self.current_user = {}, None + + async def load_parts(self, parts: list) -> None: + for part in parts: + data = dict(part_data(part)) + if CURRENT_USER in data: + self.current_user = data.pop(CURRENT_USER) + await _maybe_await(self.hooks.load(data)) + self.initial = copy.deepcopy(await self.export()) + + async def export(self) -> dict: + data = dict(await _maybe_await(self.hooks.export())) + if self.current_user is not None: + data[CURRENT_USER] = self.current_user + return data + + async def state(self) -> dict: + key = getattr(self.hooks, "key", None) + return verifier_state(copy.deepcopy(self.initial), await self.export(), key) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py index ac05ba6be..9a7a27059 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -2,12 +2,23 @@ plain HTTP on ``PORT`` for any host name: its ``@api`` routes, its static folder at ``/`` with ``index.html`` for the pages a single-page app routes itself, its health path, and the data plane and card the base class serves. Every answer carries ``X-Universe-Now``, the universe's time, which ``self.now()`` reads and ``clock/v1``'s ``sync_time`` ties to -the gateway's virtual clock. Without data hooks of its own a site keeps its data in ``self.store``, which the data -plane loads, exports and resets: ``data/get`` is how agentenv reads what a task left a site with. - -One route is built in, replaced by a site's own ``@api`` route at its very path and answering before one whose path -parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the acting -user in when nobody is (GET), or to sign out (DELETE).""" +the gateway's virtual clock. Without data hooks of its own a site keeps its state in ``self.store``, or with +``@website(models=...)`` in an MCP server's database, which ``self.db`` opens sessions of (``website.database``). + +A store-backed site keeps a state per session, the CUA-Gym convention the website state verifier reads: a request's +session is its ``?sid=``, else the one its browser last named, which the site keeps in the ``agentenv_sid_`` +cookie, else the default one. ``self.store`` is the request's session's store, each session started as a copy of the +initial state; the default session, ``sid=default``, is the state the data plane loads, exports and resets, and the +one a browser that names no session uses, so a site browsed without ``sid`` works as it did before sessions. + +Two routes are built in, each replaced by a site's own ``@api`` route at its very path and answering before one whose +path parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the +acting user in when nobody is (GET), or to sign out (DELETE); and, for a store-backed site, ``/go?sid=``, that +session's ``{initial_state, current_state, state_diff}`` at the path ``website/v1``'s ``params.state.path`` names, +which a reader of a card without ``state`` falls back to, with the site's ``changed_sessions``, the named sessions whose +state changed, which data/reset would drop; a session no request has used yet answers the initial state, unchanged. A +store-backed site answers ``website/v1``'s ``state``, of the session its ``sid`` names, with the same ``WebsiteState``, +among agentenv's own routes, which is how the website state verifier reads it.""" from __future__ import annotations import inspect @@ -27,6 +38,7 @@ from starlette.routing import Route from ..agent_env_environment import ( + _CARD_CONFIG_ATTR, _OP_ATTR, AgentEnvEnvironment, AgentEnvStarletteApplication, @@ -37,17 +49,20 @@ get_data, reset_data, ) -from ..types import DataPart +from ..types import DataPart, EnvironmentExtension from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock, clock_patch_script, iso +from .database import ModelState, Sessions, Tables, is_hooks, metadata_of from .errors import NotFound from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness from .routes import (ApiContext, ApiRoute, endpoint, error_response, forwarded_prefix, openapi, openapi_path, request_role, reserved_path, routes_of) -from .session import SESSION_COOKIE, SessionSigner, expires_at -from .store import Collections +from .session import SessionSigner, expires_at, session_cookie_name, sid_cookie_name +from .state import WEBSITE_STATE_OP, WebsiteState +from .store import DEFAULT_SID, Collections, active_sid, session_id DEFAULT_HEALTH_PATH = "/api/health" SESSION_PATH = "/api/session" +STATE_PATH = "/go" _WEBSITE_ATTR = "_aee_website" _READY_ATTR = "_aee_website_ready" _DEFAULT_HOOK_ATTR = "_aee_default_store_hook" @@ -57,27 +72,36 @@ class WebsiteDeclaration(NamedTuple): """What ``@website`` declares: the site's domains, its static folder and seed data file (relative to the class's - file), its health path, and whether its HTML pages read the universe's time.""" + file), its health path, whether its HTML pages read the universe's time, and the models whose database keeps its + state instead of the default store.""" domains: tuple[str, ...] = () static: Optional[str] = None health: str = DEFAULT_HEALTH_PATH clock_patch: bool = False data: Optional[str] = None + models: Any = None def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = (), static: str | None = None, - health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None) -> Any: - """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with.""" + health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None, + models: Any = None) -> Any: + """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with; ``models`` + keeps the site's state in a database instead, an MCP server's (see ``website.database``), and the site's + data plane and ``/go`` are that database's.""" if _cls is not None: raise TypeError("use @website(...) with parentheses") if not health.startswith("/") or health == "/" or reserved_path(health): raise ValueError(f"health is a path of its own under '/', not {health!r}") + if models is not None and data is not None: + raise ValueError("data= seeds the default store; a site of models= is seeded through data/add") + if models is not None and not is_hooks(models): + metadata_of(models) def deco(cls: type) -> type: if not (isinstance(cls, type) and issubclass(cls, WebsiteEnvironment)): raise TypeError(f"@website declares a WebsiteEnvironment subclass, not {cls!r}") - setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data)) + setattr(cls, _WEBSITE_ATTR, WebsiteDeclaration(tuple(domains), static, health, clock_patch, data, models)) _check_health(cls) return cls @@ -98,6 +122,17 @@ class WebsiteEnvironment(AgentEnvEnvironment): app: ClassVar[Any] = None asgi: Any = None + db: Any = None + _store: Collections + + @property + def store(self) -> Collections: + """The state store of the request's session: the default session's outside a request.""" + return self._store.session(active_sid.get()) + + @store.setter + def store(self, value: Collections) -> None: + self._store = value def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) @@ -113,8 +148,14 @@ def __init__(self) -> None: self.store = Collections() if declaration.data is not None: self.store.load(json.loads(self._site_path(declaration.data).read_text(encoding="utf-8"))) + self.store.mark_initial() self.clock = UniverseClock() - self._signer = SessionSigner() + self._signer = SessionSigner(cookie_name=session_cookie_name(self.environment_name())) + models = declaration.models + self._tables = (None if models is None or is_hooks(models) + else Tables(metadata_of(models), lambda: self.database_url)) + self._models = None if models is None else ModelState(models if self._tables is None else self._tables) + self.db = Sessions(self._tables) if self._tables is not None else None @classmethod def declaration(cls) -> WebsiteDeclaration: @@ -124,6 +165,17 @@ def declaration(cls) -> WebsiteDeclaration: def routes(cls) -> list[ApiRoute]: return routes_of(cls) + def environment_name(self) -> str: + """The site's name, as its card names it: ``ENVIRONMENT_NAME``, else the card's declared name, else the + class's.""" + declared = (getattr(self, _CARD_CONFIG_ATTR, None) or {}).get("name") + return os.environ.get("ENVIRONMENT_NAME") or declared or type(self).__name__ + + @property + def session_cookie(self) -> str: + """The name of the site's session cookie, ``agentenv_session_``.""" + return self._session_signer().cookie_name + @property def database_url(self) -> str | None: """Where the site keeps its task state: its own schema of the env's database.""" @@ -145,12 +197,15 @@ def timezone(self) -> str | None: return self._clock().timezone def acting_user(self) -> str | None: - """Who a browser with no session is signed in as: the store's ``current_user`` unless a site says else.""" + """Who a browser with no session is signed in as: the store's ``current_user``, or the one a models site's data + named, unless a site says else.""" + if getattr(self, "_models", None) is not None: + return self._models.current_user return self.store.current_user if hasattr(self, "store") else None def context(self, request: Request) -> ApiContext: """The ``ApiContext`` of ``request``, for a mounted ``app``'s routes as for ``@api`` ones.""" - session = self._session_signer().decode(request.cookies.get(SESSION_COOKIE)) + session = self._session_signer().read(request.cookies) user = session["user"] if session is not None else self.acting_user() return ApiContext(request, user, request_role(request), session, self._session_signer()) @@ -162,11 +217,14 @@ def create_app(self) -> Starlette: api_routes = self.routes() taken = {openapi_path(r.path) for r in api_routes} root = self._site_path(declaration.static) if declaration.static is not None else None - asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self)]) + asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self), + Middleware(_Session, cookie=sid_cookie_name(self.environment_name()))]) AgentEnvStarletteApplication(self._build_card(), self).add_routes_to_app(asgi) asgi.router.routes.append(Route(declaration.health, self._health, methods=["GET"])) if SESSION_PATH not in taken: asgi.router.routes.append(Route(SESSION_PATH, self._session, methods=["GET", "DELETE"])) + if self._serves_state_path(): + asgi.router.routes.append(Route(STATE_PATH, self._state, methods=["GET"])) asgi.router.routes.extend(Route(r.path, _unreserved(endpoint(getattr(self, r.fn.__name__), r, self.context)), methods=[r.method]) for r in api_routes) asgi.router.default = _Fallback(root, type(self).app) @@ -190,10 +248,18 @@ def extension_params(self, uri: str) -> dict: if uri == WEBSITE_EXTENSION_URI: declaration = self.declaration() return {"domains": list(declaration.domains), "health": declaration.health, - "routes": [r.advertised() for r in self.routes()], "session_cookie": SESSION_COOKIE, - "session_path": SESSION_PATH} + "routes": [r.advertised() for r in self.routes()], "session_cookie": self.session_cookie, + "session_path": SESSION_PATH, **({"state": {"path": STATE_PATH}} if self._serves_state_path() else {})} return super().extension_params(uri) + def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: + """The extensions the base class serves, ``website/v1``'s ``state`` only for a store-backed site: one with data + hooks of its own keeps its state where ``self.store`` can't see it.""" + served = super().served_extensions(extensions) + if self.uses_store: + return served + return [_without_method(e, WEBSITE_STATE_OP) if e.uri == WEBSITE_EXTENSION_URI else e for e in served] + @extension(CLOCK_EXTENSION_URI, description="Sync this site to the gateway's virtual clock.") async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) -> dict: """Follow the gateway's virtual time at ``env_get_time_url`` from now on, in ``timezone`` when the task names @@ -204,8 +270,9 @@ async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) @extension(WEBSITE_EXTENSION_URI, op="openapi", method="GET", description=( "A site's server: domains, its health path and its @api routes by name; openapi returns the OpenAPI document " - "of its routes, ready whether the site is usable now and why not, and session the cookies a browser adds to " - "start signed in as user, or signed out without one.")) + "of its routes, ready whether the site is usable now and why not, state the task state it is in (its " + "initial and current state and their diff), and session the cookies a browser adds to start signed in as user, " + "or signed out without one.")) async def _website_openapi(self) -> dict: name = self._application.environment_card.name if self._application is not None else type(self).__name__ return openapi(name, self.routes()) @@ -217,6 +284,16 @@ async def _website_ready(self) -> dict: problems = await self._not_ready() return WebsiteReadiness(ready=not problems, problems=problems).model_dump() + @extension(WEBSITE_EXTENSION_URI, op=WEBSITE_STATE_OP, method="GET") + async def _website_state(self, sid: Optional[str] = None) -> dict: + """The site's task state, a ``WebsiteState``: its state when the task started, now, and what changed between + them, of session ``sid``'s store (the default session's without one), with the site's ``changed_sessions`` as + ``/go`` names them, or of a models site's database, which keeps one.""" + if self._models is not None: + return WebsiteState.model_validate(await self._models.state()).model_dump() + return {**WebsiteState.model_validate(self._store.session(sid).state()).model_dump(), + "changed_sessions": self._store.changed_sessions()} + @extension(WEBSITE_EXTENSION_URI, op="session") async def _website_session(self, user: Optional[str] = None) -> dict: cookie = self._session_signer().cookie(user) @@ -224,15 +301,20 @@ async def _website_session(self, user: Optional[str] = None) -> dict: @reset_data async def _store_reset(self) -> None: - self.store.reset() + if self._models is not None: + return await self._models.reset() + self._store.reset() @add_data async def _store_add(self, parts: list) -> None: - self.store.load_parts(parts) + if self._models is not None: + return await self._models.load_parts(parts) + self._store.load_parts(parts) + self._store.mark_initial() @get_data async def _store_get(self) -> list: - return [DataPart(data=self.store.export())] + return [DataPart(data=await self._models.export() if self._models is not None else self._store.export())] async def _not_ready(self) -> list[str]: return [check.__name__ for check, _marker in _marked_members(self, _READY_ATTR) if not await _ready(check)] @@ -252,6 +334,15 @@ async def _session(self, request: Request) -> Response: return context.apply(JSONResponse({"user": context.user, "role": context.role, "signed_in": context.user is not None})) + def _serves_state_path(self) -> bool: + """Whether the site serves its state at ``STATE_PATH``: it is store-backed and no ``@api`` route takes the path.""" + return self.uses_store and STATE_PATH not in {openapi_path(r.path) for r in self.routes()} + + async def _state(self, request: Request) -> Response: + if self._models is not None: + return JSONResponse(await self._models.state()) + return JSONResponse({**self.store.state(), "changed_sessions": self._store.changed_sessions()}) + def _clock(self) -> UniverseClock: if not hasattr(self, "clock"): raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") @@ -272,6 +363,12 @@ def _site_path(cls, path: str) -> Path: setattr(getattr(WebsiteEnvironment, _hook), _DEFAULT_HOOK_ATTR, True) +def _without_method(extension: EnvironmentExtension, method: str) -> EnvironmentExtension: + params = dict(extension.params or {}) + params["methods"] = {name: entry for name, entry in (params.get("methods") or {}).items() if name != method} + return extension.model_copy(update={"params": params}) + + async def _ready(check: Any) -> bool: try: return bool(await _maybe_await(check())) @@ -319,6 +416,34 @@ def _first(value: bytes | None) -> str: return value.decode("latin-1").split(",")[0].strip() if value else "" +class _Session: + """Runs the request in its session (``active_sid``): its ``sid`` query parameter's, else its ``cookie``'s; an + answer to a request naming a session other than its cookie's sets the cookie, so a page opened at ``?sid=`` keeps + its session for the requests it makes after.""" + + def __init__(self, app: Any, cookie: str) -> None: + self.app, self.cookie = app, cookie + + async def __call__(self, scope: dict, receive: Any, send: Any) -> None: + if scope["type"] != "http": + return await self.app(scope, receive, send) + request = Request(scope) + named, kept = request.query_params.get("sid"), request.cookies.get(self.cookie) + sid = session_id(named if named is not None else kept) + + async def remembered(message: dict) -> None: + if message["type"] == "http.response.start" and named is not None and session_id(kept) != sid: + MutableHeaders(scope=message).append( + "set-cookie", f"{self.cookie}={sid or DEFAULT_SID}; Path=/; HttpOnly; SameSite=Lax") + await send(message) + + token = active_sid.set(sid) + try: + await self.app(scope, receive, remembered) + finally: + active_sid.reset(token) + + class _Universe: """Stamps ``X-Universe-Now`` on every answer, and with ``clock_patch`` puts the clock script in its whole HTML (never a range of it), holding only the HTML up to its ```` (at most ``HEAD_SCAN_BYTES``), so a page streamed in chunks reaches the browser diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py index 939803342..15158ddc8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/routes.py @@ -26,7 +26,7 @@ from ..tool_context import DEFAULT_ROLE, caller_from_request from ..types import RPC_PATH, WELL_KNOWN_PATH from .errors import ApiError, BadRequest, Forbidden -from .session import SESSION_COOKIE, SessionSigner +from .session import SessionSigner logger = logging.getLogger(__name__) @@ -154,7 +154,7 @@ def _set_session(self, user: str | None) -> None: def apply(self, response: Response) -> Response: for value, expires in self._set_cookie: at = datetime.datetime.fromtimestamp(expires, datetime.timezone.utc) - response.set_cookie(SESSION_COOKIE, value, expires=at, path="/", httponly=True, samesite="lax") + response.set_cookie(self._signer.cookie_name, value, expires=at, path="/", httponly=True, samesite="lax") return response diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py index 20e4ad36f..b798afcd4 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py @@ -1,6 +1,8 @@ """A site's signed session cookie: who the browser is signed in as, or that it signed out, signed with HMAC-SHA256 under ``WEBSITE_SESSION_SECRET``, or a secret of the process's own when that isn't set. The cookie names no domain, -so it holds under whatever host the site is reached at.""" +so it holds under whatever host the site is reached at, and its name is the site's own, ``agentenv_session_``, so +two sites a browser reaches under one gateway host (``/website/a/``, ``/website/b/``) keep a session each; a site still +reads the shared ``agentenv_session`` of older sites, for one release.""" from __future__ import annotations import base64 @@ -10,19 +12,46 @@ import hmac import json import os +import re import secrets import time SESSION_COOKIE = "agentenv_session" +SID_COOKIE = "agentenv_sid" SECRET_ENV = "WEBSITE_SESSION_SECRET" SESSION_TTL_S = 7 * 24 * 3600 SECRET_BYTES = 32 +_NOT_TOKEN = re.compile(r"[^A-Za-z0-9_.-]") + + +def _cookie_suffix(environment_name: str) -> str: + """The part of a site's cookie names that is its own: the name itself when a cookie name can hold each of its + characters, else the name with each such character made ``_`` and ``~`` and a hash of the whole name appended. No + name kept as itself holds a ``~``, so two distinct names never make one suffix.""" + if not _NOT_TOKEN.search(environment_name): + return environment_name + digest = hashlib.sha256(environment_name.encode("utf-8", "surrogatepass")).hexdigest()[:12] + return f"{_NOT_TOKEN.sub('_', environment_name)}~{digest}" + + +def session_cookie_name(environment_name: str | None) -> str: + """The session cookie of the site named ``environment_name``: ``agentenv_session_``, a name a cookie name + can't hold spelled as ``_cookie_suffix`` spells it; the shared ``agentenv_session`` for a site with no name.""" + return f"{SESSION_COOKIE}_{_cookie_suffix(environment_name)}" if environment_name else SESSION_COOKIE + + +def sid_cookie_name(environment_name: str | None) -> str: + """The cookie in which the site named ``environment_name`` keeps the store session its browser last named: + ``agentenv_sid_``, made as ``session_cookie_name`` makes its name, ``agentenv_sid`` for a site with no name. + Its prefix is no session cookie's, so no site's store-selection cookie is another site's session cookie.""" + return f"{SID_COOKIE}_{_cookie_suffix(environment_name)}" if environment_name else SID_COOKIE class SessionSigner: - def __init__(self, secret: str | bytes | None = None) -> None: + def __init__(self, secret: str | bytes | None = None, cookie_name: str = SESSION_COOKIE) -> None: secret = secret if secret is not None else os.environ.get(SECRET_ENV) self._key = (secret.encode() if isinstance(secret, str) else secret) or secrets.token_bytes(SECRET_BYTES) + self.cookie_name = cookie_name def encode(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> tuple[str, int]: """The cookie value for ``user`` signed in, or signed out when None, and when it expires, in epoch @@ -45,11 +74,18 @@ def decode(self, value: str | None) -> dict | None: return None return {"user": data.get("u")} + def read(self, cookies: dict) -> dict | None: + """The session of a request's ``cookies``: its own cookie's, else the shared legacy cookie's.""" + session = self.decode(cookies.get(self.cookie_name)) + if session is None and self.cookie_name != SESSION_COOKIE: + session = self.decode(cookies.get(SESSION_COOKIE)) + return session + def cookie(self, user: str | None, ttl_s: int = SESSION_TTL_S) -> dict: """The cookie a browser sets to be signed in as ``user``, as a browser automation adds one; its domain is left to whoever adds it.""" value, expires = self.encode(user, ttl_s) - return {"name": SESSION_COOKIE, "value": value, "path": "/", "expires": expires, "httpOnly": True, + return {"name": self.cookie_name, "value": value, "path": "/", "expires": expires, "httpOnly": True, "sameSite": "Lax"} def _sign(self, payload: str) -> str: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py index 657ea78f0..63c316a75 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py @@ -1,11 +1,19 @@ -"""The default data store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON -object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as; -the data plane loads, exports and resets it.""" +"""The default state store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON +object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as. +It remembers the state it was last loaded with, so a verifier can read the initial state, the current one and the +difference between them. + +A store also keeps sessions, as CUA-Gym sites do, so browsers that each name a session id (``?sid=``) never see one +another's changes: ``session(sid)`` is a store of its own, started on first use as a copy of this store's initial +state, and the session ``default``, or none, is this store itself. ``reset`` drops every session; a session started +before a later ``load`` keeps the state it was started with.""" from __future__ import annotations import base64 +import contextvars import copy import json +import re from pathlib import Path from typing import Any, Iterable from urllib.parse import unquote, urlparse @@ -13,6 +21,17 @@ from ..types import DataPart, FilePart, Part, TextPart CURRENT_USER = "current_user" +DEFAULT_SID = "default" +SID_MAX = 128 +_NOT_SID = re.compile(r"[^A-Za-z0-9_-]") +active_sid: contextvars.ContextVar[str | None] = contextvars.ContextVar("agentenv_website_sid", default=None) + + +def session_id(value: str | None) -> str | None: + """The session id ``value`` names, as CUA-Gym sites read ``sid``: its letters, digits, ``_`` and ``-``, at most + ``SID_MAX`` of them; None, the default session, for one that is empty or ``default``.""" + sid = _NOT_SID.sub("", value or "")[:SID_MAX] + return None if sid in ("", DEFAULT_SID) else sid class Collections: @@ -21,12 +40,33 @@ class Collections: def __init__(self, data: dict | None = None) -> None: self._collections: dict[str, list[dict]] = {} self.current_user: str | None = None + self.initial: dict = {} + self._sessions: dict[str, Collections] = {} if data: self.load(data) + self.mark_initial() def __getitem__(self, name: str) -> list[dict]: return self._collections.setdefault(name, []) + def session(self, sid: str | None) -> Collections: + """The store of session ``sid``, started as a copy of this one's initial state on first use; this store + itself for the default session.""" + sid = session_id(sid) + if sid is None: + return self + if sid not in self._sessions: + self._sessions[sid] = Collections(self.initial) + return self._sessions[sid] + + def sessions(self) -> list[str]: + """The ids of the sessions started since the store was last reset, the default one aside.""" + return list(self._sessions) + + def changed_sessions(self) -> list[str]: + """The ids of the sessions, the default one aside, whose state differs from the one they started with.""" + return [sid for sid, store in self._sessions.items() if store.diff()] + def names(self) -> list[str]: return list(self._collections) @@ -70,11 +110,13 @@ def load(self, data: dict) -> None: def load_parts(self, parts: Iterable[Part]) -> None: for part in parts: - self.load(_part_data(part)) + self.load(part_data(part)) def reset(self) -> None: self._collections.clear() self.current_user = None + self.initial = {} + self._sessions.clear() def export(self) -> dict: data: dict = copy.deepcopy(self._collections) @@ -82,8 +124,49 @@ def export(self) -> dict: data[CURRENT_USER] = self.current_user return data + def mark_initial(self) -> None: + self.initial = self.export() + + def diff(self) -> dict: + return state_diff(self.initial, self.export()) + + def state(self) -> dict: + return verifier_state(self.initial, self.export()) + + +def state_diff(initial: dict, current: dict, key: Any = None) -> dict: + """Each collection that changed from ``initial`` to ``current``, as its ``added``, ``removed`` and ``changed`` + objects, matched by ``key(name, doc)``, their ``id`` by default; a changed one as its ``id``, ``before`` and + ``after``.""" + key = key or (lambda name, doc: _key(doc)) + changes: dict = {} + for name in dict.fromkeys([*initial, *current]): + if name == CURRENT_USER: + continue + before = {key(name, d): d for d in initial.get(name, [])} + after = {key(name, d): d for d in current.get(name, [])} + entry = { + "added": [d for k, d in after.items() if k not in before], + "removed": [d for k, d in before.items() if k not in after], + "changed": [{"id": d.get("id"), "before": before[k], "after": d} + for k, d in after.items() if k in before and before[k] != d], + } + if any(entry.values()): + changes[name] = entry + return changes + + +def verifier_state(initial: dict, current: dict, key: Any = None) -> dict: + """What ``/go`` answers: ``{initial_state, current_state, state_diff}``.""" + return {"initial_state": copy.deepcopy(initial), "current_state": current, + "state_diff": state_diff(initial, current, key)} + + +def _key(doc: dict) -> str: + return json.dumps(doc.get("id")) if doc.get("id") is not None else json.dumps(doc, sort_keys=True) + -def _part_data(part: Part) -> dict: +def part_data(part: Part) -> dict: """A part's JSON: a data part's data, a text part's text, or a file part's bytes or ``file://`` file.""" if isinstance(part, DataPart): return part.data diff --git a/packages/agentenv-protocol/tests/test_website_conformance.py b/packages/agentenv-protocol/tests/test_website_conformance.py index 281561bd0..af03260b3 100644 --- a/packages/agentenv-protocol/tests/test_website_conformance.py +++ b/packages/agentenv-protocol/tests/test_website_conformance.py @@ -6,6 +6,7 @@ import datetime from pathlib import Path +import httpx import pytest from starlette.applications import Starlette from starlette.requests import Request @@ -17,7 +18,7 @@ from agentenv_protocol.website import conformance from agentenv_protocol.website.clock import UNIVERSE_NOW_HEADER, iso -CHECKS = ["card", "any_host", "health", "universe_now", "data_plane", "clock", "forwarded", "cookies"] +CHECKS = ["card", "any_host", "health", "universe_now", "reserved_paths", "data_plane", "clock", "forwarded", "cookies"] def statuses(results: list[conformance.Result]) -> dict[str, str]: @@ -36,6 +37,7 @@ def notes(self) -> list: env = Notes() env.store.load({"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]}) + env.store.mark_initial() return env @@ -56,10 +58,12 @@ def _handwritten(*, leak_host: bool = False, domain_cookie: bool = False, clock: clock_ops_first: tuple[str, ...] = (), leak_redirect: bool = False, stamp_rpc: bool = True, reset_keeps: bool = False, get_fails_after_reset: bool = False, rpc_reply: tuple[int, dict] | None = None, - ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None) -> Starlette: + ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None, + state_path: str | None = None, changed: bool = False) -> Starlette: """A server written without the SDK: the contract by hand, optionally broken. Its state is its ``state.data``; with ``ready_answer`` its card advertises ``website/v1``'s ``ready``, which answers it with ``ready_status``. With ``rpc_path`` its card's - ``url`` names its data plane there, not at ``/agentenv``.""" + ``url`` names its data plane there, not at ``/agentenv``, and with ``state_path`` its ``params.state.path`` names + where it serves its state, which shows changes since the initial state when ``changed``.""" state = {"items": ["a"], "url": None, "at": None, "reset": False} def now() -> datetime.datetime: @@ -94,6 +98,8 @@ async def card(request: Request) -> Response: "routes": [{"method": "GET", "path": "/items", "name": "items"}]}}] if ready_answer is not None: extensions[0]["params"]["methods"] = {"ready": {"method": "GET", "endpoint": "/ready"}} + if state_path is not None: + extensions[0]["params"]["state"] = {"path": state_path} if clock: extensions.append({"uri": "urn:agentenv:clock/v1", "params": { "endpoint": "/agentenv/ext/sync_time", "methods": { @@ -103,6 +109,9 @@ async def card(request: Request) -> Response: "capabilities": {"extensions": extensions, "operations": ["data/reset", "data/add", "data/get"]}})) + async def site_state(request: Request) -> Response: + return JSONResponse({"state_diff": {"items": {"added": ["b"]}} if changed else {}}) + async def rpc(request: Request) -> Response: body = await request.json() method = body["method"] @@ -132,10 +141,15 @@ async def sync_time(request: Request) -> Response: state["at"] = datetime.datetime.fromisoformat(virtual.replace("Z", "+00:00")) return stamp(JSONResponse({"gateway_virtual_time": virtual})) + async def unrouted(request: Request, exc: Exception) -> Response: + return stamp(JSONResponse({"error": "not found"}, status_code=getattr(exc, "status_code", 404))) + app = Starlette(routes=[Route("/", page), Route("/healthz", health), Route("/ready", ready), Route("/items", items), Route("/items/", items_slash), Route("/.well-known/agent-env.json", card), Route(rpc_path or "/agentenv", rpc, methods=["POST"]), - Route("/agentenv/ext/sync_time", sync_time, methods=["POST"])]) + Route("/agentenv/ext/sync_time", sync_time, methods=["POST"]), + *([Route(state_path, site_state)] if state_path else [])], + exception_handlers={404: unrouted, 405: unrouted}) app.state.data = state return app @@ -181,7 +195,7 @@ def test_an_answer_to_ready_that_isnt_a_website_readiness_fails_health_saying_wh def test_a_server_breaking_the_contract_fails_those_checks() -> None: results = statuses(conformance.check("http://testserver", client=TestClient(_handwritten(leak_host=True, domain_cookie=True, clock=False)))) - assert results["forwarded"] == results["cookies"] == results["clock"] == "failed" + assert results["forwarded"] == results["cookies"] == results["clock"] == "warning" assert results["card"] == results["health"] == results["data_plane"] == "passed" @@ -190,13 +204,13 @@ def test_the_forwarded_check_follows_no_redirect_and_reads_its_location(sdk_site assert forwarded.status == "passed" and "/api/notes/" in forwarded.detail leaked = {r.name: r for r in conformance.check("http://testserver", client=TestClient(_handwritten(leak_redirect=True)))}["forwarded"] - assert leaked.status == "failed" and "Location of /items/" in leaked.detail + assert leaked.status == "warning" and "Location of /items/" in leaked.detail def test_universe_now_is_checked_on_the_answers_of_every_check() -> None: result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(_handwritten(stamp_rpc=False)))}["universe_now"] - assert result.status == "failed" and "/agentenv" in result.detail + assert result.status == "warning" and "/agentenv" in result.detail def test_a_reset_that_clears_nothing_fails_the_data_plane_and_leaves_the_data() -> None: @@ -220,14 +234,28 @@ def test_a_data_request_without_a_2xx_json_rpc_result_fails_the_data_plane(reply assert result.status == "failed" and "no JSON-RPC result" in result.detail -def test_an_sdk_sites_data_round_trips_and_is_left_as_the_check_found_it_changed_or_not( +def test_an_sdk_site_keeps_its_initial_state_and_one_changed_since_skips_the_round_trip( + notes: WebsiteEnvironment, sdk_site: TestClient) -> None: + before = sdk_site.get("/go").json() + conformance.check("http://testserver", client=sdk_site) + assert sdk_site.get("/go").json() == before + notes.store.insert("notes", {"text": "new"}) + changed = sdk_site.get("/go").json() + assert changed["state_diff"] + result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] + assert result.status == "skipped" and "website/v1 state shows changes" in result.detail + assert sdk_site.get("/go").json() == changed + + +def test_a_named_session_with_changes_skips_the_round_trip_and_keeps_its_state( notes: WebsiteEnvironment, sdk_site: TestClient) -> None: - for _ in range(2): - before = notes.store.export() - result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] - assert result.status == "passed", result - assert notes.store.export() == before - notes.store.insert("notes", {"text": "new"}) + sdk_site.get("/api/notes", params={"sid": "task-1"}) + assert {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"].status == "passed" + notes.store.session("task-2")["notes"].append({"id": 2, "text": "draft"}) + result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] + assert result.status == "skipped" and "named sessions" in result.detail + assert sdk_site.get("/go", params={"sid": "task-2"}).json()["state_diff"]["notes"]["added"] == [ + {"id": 2, "text": "draft"}] def test_the_data_plane_is_reached_at_the_cards_url() -> None: @@ -243,22 +271,82 @@ def test_the_clock_check_calls_sync_time_whichever_operation_the_card_lists_firs assert results["clock"] == "passed", results -def test_a_server_with_no_card_health_or_header_fails_them() -> None: +@pytest.mark.parametrize(("changed", "status"), [(True, "skipped"), (False, "passed")]) +def test_a_card_without_state_is_read_at_the_state_path_it_names(changed: bool, status: str) -> None: + app = _handwritten(state_path="/grading/state", changed=changed) + result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] + assert result.status == status, result + assert ("/grading/state shows changes" in result.detail) is changed + + +def test_a_server_with_no_card_health_or_header_fails_the_contract_and_warns_of_the_rest() -> None: bare = Starlette(routes=[Route("/", lambda request: JSONResponse({"not": "html"}))]) - results = statuses(conformance.check("http://testserver", client=TestClient(bare))) - assert results["card"] == results["any_host"] == results["health"] == results["universe_now"] == "failed" - assert results["data_plane"] == results["clock"] == "failed" - assert results["cookies"] == "skipped" + results = conformance.check("http://testserver", client=TestClient(bare)) + assert statuses(results) == {"card": "warning", "any_host": "failed", "health": "failed", "universe_now": "warning", + "reserved_paths": "passed", "data_plane": "skipped", "clock": "warning", + "forwarded": "passed", "cookies": "skipped"} + assert [r.name for r in results if r.required] == ["any_host", "health", "reserved_paths", "data_plane"] + + +def test_a_bare_server_meeting_only_the_contract_passes_with_warnings() -> None: + async def page(request: Request) -> Response: + return HTMLResponse("hi") + + bare = Starlette(routes=[Route("/", page), Route("/api/health", lambda request: JSONResponse({"ok": True}))]) + results = conformance.check("http://testserver", client=TestClient(bare)) + assert not any(r.fails() for r in results) and any(r.fails(strict=True) for r in results) + assert {r.name for r in results if r.status == "warning"} == {"card", "universe_now", "clock"} + + +def test_a_spa_fallback_that_serves_the_reserved_paths_fails() -> None: + async def page(request: Request) -> Response: + return HTMLResponse("app") + + spa = Starlette(routes=[Route("/{path:path}", page)]) + reserved = {r.name: r for r in conformance.check("http://testserver", client=TestClient(spa))}["reserved_paths"] + assert reserved.status == "failed" and reserved.detail.startswith("/agentenv answered a page") + + +@pytest.mark.parametrize("exact", ["/agentenv", "/.well-known/agent-env.json"]) +def test_a_page_at_a_reserved_path_itself_fails(exact: str) -> None: + async def page(request: Request) -> Response: + return HTMLResponse("app") + + server = Starlette(routes=[Route("/", page), Route(exact, page)]) + reserved = {r.name: r for r in conformance.check("http://testserver", client=TestClient(server))}["reserved_paths"] + assert reserved.status == "failed" and reserved.detail.startswith(f"{exact} answered a page") def test_main_prints_each_check_and_exits_by_outcome(monkeypatch, capsys) -> None: - monkeypatch.setattr(conformance, "check", lambda url, **options: [conformance.Result("card", "passed"), - conformance.Result("clock", "failed", "late")]) - assert conformance.main(["http://site"]) == 1 + results = [conformance.Result("health", "passed"), conformance.Result("clock", "warning", "late", required=False)] + monkeypatch.setattr(conformance, "check", lambda url, **options: results) + assert conformance.main(["http://site"]) == 0 printed = capsys.readouterr().out - assert "PASS card" in printed and "FAIL clock: late" in printed + assert "PASS health (required)" in printed and "WARN clock (recommended): late" in printed + assert "1 passed, 0 failed, 1 warning(s), 0 skipped" in printed + assert conformance.main(["http://site", "--strict"]) == 1 + results.append(conformance.Result("any_host", "failed", "json")) + assert conformance.main(["http://site"]) == 1 + assert "FAIL any_host (required): json" in capsys.readouterr().out def test_main_exits_2_when_the_server_cannot_be_reached(capsys) -> None: assert conformance.main(["http://127.0.0.1:9", "--timeout", "2"]) == 2 assert "couldn't reach" in capsys.readouterr().err + + +def test_the_cookie_check_holds_the_session_path_to_the_cookie_the_card_names(sdk_site: TestClient) -> None: + card = sdk_site.get("/.well-known/agent-env.json").json() + for extension in card["capabilities"]["extensions"]: + if extension["uri"] == "urn:agentenv:website/v1": + extension["params"]["session_cookie"] = "agentenv_session_other" + real = sdk_site.get + + def get(url, **kwargs): + if url.endswith("/.well-known/agent-env.json"): + return httpx.Response(200, json=card, request=httpx.Request("GET", url)) + return real(url, **kwargs) + + sdk_site.get = get + cookies = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["cookies"] + assert cookies.status == "warning" and "not the card's agentenv_session_other" in cookies.detail diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py new file mode 100644 index 000000000..1b12d96de --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -0,0 +1,213 @@ +"""A site of ``@website(models=...)``: its state is a database's, SQLAlchemy models' tables on DATABASE_URL (sqlite +here) or an object's own load/export/reset, which its data plane and /go read and write, and its routes reach +through ``self.db``.""" + +from __future__ import annotations + +import datetime +import sys +import types +from pathlib import Path + +import pytest +from starlette.testclient import TestClient + +from agentenv_protocol.website import ApiContext, WebsiteEnvironment, api, conformance, website + +sqlalchemy = pytest.importorskip("sqlalchemy") +from sqlalchemy import DateTime, ForeignKey, Integer, String, select # noqa: E402 +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column # noqa: E402 + +RPC = "/agentenv" + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = "users" + id: Mapped[str] = mapped_column(String, primary_key=True) + name: Mapped[str] = mapped_column(String) + + +class Ticket(Base): + __tablename__ = "tickets" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + owner: Mapped[str] = mapped_column(ForeignKey("users.id")) + title: Mapped[str] = mapped_column(String) + opened_at: Mapped[datetime.datetime] = mapped_column(DateTime) + + +SEED = {"current_user": "ann", "users": [{"id": "ann", "name": "Ann"}], + "tickets": [{"id": 1, "owner": "ann", "title": "printer", "opened_at": "2031-01-02T03:04:05"}]} + + +def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}}).json() + assert "error" not in body, body + return body["result"] + + +@pytest.fixture +def dist(tmp_path: Path) -> Path: + (tmp_path / "index.html").write_text("desk") + return tmp_path + + +@pytest.fixture +def database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> str: + url = f"sqlite:///{tmp_path / 'service.db'}" + monkeypatch.setenv("DATABASE_URL", url) + return url + + +def helpdesk(dist: Path, models: object) -> type[WebsiteEnvironment]: + @website(domains=["desk.example"], static=str(dist), models=models) + class Helpdesk(WebsiteEnvironment): + @api.get("/api/tickets") + def tickets(self, ctx: ApiContext) -> list: + with self.db() as session: + return [{"id": t.id, "title": t.title} for t in session.scalars( + select(Ticket).where(Ticket.owner == ctx.user).order_by(Ticket.id))] + + @api.post("/api/tickets", status=201) + def open_ticket(self, body: dict, ctx: ApiContext) -> dict: + with self.db.begin() as session: + ticket = Ticket(owner=ctx.user, title=body["title"], opened_at=self.now().replace(tzinfo=None)) + session.add(ticket) + session.flush() + return {"id": ticket.id} + + return Helpdesk + + +def test_the_data_plane_loads_exports_and_resets_the_models_tables(dist: Path, database: str) -> None: + env = helpdesk(dist, Base)() + client = TestClient(env.create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + assert client.get("/api/session").json()["user"] == "ann" + assert client.get("/api/tickets").json() == [{"id": 1, "title": "printer"}] + rpc(client, "data/reset") + assert rpc(client, "data/get")["parts"][0]["data"] == {"users": [], "tickets": []} + + +def test_rows_that_leave_an_optional_column_out_load_as_given_in_either_order(dist: Path, database: str) -> None: + class Threads(DeclarativeBase): + pass + + class Note(Threads): + __tablename__ = "notes" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + parent_id: Mapped[int | None] = mapped_column(ForeignKey("notes.id"), nullable=True) + + client = TestClient(helpdesk(dist, Threads)().create_app()) + for seed in ([{"id": 1}, {"id": 2, "parent_id": 1}, {"id": 3, "parent_id": 1}, {"id": 4}], + [{"id": 1, "parent_id": None}, {"id": 2}, {"id": 3, "parent_id": 2}]): + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"notes": seed}}]}) + assert rpc(client, "data/get")["parts"][0]["data"]["notes"] == [{"parent_id": None, **row} for row in seed] + + +def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, database: str) -> None: + env = helpdesk(dist, Base)() + client = TestClient(env.create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) + assert client.post("/api/tickets", json={"title": "badge"}).json() == {"id": 2} + server = sqlalchemy.create_engine(database) + with server.connect() as connection: + assert connection.execute(select(Ticket.title).order_by(Ticket.id)).scalars().all() == ["printer", "badge"] + connection.execute(sqlalchemy.update(Ticket).where(Ticket.id == 1).values(title="printer jam")) + connection.commit() + state = client.get("/go", params={"sid": "default"}).json() + assert state["initial_state"] == SEED + diff = state["state_diff"]["tickets"] + assert [t["title"] for t in diff["added"]] == ["badge"] and diff["removed"] == [] + assert diff["changed"][0]["before"]["title"] == "printer" and diff["changed"][0]["after"]["title"] == "printer jam" + assert client.get("/go", params={"sid": "other"}).json() == state + + +def test_a_models_site_passes_the_conformance_check(dist: Path, database: str) -> None: + client = TestClient(helpdesk(dist, Base)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) + results = conformance.check("http://testserver", client=client) + assert all(r.status == "passed" for r in results), results + assert rpc(client, "data/get")["parts"][0]["data"] == SEED + + +def test_models_may_be_a_metadata_or_a_module_and_refuse_anything_else(dist: Path, database: str) -> None: + module = types.ModuleType("helpdesk_models") + module.Base, module.Ticket = Base, Ticket + for models in (Base.metadata, module): + assert helpdesk(dist, models)()._tables.metadata is Base.metadata + with pytest.raises(TypeError, match="models="): + helpdesk(dist, object()) + with pytest.raises(ValueError, match="data/add"): + website(models=Base, data="seed.json") + + +def test_unknown_tables_and_a_missing_database_url_are_refused(dist: Path, monkeypatch: pytest.MonkeyPatch, + database: str) -> None: + client = TestClient(helpdesk(dist, Base)().create_app()) + body = client.post(RPC, json={"jsonrpc": "2.0", "id": 1, "method": "data/add", + "params": {"parts": [{"kind": "data", "data": {"orders": []}}]}}).json() + assert "error" in body + monkeypatch.delenv("DATABASE_URL") + with pytest.raises(RuntimeError, match="DATABASE_URL"): + helpdesk(dist, Base)().db() + + +def test_a_database_url_whose_driver_isnt_installed_names_it(dist: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "psycopg2", None) + monkeypatch.setitem(sys.modules, "psycopg", None) + monkeypatch.setenv("DATABASE_URL", "postgresql://desk@localhost/service") + with pytest.raises(ImportError, match=r"postgresql:// needs the database driver psycopg, .* carries psycopg"): + helpdesk(dist, Base)().db() + + +def test_an_object_with_its_own_hooks_is_the_sites_data(dist: Path) -> None: + class Rows: + def __init__(self) -> None: + self.rows: list = [] + + def load(self, data: dict) -> None: + self.rows += data["rows"] + + async def export(self) -> dict: + return {"rows": list(self.rows)} + + def reset(self) -> None: + self.rows.clear() + + rows = Rows() + env = helpdesk(dist, rows)() + client = TestClient(env.create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"current_user": "bo", "rows": [{"id": 1}]}}]}) + rows.rows.append({"id": 2}) + assert rpc(client, "data/get")["parts"][0]["data"] == {"current_user": "bo", "rows": [{"id": 1}, {"id": 2}]} + assert client.get("/go").json()["state_diff"] == {"rows": {"added": [{"id": 2}], "removed": [], "changed": []}} + assert env.acting_user() == "bo" + assert env.db is None + + +def test_editing_a_row_a_hook_still_owns_leaves_the_initial_state(dist: Path) -> None: + class Rows: + def __init__(self) -> None: + self.rows: list = [] + + def load(self, data: dict) -> None: + self.rows += data["rows"] + + def export(self) -> dict: + return {"rows": list(self.rows)} + + def reset(self) -> None: + self.rows.clear() + + rows = Rows() + client = TestClient(helpdesk(dist, rows)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"rows": [{"id": 1, "status": "open"}]}}]}) + rows.rows[0]["status"] = "closed" + assert client.get("/go").json()["state_diff"] == {"rows": {"added": [], "removed": [], "changed": [ + {"id": 1, "before": {"id": 1, "status": "open"}, "after": {"id": 1, "status": "closed"}}]}} diff --git a/packages/agentenv-protocol/tests/test_website_environment.py b/packages/agentenv-protocol/tests/test_website_environment.py index 05c118591..e2ccfccbb 100644 --- a/packages/agentenv-protocol/tests/test_website_environment.py +++ b/packages/agentenv-protocol/tests/test_website_environment.py @@ -37,6 +37,7 @@ from agentenv_protocol.website.clock import parse_time RPC = "/agentenv" +COOKIE = "agentenv_session_Shop" def rpc(client: TestClient, method: str, params: dict | None = None) -> dict: @@ -238,6 +239,39 @@ def test_a_sites_data_is_read_through_the_data_plane_alone(client: TestClient) - assert "state" not in website_params and "state" not in website_params.get("methods", {}) +def test_each_sid_keeps_a_state_of_its_own_started_from_the_initial_state(client: TestClient, + shop: WebsiteEnvironment) -> None: + client.post("/api/orders", json={"sku": "mug"}) + assert TestClient(shop.asgi).get("/go", params={"sid": "task-7"}).json() == { + "initial_state": SEED, "current_state": SEED, "state_diff": {}, "changed_sessions": []} + page = client.get("/", params={"sid": "task-7"}) + assert page.headers["set-cookie"].startswith("agentenv_sid_Shop=task-7;") + assert "set-cookie" not in client.get("/", params={"sid": "task-7"}).headers + client.post("/api/orders", json={"sku": "pen"}) + client.post("/api/orders", params={"sid": "other"}, json={"sku": "cup"}) + added = {sid: [o.get("sku") for d in [client.get("/go", params={"sid": sid}).json()["state_diff"]] + for o in d.get("orders", {}).get("added", [])] for sid in ("default", "task-7", "other")} + assert added == {"default": ["mug"], "task-7": ["pen"], "other": ["cup"]} + assert client.get("/go").json()["state_diff"]["orders"]["added"][0]["sku"] == "cup" + client.get("/", params={"sid": "default"}) + assert client.get("/go").json()["state_diff"]["orders"]["added"][0]["sku"] == "mug" + assert rpc(client, "data/get")["parts"][0]["data"]["orders"][-1]["sku"] == "mug" + assert sorted(shop._store.sessions()) == ["other", "task-7"] + assert sorted(client.get("/go").json()["changed_sessions"]) == ["other", "task-7"] + rpc(client, "data/reset") + assert shop._store.sessions() == [] and client.get("/go", params={"sid": "task-7"}).json()["current_state"] == {} + + +def test_a_session_id_is_sanitised_as_cua_gym_sites_do() -> None: + from agentenv_protocol.website.store import session_id + + assert [session_id(v) for v in (None, "", "default", "../a b", "x" * 200)] == [None, None, None, "ab", "x" * 128] + store = Collections({"items": [{"id": 1}]}) + store.session("s")["items"].append({"id": 2}) + assert store.session("default") is store and store.export() == {"items": [{"id": 1}]} + assert store.session("s").diff() == {"items": {"added": [{"id": 2}], "removed": [], "changed": []}} + + def test_broad_routes_leave_the_built_in_ones_answering_their_paths(dist: Path) -> None: class Broad(site(dist)): @api.get("/{slug}") @@ -291,7 +325,7 @@ def test_the_acting_user_is_signed_in_until_a_session_says_else(client: TestClie first = client.get("/api/session") assert first.json() == {"user": "alice", "role": "default", "signed_in": True} cookie = first.headers["set-cookie"] - assert cookie.startswith(f"{SESSION_COOKIE}=") and "domain" not in cookie.lower() and "httponly" in cookie.lower() + assert cookie.startswith(f"{COOKIE}=") and "domain" not in cookie.lower() and "httponly" in cookie.lower() client.post("/api/sign-in", json={"user": "bob"}) assert client.get("/api/orders").json() == [{"id": 1, "user": "bob"}] assert client.get("/api/session").json()["user"] == "bob" @@ -299,7 +333,7 @@ def test_the_acting_user_is_signed_in_until_a_session_says_else(client: TestClie assert client.get("/api/session").json()["user"] is None # The jar holds the site's signed-out cookie under its own domain; a second cookie of the name would race it. client.cookies.clear() - client.cookies.set(SESSION_COOKIE, "forged.signature") + client.cookies.set(COOKIE, "forged.signature") assert client.get("/api/session").json()["user"] == "alice" @@ -317,11 +351,65 @@ def test_the_session_operation_mints_cookies_for_a_user_or_signed_out(shop: Webs minted = client.post("/agentenv/ext/website/session", json={"user": "bob"}).json() assert minted["user"] == "bob" and minted["expires_at"].endswith("Z") (cookie,) = minted["cookies"] - assert cookie["name"] == SESSION_COOKIE and "domain" not in cookie and cookie["path"] == "/" - browser = TestClient(shop.asgi, cookies={SESSION_COOKIE: cookie["value"]}) + assert cookie["name"] == COOKIE and "domain" not in cookie and cookie["path"] == "/" + browser = TestClient(shop.asgi, cookies={COOKIE: cookie["value"]}) assert browser.get("/api/session").json()["user"] == "bob" signed_out = client.post("/agentenv/ext/website/session", json={}).json()["cookies"][0]["value"] - assert TestClient(shop.asgi, cookies={SESSION_COOKIE: signed_out}).get("/api/session").json()["user"] is None + assert TestClient(shop.asgi, cookies={COOKIE: signed_out}).get("/api/session").json()["user"] is None + + +def test_each_site_keeps_its_own_session_cookie_and_reads_the_legacy_one(dist: Path, monkeypatch) -> None: + monkeypatch.setenv("ENVIRONMENT_NAME", "shop a/1") + first = site(dist)() + monkeypatch.setenv("ENVIRONMENT_NAME", "shop-b") + second = site(dist)() + for env in (first, second): + env.store.load(SEED) + named = "agentenv_session_shop_a_1~030f5785d67a" + assert (first.session_cookie, second.session_cookie) == (named, "agentenv_session_shop-b") + browser = TestClient(first.create_app()) + browser.post("/api/sign-in", json={"user": "bob"}) + other = TestClient(second.create_app(), cookies=browser.cookies) + other.post("/api/sign-in", json={"user": "alice"}) + assert set(other.cookies.keys()) == {named, "agentenv_session_shop-b"} + assert TestClient(first.asgi, cookies=other.cookies).get("/api/session").json()["user"] == "bob" + card = browser.get("/.well-known/agent-env.json").json() + assert _extension(card, WEBSITE_EXTENSION_URI)["params"]["session_cookie"] == named + legacy = first._session_signer().encode("bob")[0] + assert TestClient(first.asgi, cookies={SESSION_COOKIE: legacy}).get("/api/orders").json() == [{"id": 1, "user": "bob"}] + + +def test_no_site_s_session_choice_is_another_site_s_sign_in(dist: Path, monkeypatch) -> None: + monkeypatch.setenv("ENVIRONMENT_NAME", "notes") + notes = site(dist)() + monkeypatch.setenv("ENVIRONMENT_NAME", "notes_sid") + notes_sid = site(dist)() + browser = TestClient(notes.create_app()) + browser.post("/api/orders", params={"sid": "task-1"}, json={"sku": "mug"}) + other = TestClient(notes_sid.create_app(), cookies=browser.cookies) + other.post("/api/sign-in", json={"user": "bob"}) + back = TestClient(notes.asgi, cookies=other.cookies) + assert [o["sku"] for o in back.get("/go").json()["current_state"]["orders"]] == ["mug"] + assert TestClient(notes_sid.asgi, cookies=back.cookies).get("/api/session").json()["user"] == "bob" + + +def test_sites_whose_names_spell_alike_keep_their_own_cookies(dist: Path, monkeypatch) -> None: + apps = [] + for name in ("shop a/1", "shop_a_1"): + monkeypatch.setenv("ENVIRONMENT_NAME", name) + env = site(dist)() + env.store.load(SEED) + apps.append(env.create_app()) + browser = TestClient(apps[0]) + browser.post("/api/sign-in", json={"user": "bob"}) + browser.get("/", params={"sid": "task-1"}) + other = TestClient(apps[1], cookies=browser.cookies) + other.post("/api/sign-in", json={"user": "alice"}) + other.get("/", params={"sid": "task-2"}) + assert len(other.cookies.keys()) == 4 + back = TestClient(apps[0], cookies=other.cookies) + assert back.get("/api/session").json()["user"] == "bob" + assert back.cookies.get("agentenv_sid_shop_a_1~030f5785d67a") == "task-1" def test_every_answer_carries_the_universe_time(client: TestClient) -> None: @@ -335,13 +423,16 @@ def test_every_answer_carries_the_universe_time(client: TestClient) -> None: class _Clock: """A gateway-shaped /clock/time whose time a test sets.""" - def __init__(self, virtual: str) -> None: - self.virtual = virtual + def __init__(self, virtual: str, rate: object = None) -> None: + self.virtual, self.rate = virtual, rate outer = self class Handler(BaseHTTPRequestHandler): def do_GET(self) -> None: - body = json.dumps({"virtual_time": outer.virtual}).encode() + answer = {"virtual_time": outer.virtual} + if outer.rate is not None: + answer["virtual_seconds_per_real_second"] = outer.rate + body = json.dumps(answer).encode() self.send_response(200) self.send_header("Content-Length", str(len(body))) self.end_headers() @@ -544,8 +635,8 @@ async def _clock_rates(monkeypatch) -> None: real, virtual = [0.0], [datetime.datetime(2031, 1, 1, tzinfo=datetime.timezone.utc)] - async def read(url: str) -> datetime.datetime: - return virtual[0] + async def read(url: str) -> tuple[datetime.datetime, None]: + return virtual[0], None monkeypatch.setattr(clock_module.time, "monotonic", lambda: real[0]) monkeypatch.setattr(clock_module, "_read", read) @@ -573,6 +664,39 @@ def tick(seconds: float, virtual_seconds: float) -> None: assert clock._rate == 100.0 and clock.now() == virtual[0] +def test_the_clock_extrapolates_at_the_rate_the_clock_answers_with(monkeypatch) -> None: + asyncio.run(_answered_rates(monkeypatch)) + + +async def _answered_rates(monkeypatch) -> None: + from agentenv_protocol.website import UniverseClock + + gateway = _Clock("2031-01-01T00:00:00Z", rate=60) + try: + assert await clock_module._read(gateway.url) == (parse_time("2031-01-01T00:00:00Z"), 60.0) + for unusable in ("fast", True, -1, float("inf")): + gateway.rate = unusable + assert (await clock_module._read(gateway.url))[1] is None + finally: + gateway.close() + real, answers = [0.0], [(parse_time("2031-01-01T00:00:00Z"), 60.0)] + + async def read(url: str) -> tuple[datetime.datetime, float | None]: + return answers[0] + + monkeypatch.setattr(clock_module.time, "monotonic", lambda: real[0]) + monkeypatch.setattr(clock_module, "_read", read) + clock = UniverseClock() + await clock.sync("http://gateway/clock/time") + real[0] += 0.5 + assert clock._rate == 60.0 and clock.now() == answers[0][0] + datetime.timedelta(seconds=30) + real[0] += 0.5 + answers[0] = (parse_time("2031-02-01T00:00:00Z"), 0.0) + await clock.refresh() + real[0] += 10 + assert clock._rate == 0.0 and clock.now() == answers[0][0] + + def test_a_refresh_of_the_old_clock_never_overwrites_a_sync_to_a_new_one(monkeypatch) -> None: asyncio.run(_refresh_during_sync(monkeypatch)) @@ -583,12 +707,12 @@ async def _refresh_during_sync(monkeypatch) -> None: old, new = (datetime.datetime(year, 1, 1, tzinfo=datetime.timezone.utc) for year in (2031, 2040)) release, held = asyncio.Event(), [] - async def read(url: str) -> datetime.datetime: + async def read(url: str) -> tuple[datetime.datetime, None]: if url == "http://old/clock/time": if held: await release.wait() - return old - return new + return old, None + return new, None monkeypatch.setattr(clock_module, "_read", read) monkeypatch.setattr(clock_module, "REFRESH_S", 0.0) diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py index 4d7110657..93eb51cc1 100644 --- a/packages/agentenv-protocol/tests/test_website_notes_example.py +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -18,7 +18,7 @@ from starlette.responses import PlainTextResponse from starlette.testclient import TestClient -from agentenv_protocol.website import (ROLE_KEY_ENV, ROLE_KEY_HEADER, SESSION_COOKIE, UNIVERSE_NOW_HEADER, WebsiteEnvironment, +from agentenv_protocol.website import (ROLE_KEY_ENV, ROLE_KEY_HEADER, UNIVERSE_NOW_HEADER, WebsiteEnvironment, conformance, website) NOTES_DIR = Path(__file__).resolve().parents[1] / "examples" / "website" / "notes" @@ -104,7 +104,8 @@ def gateway_clock(): def test_the_example_passes_every_conformance_check(client: TestClient) -> None: results = conformance.check("http://testserver", client=client) assert {r.name: r.status for r in results} == dict.fromkeys( - ["card", "any_host", "health", "universe_now", "data_plane", "clock", "forwarded", "cookies"], "passed"), results + ["card", "any_host", "health", "universe_now", "reserved_paths", "data_plane", "clock", "forwarded", + "cookies"], "passed"), results def test_the_card_declares_the_site_its_health_path_and_its_routes_by_name(client: TestClient) -> None: @@ -121,7 +122,7 @@ def test_the_card_declares_the_site_its_health_path_and_its_routes_by_name(clien def test_a_browser_is_signed_in_as_the_acting_user_and_can_sign_in_as_another(client: TestClient) -> None: session = client.get("/api/session") assert session.json() == {"user": "ann", "role": "default", "signed_in": True} - assert SESSION_COOKIE in session.cookies + assert "agentenv_session_notes" in session.cookies assert [n["title"] for n in client.get("/api/notes").json()] == ["Buy milk", "Book dentist"] assert client.post("/api/sign-in", json={"user": "ben"}).json() == {"user": "ben"} assert client.get("/api/session").json()["user"] == "ben" @@ -195,6 +196,15 @@ def test_the_seed_loads_exports_and_resets_through_the_data_plane(client: TestCl assert next(note for note in toggled if note["id"] == 1)["done"] is True + +def test_a_page_opened_at_a_sid_keeps_that_sessions_notes_apart(client: TestClient, env: WebsiteEnvironment) -> None: + other = TestClient(env.asgi) + other.get("/", params={"sid": "task-1"}) + created = other.post("/api/notes", json={"title": "only in task-1"}).json() + assert client.get("/go", params={"sid": "task-1"}).json()["state_diff"]["notes"]["added"] == [created] + assert client.get("/go", params={"sid": "default"}).json()["state_diff"] == {} + assert all(note["title"] != "only in task-1" for note in client.get("/api/notes").json()) + GATEWAY_PREFIX = "/website/notes" diff --git a/packages/agentenv-protocol/tests/test_website_state.py b/packages/agentenv-protocol/tests/test_website_state.py new file mode 100644 index 000000000..b4e4ec882 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_website_state.py @@ -0,0 +1,132 @@ +"""What a website env's task state is: WebsiteState, the SDK's website/v1 state, and website_state and website_sites +reading it through cards.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from unittest.mock import patch + +import httpx +import pytest +from pydantic import ValidationError +from starlette.testclient import TestClient + +from agentenv_protocol import DataPart +from agentenv_protocol.agent_env_environment import get_data +from agentenv_protocol.client import find_extension_method +from agentenv_protocol.website import ( + STATE_PATH, + WEBSITE_EXTENSION_URI, + WEBSITE_STATE_OP, + WebsiteEnvironment, + WebsiteState, + api, + website, + website_sites, + website_state, +) + +_GW = "http://gw" + + +def _site_card(name: str, *, state: bool = True) -> dict: + methods = {"state": {"method": "GET", "endpoint": f"/svc/{name}/agentenv/ext/website/state"}} if state else {} + return {"name": name, "url": f"/svc/{name}/agentenv", "capabilities": {"extensions": [ + {"uri": WEBSITE_EXTENSION_URI, "params": {"methods": methods}}]}} + + +def _routed(handler): + real = httpx.AsyncClient + return patch.object(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handler))) + + +@pytest.fixture +def shop(tmp_path: Path) -> WebsiteEnvironment: + (tmp_path / "index.html").write_text("") + + @website(domains=["shop.test"], static=str(tmp_path)) + class Shop(WebsiteEnvironment): + @api.post("/api/orders") + def place(self, order: dict) -> dict: + return self.store.insert("orders", order) + + env = Shop() + env.store.load({"orders": [{"id": 1, "sku": "pen"}]}) + env.store.mark_initial() + return env + + +def test_a_website_state_ignores_fields_it_doesnt_know_and_refuses_a_state_that_isnt_an_object() -> None: + assert WebsiteState.model_validate({"current_state": {"a": 1}, "seen_at": "now"}).model_dump() == { + "initial_state": None, "current_state": {"a": 1}, "state_diff": None} + with pytest.raises(ValidationError): + WebsiteState.model_validate({"current_state": [1]}) + + +def test_an_sdk_site_advertises_state_and_answers_its_stores_state_there_and_at_the_path_it_names( + shop: WebsiteEnvironment) -> None: + client = TestClient(shop.create_app()) + card = client.get("/.well-known/agent-env.json").json() + op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) + params = next(e for e in card["capabilities"]["extensions"] if e["uri"] == WEBSITE_EXTENSION_URI)["params"] + assert (op["method"], op["endpoint"], params["state"]) == ("GET", "/agentenv/ext/website/state", {"path": STATE_PATH}) + client.post("/api/orders", json={"sku": "mug"}) + answer = client.get(op["endpoint"], params={"sid": "default"}).json() + assert answer == client.get(STATE_PATH).json() == {**shop.store.state(), "changed_sessions": []} + assert answer["state_diff"]["orders"]["added"][0]["sku"] == "mug" + + +def test_an_sdk_sites_state_op_answers_the_session_its_sid_names(shop: WebsiteEnvironment) -> None: + client = TestClient(shop.create_app()) + client.post("/api/orders", params={"sid": "task-1"}, json={"sku": "mug"}) + named = client.get("/agentenv/ext/website/state", params={"sid": "task-1"}).json() + assert named == client.get(STATE_PATH, params={"sid": "task-1"}).json() + assert named["state_diff"]["orders"]["added"][0]["sku"] == "mug" and named["changed_sessions"] == ["task-1"] + default = client.get("/agentenv/ext/website/state").json() + assert WebsiteState.model_validate(default).state_diff == shop.store.state()["state_diff"] + assert default["current_state"]["orders"] == [{"id": 1, "sku": "pen"}] + + +def test_a_site_with_data_hooks_of_its_own_advertises_no_state(tmp_path: Path) -> None: + @website(domains=["own.test"]) + class Own(WebsiteEnvironment): + @get_data + def export(self) -> list: + return [DataPart(data={"mine": True})] + + client = TestClient(Own().create_app()) + card = client.get("/.well-known/agent-env.json").json() + params = next(e for e in card["capabilities"]["extensions"] if e["uri"] == WEBSITE_EXTENSION_URI)["params"] + assert find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) is None and "state" not in params + assert client.get("/agentenv/ext/website/state").status_code == 404 + + +def test_website_state_asks_the_cards_state_of_the_session_with_the_headers_given() -> None: + seen = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append((request.url.path, dict(request.url.params), request.headers.get("authorization"))) + return httpx.Response(200, json={"initial_state": {}, "current_state": {"n": 1}, "state_diff": {"n": {}}}) + + with _routed(handler): + state = asyncio.run(website_state(_GW, _site_card("mail"), sid="s1", headers={"Authorization": "Bearer t"})) + assert asyncio.run(website_state(_GW, _site_card("mail", state=False))) is None + assert state == WebsiteState(initial_state={}, current_state={"n": 1}, state_diff={"n": {}}) + assert seen == [("/svc/mail/agentenv/ext/website/state", {"sid": "s1"}, "Bearer t")] + + +def test_an_answer_that_isnt_a_website_state_is_refused() -> None: + with _routed(lambda request: httpx.Response(200, json={"current_state": "full"})): + with pytest.raises(ValueError): + asyncio.run(website_state(_GW, _site_card("mail"))) + + +def test_website_sites_are_a_single_sites_own_card_or_a_composed_cards_website_children() -> None: + mail, calendar = _site_card("mail"), _site_card("calendar", state=False) + mcp = {"name": "mail", "url": "/svc/mcp-mail/agentenv", "capabilities": {"extensions": []}} + composed = {"name": "gw", "children_environments": [mail, mcp, calendar, "junk"]} + assert [(site.name, site.url) for site in website_sites(composed)] == [ + ("mail", "/svc/mail/agentenv"), ("calendar", "/svc/calendar/agentenv")] + assert [site.card for site in website_sites(mail)] == [mail] + assert website_sites({"name": "plain", "children_environments": [mcp]}) == [] diff --git a/src/agent_env/env/gateway/clock.py b/src/agent_env/env/gateway/clock.py index cceb14364..dda1dd7eb 100644 --- a/src/agent_env/env/gateway/clock.py +++ b/src/agent_env/env/gateway/clock.py @@ -217,11 +217,13 @@ def at_elapsed(self, elapsed: float) -> datetime | None: return _advance(self._t0, elapsed - self._lived + self._ahead) def read(self) -> dict: - """The frozen consumer contract: {"virtual_time": ""} — pure read, never advances.""" - current = self.now() - if current is None: - raise ClockError("clock is not armed") - return {"virtual_time": _iso(current)} + """The frozen consumer contract: {"virtual_time": ""} — pure read, never advances — plus the rate it + runs at, "virtual_seconds_per_real_second", so a reader extrapolates between reads without guessing it.""" + with self._lock: + if not self._armed or self._t0 is None: + raise ClockError("clock is not armed") + current = _advance(self._t0, (self._now() - self._anchor) * self._rate + self._ahead) + return {"virtual_time": _iso(current), "virtual_seconds_per_real_second": self._rate} def state(self) -> dict: with self._lock: diff --git a/tst/unit/env/gateway/clock_advance_test.py b/tst/unit/env/gateway/clock_advance_test.py index 0a76031f9..53bdbfbab 100644 --- a/tst/unit/env/gateway/clock_advance_test.py +++ b/tst/unit/env/gateway/clock_advance_test.py @@ -165,3 +165,22 @@ def test_route_refuses_a_backwards_advance(gateway): gateway._clock.set_time("2026-01-04T08:00:00Z", rate=0) resp = _put(gateway, "/clock/advance", {"virtual_time": T0}) assert resp.status_code == 400 and "only moves forward" in resp.json()["error"] + + +def _get(gw: Gateway, path: str) -> httpx.Response: + async def _do() -> httpx.Response: + transport = httpx.ASGITransport(app=gw._mcp.streamable_http_app()) + async with httpx.AsyncClient(transport=transport, base_url="http://gateway") as client: + return await client.get(path) + + return asyncio.run(_do()) + + +def test_clock_time_answers_the_rate_beside_the_time(gateway): + assert _get(gateway, "/clock/time").status_code == 404 + gateway._clock.set_time(T0, rate=60) + gateway._clock._now.tick(2) + assert _get(gateway, "/clock/time").json() == {"virtual_time": "2026-01-01T00:02:00Z", + "virtual_seconds_per_real_second": 60.0} + gateway._clock.advance("2026-01-02T00:00:00Z") + assert _get(gateway, "/clock/time").json()["virtual_seconds_per_real_second"] == 60.0 diff --git a/uv.lock b/uv.lock index f0342e1ea..241ad385f 100644 --- a/uv.lock +++ b/uv.lock @@ -298,6 +298,10 @@ computer = [ { name = "pillow" }, { name = "uvicorn" }, ] +db = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] dev = [ { name = "a2a-sdk", extra = ["http-server"] }, { name = "av", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, @@ -309,6 +313,7 @@ dev = [ { name = "pytest-asyncio" }, { name = "python-multipart" }, { name = "regex" }, + { name = "sqlalchemy" }, ] [package.metadata] @@ -325,6 +330,7 @@ requires-dist = [ { name = "mcp", marker = "extra == 'dev'", specifier = ">=1.25,<2" }, { name = "pillow", marker = "extra == 'computer'", specifier = ">=10.4" }, { name = "pillow", marker = "extra == 'dev'", specifier = ">=10.4" }, + { name = "psycopg", extras = ["binary"], marker = "extra == 'db'", specifier = ">=3.1" }, { name = "pydantic", specifier = ">=2.11" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=9.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, @@ -332,12 +338,14 @@ requires-dist = [ { name = "python-multipart", marker = "extra == 'dev'", specifier = ">=0.0.13" }, { name = "regex", marker = "extra == 'agent'", specifier = ">=2023.12.25" }, { name = "regex", marker = "extra == 'dev'", specifier = ">=2023.12.25" }, + { name = "sqlalchemy", marker = "extra == 'db'", specifier = ">=2.1" }, + { name = "sqlalchemy", marker = "extra == 'dev'", specifier = ">=2.1" }, { name = "starlette", specifier = ">=0.37" }, { name = "uvicorn", marker = "extra == 'agent'", specifier = ">=0.30" }, { name = "uvicorn", marker = "extra == 'app'", specifier = ">=0.30" }, { name = "uvicorn", marker = "extra == 'computer'", specifier = ">=0.30" }, ] -provides-extras = ["agent", "app", "computer", "dev"] +provides-extras = ["agent", "app", "computer", "db", "dev"] [[package]] name = "aiofile" @@ -5004,6 +5012,86 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, ] +[[package]] +name = "psycopg" +version = "3.3.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/a3/55c9fb649391fddec8d7ba3005162cd76e8cb3b2eea3dc922684aa0013e4/psycopg-3.3.7.tar.gz", hash = "sha256:6574f23fd79bf129ee0616b49f6e358faf7ae1bf3a362d2ad599cf8f9c3d4935", size = 168912, upload-time = "2026-11-03T03:58:07.876Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/2a/2e05cc01b2817b8baff419823774bb928efd0a36de12da78f19482cfa329/psycopg-3.3.7-py3-none-any.whl", hash = "sha256:d592e88a077e01ff57f22d59f16e828b92f2b2d9f439b6679d582b48d74cf306", size = 216207, upload-time = "2026-11-03T03:51:21.09Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.7" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/19/07272d51290a502281fa3f54e76ef95fa72bd05abd913a57f033162d7768/psycopg_binary-3.3.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4f9ec0cc3b327f8ed1e2ba40441dd843be9c82b3ac5dfe75eb00ece02a454ec3", size = 4731229, upload-time = "2026-11-03T03:52:28.922Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a3/beb642b331f1611f7f87c75c5b0803bfa779f6ccd4e79e381961484a20b2/psycopg_binary-3.3.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0734da871182afdefcc9b3fbaf13c70408f47a2de64394208606d8f45c4027ec", size = 4779422, upload-time = "2026-11-03T03:52:34.545Z" }, + { url = "https://files.pythonhosted.org/packages/be/5d/41538a43e0364b8f8fc669056f8f2eeb884a7374c16f65695ee3868113af/psycopg_binary-3.3.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:05fa93c416af799f8d0db325e4a58ed0df68ef7ff85a48192b616b4e1be7be26", size = 5616589, upload-time = "2026-11-03T03:52:40.664Z" }, + { url = "https://files.pythonhosted.org/packages/5f/87/c290af551e9b71ef048601fe261e7f91212274fa1fb224619d35818aca33/psycopg_binary-3.3.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:652a6029e53635ef3106c726ef3b4c01429a586372dd516fc56e75ab8a2bd934", size = 5330007, upload-time = "2026-11-03T03:52:47.627Z" }, + { url = "https://files.pythonhosted.org/packages/af/e1/efa4ea43b53fcf0f33be16d968851b610fd02c45211306a97fc84446b877/psycopg_binary-3.3.7-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2fa90f30e4a5a1dff00301797ec66b5eba991566291dcc9df6dfb6fd68965e1", size = 6879399, upload-time = "2026-11-03T03:52:57.022Z" }, + { url = "https://files.pythonhosted.org/packages/f3/2e/7d6d734fc4be304b567c47e28c6eca98cdcb30eeb84d8cb352f4bdadb0c0/psycopg_binary-3.3.7-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:496da9a6f4f7fcdb8394b1273f9ffc10deb5a95dd2aa6e85212a732bfd9a2424", size = 5110921, upload-time = "2026-11-03T03:53:01.907Z" }, + { url = "https://files.pythonhosted.org/packages/d0/3c/d6e035a73062d3abbb8829bc294ae027eb7f99ca09bc0ddbea1d78627972/psycopg_binary-3.3.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d7434c3fc858d86eb09217bce226274245957299d22e2c7d8fc7b564409d242e", size = 4647372, upload-time = "2026-11-03T03:53:06.633Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f3/8078373f96894d3f4988a3ef924646bcecddce884dfc8f547a6d66cdb517/psycopg_binary-3.3.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a07ead8eacaa3b2e130ebf5357035c53fc9bbb623fe5d7c02d83944ffeae5d09", size = 4343403, upload-time = "2026-11-03T03:53:16.037Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4a/236c3be410e3c420249119c0ed7fc0372bd292f35f980efe1ca0825e42fc/psycopg_binary-3.3.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:4e960429b237ccff892a489b9348ade8cf780a103fcea403ddf924e0a07ad671", size = 4058720, upload-time = "2026-11-03T03:53:22.364Z" }, + { url = "https://files.pythonhosted.org/packages/8a/78/50b5b85c16e3526c2e001d0faccf9241e2bf7407cd4ad2a16bd353e4a991/psycopg_binary-3.3.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34e2cff14aafa3d0649b48cd18c447c163e0a3ec7bbfd2e38314cd721f33add7", size = 4363326, upload-time = "2026-11-03T03:53:26.846Z" }, + { url = "https://files.pythonhosted.org/packages/80/95/559b7910fe35a5f63adc7425bcad46c85ee10880dae8096b7789ea6d75ac/psycopg_binary-3.3.7-cp311-cp311-win_amd64.whl", hash = "sha256:cb6fe750879c781d3d3667b6d0f66318071d0245e60b51ad9a00186708978089", size = 3636509, upload-time = "2026-11-03T03:53:31.315Z" }, + { url = "https://files.pythonhosted.org/packages/bb/da/3a41bfb4ca39f601abaa20f8170545b27247df4d4850ee3647a10091c219/psycopg_binary-3.3.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:edfd6abb5da972cae50984d41d44cc744a6842ee359b0df24b39fb1524b1e654", size = 4710682, upload-time = "2026-11-03T03:53:37.421Z" }, + { url = "https://files.pythonhosted.org/packages/df/d9/775f908d6af11f0d658526cdd564dc54f03463c53de245cd7e23544c133b/psycopg_binary-3.3.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6d43d35af50c66fd5b0d2537ef20bddee61b68fbfbd24f8c44a0be6f65a68301", size = 4773036, upload-time = "2026-11-03T03:53:44.926Z" }, + { url = "https://files.pythonhosted.org/packages/8f/7f/40944fa341e752bb79410bd2f6542c71e3103a476ff4e88cc780fd53caac/psycopg_binary-3.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1a831476577db5e62f1aa3942a60c5088f459f2c14c0e2f4639a823d23b5ffb6", size = 5575526, upload-time = "2026-11-03T03:53:51.59Z" }, + { url = "https://files.pythonhosted.org/packages/df/70/0d68867bdb39834ae5a46d5afa1d12053dd12b061659652294fbaa092f71/psycopg_binary-3.3.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df30e4e2fe7b4f4ae401326f8f24fd572d1f08543b97c2daff9bcb623aebb41f", size = 5300375, upload-time = "2026-11-03T03:53:57.765Z" }, + { url = "https://files.pythonhosted.org/packages/bf/04/4ab0135ff483b6bdcdb7fa2b6c74efec93b0d5bb2e0ffc33b099d9cae98c/psycopg_binary-3.3.7-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:71601439823d28250c28255b2204376a7d686c2ab1259e5b6e0fce70cd2309d5", size = 6852910, upload-time = "2026-11-03T03:54:03.57Z" }, + { url = "https://files.pythonhosted.org/packages/55/80/0d360077293a95f3c2539e2afe054fee657e1dd5278e00f2007b645f7841/psycopg_binary-3.3.7-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c9e740f8f9fda8ac878d11d590a8b802f9b0ebf7a2c4c1fbbd577b5ceaff6694", size = 5073829, upload-time = "2026-11-03T03:54:09.479Z" }, + { url = "https://files.pythonhosted.org/packages/f1/6d/8e1b4186702b55a54a2e8f8fb7b766484659c3298de58eec754b3c801691/psycopg_binary-3.3.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e23fb1b141e9f05724f5bc83177427b3c32ca7f814450721614d23c4b2828cf5", size = 4618943, upload-time = "2026-11-03T03:54:17.659Z" }, + { url = "https://files.pythonhosted.org/packages/8d/cc/34bf9da0ce2103c7bab4de0e2716646448355fceab847620b74d67839819/psycopg_binary-3.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d52a488169e52774a9e09d4d0d0082d6f19cb59c58f8308b53bf355ae160a1ac", size = 4302298, upload-time = "2026-11-03T03:54:25.646Z" }, + { url = "https://files.pythonhosted.org/packages/26/38/75b5dca8d42e983d900688dbc9ada57b55ebcc3406cbcea7f1b8c92eac32/psycopg_binary-3.3.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:b2cbb774a73d6f12fbc2ab4d2faddd05a043dd623dbd07df59ec42535b00fedf", size = 4020129, upload-time = "2026-11-03T03:54:30.715Z" }, + { url = "https://files.pythonhosted.org/packages/0f/6c/0dfac7abe24a08b98367960535d164544f89eb3eb317e16979db36365dc6/psycopg_binary-3.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:df59dd2a1aa038856952360bbc72599ed118fe14cef558a499476bf56b0ecb45", size = 4334125, upload-time = "2026-11-03T03:54:36.812Z" }, + { url = "https://files.pythonhosted.org/packages/36/60/f412c2accfb92f8382b812f45990965e4392674a1d0d1648fe743a8fe787/psycopg_binary-3.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:7a1fd4e844630393800a86e55312c77dcdfa12dd7baf02da3230c7147a805793", size = 3619065, upload-time = "2026-11-03T03:54:41.265Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ff/936c13638c8b9bd4519d3ad7996e7600b066909156cb1cd6f55b73d2203e/psycopg_binary-3.3.7-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e0639d06c50c684bb9867fb4c0593b7327525275142c0b620053c55134572773", size = 4715405, upload-time = "2026-11-03T03:54:45.749Z" }, + { url = "https://files.pythonhosted.org/packages/7a/19/98a3d8dd5b61478ce242f017b70d68241d151826409ab3f88066ed2ec327/psycopg_binary-3.3.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:89b508e50ccfe93a440ce9450c33ff78a3e2462dab369e66ee63e725562421cb", size = 4775846, upload-time = "2026-11-03T03:54:51.138Z" }, + { url = "https://files.pythonhosted.org/packages/9d/74/c855489f8be5ee3d8bfd4cee523e036a49b3aa1bfdc6eb5b72c34828fdb2/psycopg_binary-3.3.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:a0aeb1b680b2ee414a0196a9dd94b17edaa75c1c04f7cddadc9aae95d0dd940c", size = 5575585, upload-time = "2026-11-03T03:54:57.816Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c9/860db102775b7fcf1f08fe06b0d845f926fe5ab89599d23007693354331d/psycopg_binary-3.3.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e3cc435fd83ff3fe627370bb8208bbcfd79631be2ac3b49b7a5c7edb528011c9", size = 5301826, upload-time = "2026-11-03T03:55:05.309Z" }, + { url = "https://files.pythonhosted.org/packages/a7/37/5f09fb2d1ad05f439329350097c702c52c1355156c176e3515d12496f723/psycopg_binary-3.3.7-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b311c834ece61e9be0df48c6078ca8949ef50d43a884f4c05628f77a8b31442", size = 6852614, upload-time = "2026-11-03T03:55:14.88Z" }, + { url = "https://files.pythonhosted.org/packages/6e/62/befa167c2ea5bfdeb65ce0f799aa35d78eb693250b650963ec7cba1fa17d/psycopg_binary-3.3.7-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:15873c72c6db035b7bbe9d443dedc571753c713b7d10223aea6438310c0f468e", size = 5076255, upload-time = "2026-11-03T03:55:23.593Z" }, + { url = "https://files.pythonhosted.org/packages/eb/97/8e622891df044606e0819cd17b589d4b47834eddd3f941848edab207b7e8/psycopg_binary-3.3.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:191dc5f2056368e0556359be7782a3be201399c7f1dfc3fe0ebffde37446e1f2", size = 4619359, upload-time = "2026-11-03T03:55:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/40/84/857ec7d5ab5e63ec05e98d22d0e8c3b85df5a49cdfb8b3fc84879424040a/psycopg_binary-3.3.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:21dccda9c04e52240b9d6d934c5b710c198fdd7d0c60a358d78927988bca154e", size = 4301370, upload-time = "2026-11-03T03:55:35.691Z" }, + { url = "https://files.pythonhosted.org/packages/d9/f3/9a21844b395d49dbe7e6fd53c7cddfc7e54766e7d9957c89a8fec6c75c2a/psycopg_binary-3.3.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9ae3e685302569c34bf159927e9667d3afd5dd34e68a92afabe05e2022dafc68", size = 4021521, upload-time = "2026-11-03T03:55:39.78Z" }, + { url = "https://files.pythonhosted.org/packages/08/d7/8eda1d662b7049a6729a6b5f30601561e5204bb7ff3ae15796640afdba53/psycopg_binary-3.3.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4c8ab53d4a7c9f68d4b2cf878d1e56880bb87a1008435a5c3770422ac1e1b4ef", size = 4334750, upload-time = "2026-11-03T03:55:45.384Z" }, + { url = "https://files.pythonhosted.org/packages/dd/92/2a1ad37faa52270594e92c483a28084d75a3c696b95ad8f6fc18faf39ead/psycopg_binary-3.3.7-cp313-cp313-win_amd64.whl", hash = "sha256:63fc0a5a226e63db37a3f038f2581ea8b8189112daa86bd8ae52c3897ca105da", size = 3618325, upload-time = "2026-11-03T03:55:49.757Z" }, + { url = "https://files.pythonhosted.org/packages/07/19/a0b79901d532257d28867c3b5345f37a30191670a8998b8fd779f935d053/psycopg_binary-3.3.7-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d72b6bd81535303b3f6726de330c257d8912948d9e0cc1761c41911132633112", size = 4723602, upload-time = "2026-11-03T03:55:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/55/43/a22a87c7a51fd59c90223bd4d4fc604444b8647890af2893c50e823c1bcf/psycopg_binary-3.3.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6c3753802f809d8fcec4fbf633f3c3335909a2f1638a36f9b924edfa2f6e8796", size = 4786213, upload-time = "2026-11-03T03:56:03.022Z" }, + { url = "https://files.pythonhosted.org/packages/60/bb/2f6fe94326baaedcea29fc06c13b1b8b5dfe81690c422eda09039c531dbb/psycopg_binary-3.3.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:590c075b8645bdddb332a61ca1be5be52e7ecb57d3a3be63eba5c6a38a60b789", size = 5588332, upload-time = "2026-11-03T03:56:08.561Z" }, + { url = "https://files.pythonhosted.org/packages/17/d4/4e60003f3094e71849d29ac174b355fc0b0b9b16eb5799591b05b62a3893/psycopg_binary-3.3.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:20dbb0f8e173a46feb803136446a89d4b04959342f5331e7f3f93b4fe7f2c32e", size = 5310722, upload-time = "2026-11-03T03:56:14.427Z" }, + { url = "https://files.pythonhosted.org/packages/66/76/08edccbdc1e25e99854297bdc93ad0de34b029ff707ff6567b54e0f4421f/psycopg_binary-3.3.7-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4171b7a2d0534baf7ac1792aade1f8aba74ca06357db79915eac265140fc12c8", size = 6866520, upload-time = "2026-11-03T03:56:21.962Z" }, + { url = "https://files.pythonhosted.org/packages/9f/59/b56fe4ed6747241fdd92c4859d17ea3a10bdbca586a1e1baff5bfba09eaf/psycopg_binary-3.3.7-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7965237729bfec12b3682dedf84538bc0936c2c442560dbca8cd66534d62747", size = 5085522, upload-time = "2026-11-03T03:56:30.456Z" }, + { url = "https://files.pythonhosted.org/packages/3c/07/4e7251607b1278aa47894fec895c6d961ff8b5d436be010557ba03647f69/psycopg_binary-3.3.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:100495111a2de4305c92996708469b5a461b24c28d152ba6bf2fbebe49fe02f2", size = 4633868, upload-time = "2026-11-03T03:56:36.786Z" }, + { url = "https://files.pythonhosted.org/packages/71/3d/b6b09ec08b9235eea343df05b9e5d978fb784c9adea01c4a1649c440436b/psycopg_binary-3.3.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f2018e449c7d60dd6e66261e7c4cd9a4274baf6d824994bbd55577435c081cea", size = 4319169, upload-time = "2026-11-03T03:56:41.292Z" }, + { url = "https://files.pythonhosted.org/packages/2c/93/ea77791ceaad20970a04d3fbc132138cfb2e6f845fd9604f5705da4b0dc9/psycopg_binary-3.3.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:8487a5e112b6a22508e603c78a0dc496372daefed22a9b9d4de4b89e2d4456bc", size = 4032876, upload-time = "2026-11-03T03:56:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/8c/1a/56e6bb0fad9d2751cb120a833e6113b860131e26a127f8a0e4088f56994d/psycopg_binary-3.3.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a58b9a9e7b2d31367f605c53d87d501bee7c52f5c009dd8a2d341e5f2c2b6f8e", size = 4346131, upload-time = "2026-11-03T03:56:51.75Z" }, + { url = "https://files.pythonhosted.org/packages/8c/4d/efd82743bef4b5ff120f57f2c3ffb5b5e72f3f66dec1b17ad215c090c084/psycopg_binary-3.3.7-cp314-cp314-win_amd64.whl", hash = "sha256:40a4231674cd06465384ade0fa5886820c6f25b5a26d38b4ce946c391162cb0f", size = 3715677, upload-time = "2026-11-03T03:56:56.651Z" }, + { url = "https://files.pythonhosted.org/packages/5c/69/36a022cd3739bdb2d94003c9d3c220618f8c86b8596c29831d215b490a7b/psycopg_binary-3.3.7-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:d8acf1f90217e5a398daaa0370e078675c0b767a3d0422b74fd3ffd83bcd874d", size = 4722833, upload-time = "2026-11-03T03:57:03.208Z" }, + { url = "https://files.pythonhosted.org/packages/88/52/c97e936e4db0522c93ea39ba26b5cb21de178b69cd37f37ce5d77344bad3/psycopg_binary-3.3.7-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:3919dc8d37d1ed6a1247753111f3eee5a3b906846b27eb3f08af3c907bf15806", size = 4785089, upload-time = "2026-11-03T03:57:08.978Z" }, + { url = "https://files.pythonhosted.org/packages/be/85/9f78af7011735fc98a168a8a4b41abd07df53af0749eb5cf4daf076343a3/psycopg_binary-3.3.7-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:cdc2ce73b2409d2af6d2dbd24ced79cca8408fc57faef1bfc7a1fcf50391f1e1", size = 5586061, upload-time = "2026-11-03T03:57:14.849Z" }, + { url = "https://files.pythonhosted.org/packages/37/03/28a682eda331f59452b5745d776a72ad673f240a86f096d694344ad7e25f/psycopg_binary-3.3.7-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:022d7297c6ff13e3b9fe4772979b55dcea0f043b3a2d3211b92622403faffdd5", size = 5312779, upload-time = "2026-11-03T03:57:20.976Z" }, + { url = "https://files.pythonhosted.org/packages/8b/66/db3e08d00290ff08764506c643da0682565794d037e84787346da33a5e43/psycopg_binary-3.3.7-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b20a404dc921bde74c00c2ca082ba2864cf59ecc610675ac39b3a14a5aaacc", size = 6866400, upload-time = "2026-11-03T03:57:31.307Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2f/31b275285bbcce0df0c943908ee3543bdff6d46a657a3e7e101c4a39d5a3/psycopg_binary-3.3.7-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018ba4b5eeefd1400f7e2d042c853c14bc9d7a5e2b180b07e63a59e900a90e00", size = 5093067, upload-time = "2026-11-03T03:57:38.949Z" }, + { url = "https://files.pythonhosted.org/packages/6b/69/0868542c2cbfd622a35c39ada9b604e423610ac703dfcaa5f6b7908906a8/psycopg_binary-3.3.7-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:a09474c66e0068724ffd2ab7a6ae0314526d9156dc43688d242d6d00654e4a75", size = 4634472, upload-time = "2026-11-03T03:57:44.967Z" }, + { url = "https://files.pythonhosted.org/packages/42/60/7998247ac07e5094c4c552a852ee8b0a9207083692a140d79cf29a00f696/psycopg_binary-3.3.7-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:1da7f340bc7d3a7945422148001893ee003ab0e93722d6a996010cba96b71093", size = 4316481, upload-time = "2026-11-03T03:57:51.186Z" }, + { url = "https://files.pythonhosted.org/packages/57/b0/dae410a3c17f634f18f7f6bb1003a9986b2a52347593d87deaf509dd7056/psycopg_binary-3.3.7-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:f3e0ba732d49cebe08f62e8ba8995e7cf82c74f6b43b880d4f05bc592415787a", size = 4039869, upload-time = "2026-11-03T03:57:56.388Z" }, + { url = "https://files.pythonhosted.org/packages/0e/3c/daadb40466b98e0465c767006c72bd3eda3e2d581822cd4212debd5273f2/psycopg_binary-3.3.7-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:48528deb18273d53fc49382cf4eaf309c3013f1a2aa8e22b0fdec580ba235189", size = 4349380, upload-time = "2026-11-03T03:58:00.373Z" }, + { url = "https://files.pythonhosted.org/packages/1c/85/728f34d24afdb135a1cd7bfe79dd76da3ee7a6089a1ddd61ee737f158300/psycopg_binary-3.3.7-cp315-cp315-win_amd64.whl", hash = "sha256:5ca2e91d166c1cdd44073fac07c6dd1dd80011e474650d24778086bb57f59a22", size = 3716314, upload-time = "2026-11-03T03:58:06.014Z" }, +] + [[package]] name = "psycopg2-binary" version = "2.9.12" @@ -6519,7 +6607,7 @@ name = "sqlalchemy" version = "2.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version >= '3.13.14'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/36/dc/9addbf490110b5727380059ee542ad8353bdcee45a37ba6a5e0976966ae0/sqlalchemy-2.1.6.tar.gz", hash = "sha256:499eae5c9051efe96cb9a73ec2fbde8ffa4186f102b9094ebab8fcc9d3b99f52", size = 10589700, upload-time = "2026-10-21T16:25:06.497Z" } wheels = [ From ef1c24b3f9b21ac649c4a9403d880528442ac5b5 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 15:00:25 -0800 Subject: [PATCH 2/6] fix(website): a models= site's data load leaves out computed columns, so its own export loads back (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/database.py | 9 +++++++-- .../tests/test_website_database.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index 34373ea47..97509fbcf 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -114,7 +114,8 @@ def export(self) -> dict: def load(self, data: dict) -> None: """Insert ``data``'s rows, each table's after the tables it refers to, in their order, each run of rows that name the same columns as one insert (one insert takes its columns from its first row, so a row that leaves an - optional column out goes in its own); refuses a name that is no table.""" + optional column out goes in its own), leaving out each computed column, which the database works out itself, so + what ``export`` gave loads back; refuses a name that is no table.""" tables = {table.name: table for table in self.metadata.sorted_tables} unknown = sorted(set(data) - set(tables)) if unknown: @@ -146,9 +147,13 @@ def key(self, name: str, row: dict) -> str: def _row(table: Any, row: dict) -> dict: - """``row`` with each JSON string a date, time, decimal or UUID column holds parsed to that type.""" + """``row`` without its computed columns' values, and with each JSON string a date, time, decimal or UUID column + holds parsed to that type.""" parsed = dict(row) for column in table.columns: + if column.computed is not None: + parsed.pop(column.name, None) + continue value = parsed.get(column.name) if isinstance(value, str): kind = _python_type(column) diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index 1b12d96de..7eeb96b64 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -110,6 +110,25 @@ class Note(Threads): assert rpc(client, "data/get")["parts"][0]["data"]["notes"] == [{"parent_id": None, **row} for row in seed] +def test_an_export_with_a_computed_column_loads_back_as_the_database_computes_it(dist: Path, database: str) -> None: + class Priced(DeclarativeBase): + pass + + class Item(Priced): + __tablename__ = "items" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + price: Mapped[int] = mapped_column(Integer) + doubled: Mapped[int] = mapped_column(Integer, sqlalchemy.Computed("price * 2")) + + client = TestClient(helpdesk(dist, Priced)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"items": [{"id": 1, "price": 3}]}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert exported == {"items": [{"id": 1, "price": 3, "doubled": 6}]} + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + + def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, database: str) -> None: env = helpdesk(dist, Base)() client = TestClient(env.create_app()) From 9741dbf5d627371dfee1e02731158e4b43a0feff Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 16:00:45 -0800 Subject: [PATCH 3/6] fix(website): a models= site loads a self-referencing table's rows parents first, so its export loads back (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/database.py | 38 ++++++++++++++++--- .../tests/test_website_database.py | 27 +++++++++++++ 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index 97509fbcf..34181bf91 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -112,10 +112,11 @@ def export(self) -> dict: sqlalchemy.select(table).order_by(*table.primary_key.columns))] for table in self.metadata.sorted_tables} def load(self, data: dict) -> None: - """Insert ``data``'s rows, each table's after the tables it refers to, in their order, each run of rows that name - the same columns as one insert (one insert takes its columns from its first row, so a row that leaves an - optional column out goes in its own), leaving out each computed column, which the database works out itself, so - what ``export`` gave loads back; refuses a name that is no table.""" + """Insert ``data``'s rows, each table's after the tables it refers to, in their order but each row of a table that + refers to itself after the row it refers to (``_parents_first``), each run of rows that name the same columns as + one insert (one insert takes its columns from its first row, so a row that leaves an optional column out goes in + its own), leaving out each computed column, which the database works out itself, so what ``export`` gave loads + back; refuses a name that is no table.""" tables = {table.name: table for table in self.metadata.sorted_tables} unknown = sorted(set(data) - set(tables)) if unknown: @@ -125,7 +126,7 @@ def load(self, data: dict) -> None: rows = data.get(name) or [] if not (isinstance(rows, list) and all(isinstance(row, dict) for row in rows)): raise ValueError(f"table {name!r}'s data is not a list of rows") - for _, run in itertools.groupby(rows, key=lambda row: sorted(row)): + for _, run in itertools.groupby(_parents_first(table, rows), key=lambda row: sorted(row)): connection.execute(table.insert(), [_row(table, row) for row in run]) if connection.dialect.name == "postgresql": _advance_sequences(connection, tables.values()) @@ -142,6 +143,33 @@ def key(self, name: str, row: dict) -> str: return json.dumps([row.get(c) for c in columns] if columns else row, sort_keys=True, default=str) +def _parents_first(table: Any, rows: list[dict]) -> list[dict]: + """``rows`` in their order, but each row that refers to another row of ``table`` through a foreign key of the table's + own after that row, so a constraint that isn't deferred holds at each insert; rows in a cycle keep their order.""" + links = [(fk.parent.name, fk.column.name) for fk in table.foreign_keys if fk.column.table is table] + if not links: + return rows + by_value = {(referenced, row.get(referenced)): index for index, row in enumerate(rows) + for _, referenced in links if row.get(referenced) is not None} + ordered, placed, visiting = [], set(), set() + + def place(index: int) -> None: + if index in placed or index in visiting: + return + visiting.add(index) + for local, referenced in links: + parent = by_value.get((referenced, rows[index].get(local))) + if parent is not None and parent != index: + place(parent) + visiting.discard(index) + placed.add(index) + ordered.append(rows[index]) + + for index in range(len(rows)): + place(index) + return ordered + + _PARSERS: dict = {datetime.datetime: datetime.datetime.fromisoformat, datetime.date: datetime.date.fromisoformat, datetime.time: datetime.time.fromisoformat, decimal.Decimal: decimal.Decimal, uuid.UUID: uuid.UUID} diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index 7eeb96b64..e2690f233 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -129,6 +129,33 @@ class Item(Priced): assert rpc(client, "data/get")["parts"][0]["data"] == exported +def test_an_export_whose_child_row_sorts_before_its_parent_loads_back_with_foreign_keys_enforced( + dist: Path, database: str) -> None: + class Threads(DeclarativeBase): + pass + + class Note(Threads): + __tablename__ = "notes" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + parent_id: Mapped[int | None] = mapped_column(ForeignKey("notes.id"), nullable=True) + + def enforce(connection, record) -> None: + connection.execute("PRAGMA foreign_keys=ON") + + sqlalchemy.event.listen(sqlalchemy.engine.Engine, "connect", enforce) + try: + client = TestClient(helpdesk(dist, Threads)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"notes": [ + {"id": 3, "parent_id": None}, {"id": 2, "parent_id": 3}, {"id": 1, "parent_id": 2}]}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert [row["id"] for row in exported["notes"]] == [1, 2, 3] + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + finally: + sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) + + def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, database: str) -> None: env = helpdesk(dist, Base)() client = TestClient(env.create_app()) From aabc036d8040bda1c26dcd50239336f7ac157ddb Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 18:19:47 -0800 Subject: [PATCH 4/6] refactor(website): no /go?sid= sessions and no site-side state for models= sites; data/get alone (FD-4683) - The store keeps no sessions: no `?sid=`, no `agentenv_sid_` cookie, no `changed_sessions`; `self.store` is the site's one store, which the data plane loads, exports and resets. - A models= site keeps no initial state and no diff; its data plane is its database's. - Conformance runs the round trip with no skip for changed state or sessions. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../examples/website/notes/README.md | 5 +- .../agentenv_protocol/website/conformance.py | 39 +----- .../src/agentenv_protocol/website/database.py | 25 +--- .../agentenv_protocol/website/environment.py | 127 +++-------------- .../src/agentenv_protocol/website/session.py | 8 -- .../src/agentenv_protocol/website/store.py | 89 +----------- .../tests/test_website_conformance.py | 51 ++----- .../tests/test_website_database.py | 37 +---- .../tests/test_website_environment.py | 52 +------ .../tests/test_website_notes_example.py | 8 -- .../tests/test_website_state.py | 132 ------------------ 11 files changed, 50 insertions(+), 523 deletions(-) delete mode 100644 packages/agentenv-protocol/tests/test_website_state.py diff --git a/packages/agentenv-protocol/examples/website/notes/README.md b/packages/agentenv-protocol/examples/website/notes/README.md index fc5cba314..4f0c590d3 100644 --- a/packages/agentenv-protocol/examples/website/notes/README.md +++ b/packages/agentenv-protocol/examples/website/notes/README.md @@ -29,9 +29,8 @@ them. The page shows the universe's time. store's `current_user`), and `GET /api/session` signs that user in. `POST /api/sign-in` switches to another seeded user. A user's notes are invisible to every other user, and another user's note answers 404. - **State.** The site writes no data hooks, so it uses the default store. `data/add` loads the seed (as a data part, or - as a file part by bytes or `file://`), `data/get` exports it, `data/reset` empties it, and `/go` returns - `{initial_state, current_state, state_diff}`. A page opened at `?sid=` keeps a state of its own, started - from the seed, which `/go?sid=` reports; without `sid` it is the default session, the data plane's. + as a file part by bytes or `file://`), `data/get` exports it, and `data/reset` empties it. `data/get` is how + agentenv reads what a task left the site with. - **Time.** Notes are stamped with `self.now()`, which follows the gateway's clock once `clock/v1`'s `sync_time` names it. The frontend reads `X-Universe-Now` from every API response into `universeNow()` and shows it in the header. It uses this helper rather than `clock_patch=True`, so the page works the same against a server written without the diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py index 67a8a114a..d5bc89c26 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/conformance.py @@ -14,14 +14,10 @@ redirect), and cookies tied to no domain, the session path's the one the card names. A failed one is a ``warning``. The round trip leaves the server's data as it found it, adding the export back when a step after the reset fails too. -data/add sets the state a task starts from, so a server whose state shows changes since its initial state, as an SDK -site's does, would have them folded into it, and data/reset drops a site's named sessions, which data/get doesn't -export: the round trip is skipped for a server whose state shows changes or ``changed_sessions``. Its state is its -card's ``website/v1`` ``state``, else, for a card without it, what the path its ``params.state.path`` names answers, -else ``/go``'s, deprecated. The clock check leaves the server following a clock that is gone once the check ends, so a -server it ran against keeps extrapolating from that clock's time until it's synced again. The checker's clock starts at -``CLOCK_START``, far enough from wall time that a synced server shows it, and a synced server's ``X-Universe-Now`` may -be ``CLOCK_SLACK_S`` seconds from it.""" +The clock check leaves the server following a clock that is gone once the check ends, so a server it ran against +keeps extrapolating from that clock's time until it's synced again. The checker's clock starts at ``CLOCK_START``, far +enough from wall time that a synced server shows it, and a synced server's ``X-Universe-Now`` may be ``CLOCK_SLACK_S`` +seconds from it.""" from __future__ import annotations import argparse @@ -42,9 +38,8 @@ from ..client import find_extension, find_extension_method from ..types import METHOD_ADD, METHOD_GET, METHOD_RESET, RPC_PATH, WELL_KNOWN_PATH from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, iso, parse_time -from .environment import DEFAULT_HEALTH_PATH, STATE_PATH +from .environment import DEFAULT_HEALTH_PATH from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness -from .state import WEBSITE_STATE_OP Status = Literal["passed", "failed", "warning", "skipped"] CLOCK_START = datetime.datetime(2031, 5, 6, 7, 8, 9, tzinfo=datetime.timezone.utc) @@ -204,10 +199,6 @@ def _data_plane(self, card: dict) -> tuple[Status, str]: if missing: return "failed", f"the card's operations lack {', '.join(missing)}" exported = self._rpc(METHOD_GET).get("parts") or [] - if where := self._changed_since_initial(card): - return "skipped", (f"data/get answered; {where} shows changes since the initial state, or named sessions " - "with changes, which data/reset and data/add would fold in or drop, so the round trip " - "was not run") self._rpc(METHOD_RESET) added_back = False try: @@ -229,26 +220,6 @@ def _data_plane(self, card: dict) -> tuple[Status, str]: return "failed", "data/get after data/reset and data/add of its own export returned other data" return "passed", f"{len(exported)} part(s) round-tripped through reset and add" - def _changed_since_initial(self, card: dict) -> str | None: - """Where the server's state was read when its ``state_diff`` names changes or it names ``changed_sessions``, - None when it names neither or can't be read: the card's ``website/v1`` ``state``, else the path its - ``params.state.path`` names, else ``/go``. Read aside, as no check's answer, since a server without one needn't - stamp its 404.""" - op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) - if op is not None and op.get("endpoint"): - where, request = f"website/v1 {WEBSITE_STATE_OP}", (op["method"], self.base_url + op["endpoint"]) - else: - declared = ((find_extension(card, WEBSITE_EXTENSION_URI) or {}).get("params") or {}).get("state") - path = declared.get("path") if isinstance(declared, dict) else None - where = path if isinstance(path, str) and path.startswith("/") else STATE_PATH - request = ("GET", self.base_url + where) - try: - state = self.client.request(*request).json() - except Exception: - return None - changed = isinstance(state, dict) and bool(state.get("state_diff") or state.get("changed_sessions")) - return where if changed else None - def _clock(self, card: dict, health: str) -> tuple[Status, str]: if find_extension(card, CLOCK_EXTENSION_URI) is None: return "failed", f"the card declares no {CLOCK_EXTENSION_URI}" diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index 34181bf91..dbc260f02 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -12,15 +12,12 @@ naming it. Beside the tables, the data's string ``current_user`` names the user a browser starts signed in as, as the default -store's does, and the state data/add last loaded is the initial one ``/go`` reports. A models site keeps one state, -the database's, so ``/go`` answers it for every ``sid``.""" +store's does.""" from __future__ import annotations -import copy import datetime import decimal import itertools -import json import types import uuid from typing import Any, Callable, Iterable @@ -28,7 +25,7 @@ from pydantic_core import to_jsonable_python from ..agent_env_environment import _maybe_await -from .store import CURRENT_USER, part_data, verifier_state +from .store import CURRENT_USER, part_data HOOKS = ("load", "export", "reset") @@ -136,12 +133,6 @@ def reset(self) -> None: for table in reversed(self.metadata.sorted_tables): connection.execute(table.delete()) - def key(self, name: str, row: dict) -> str: - """A row's identity, its primary key's values, which ``/go``'s diff matches rows by.""" - table = next((t for t in self.metadata.sorted_tables if t.name == name), None) - columns = [c.name for c in table.primary_key.columns] if table is not None else [] - return json.dumps([row.get(c) for c in columns] if columns else row, sort_keys=True, default=str) - def _parents_first(table: Any, rows: list[dict]) -> list[dict]: """``rows`` in their order, but each row that refers to another row of ``table`` through a foreign key of the table's @@ -230,17 +221,16 @@ def engine(self) -> Any: class ModelState: - """A models site's data plane and ``/go``: ``hooks`` loads, exports and resets its data, the state data/add last - loaded is the initial one, and the data's ``current_user`` is kept beside it.""" + """A models site's data plane: ``hooks`` loads, exports and resets its data, and the data's ``current_user`` is + kept beside it.""" def __init__(self, hooks: Any) -> None: self.hooks = hooks - self.initial: dict = {} self.current_user: str | None = None async def reset(self) -> None: await _maybe_await(self.hooks.reset()) - self.initial, self.current_user = {}, None + self.current_user = None async def load_parts(self, parts: list) -> None: for part in parts: @@ -248,14 +238,9 @@ async def load_parts(self, parts: list) -> None: if CURRENT_USER in data: self.current_user = data.pop(CURRENT_USER) await _maybe_await(self.hooks.load(data)) - self.initial = copy.deepcopy(await self.export()) async def export(self) -> dict: data = dict(await _maybe_await(self.hooks.export())) if self.current_user is not None: data[CURRENT_USER] = self.current_user return data - - async def state(self) -> dict: - key = getattr(self.hooks, "key", None) - return verifier_state(copy.deepcopy(self.initial), await self.export(), key) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py index 9a7a27059..5c3b1eda8 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/environment.py @@ -2,23 +2,13 @@ plain HTTP on ``PORT`` for any host name: its ``@api`` routes, its static folder at ``/`` with ``index.html`` for the pages a single-page app routes itself, its health path, and the data plane and card the base class serves. Every answer carries ``X-Universe-Now``, the universe's time, which ``self.now()`` reads and ``clock/v1``'s ``sync_time`` ties to -the gateway's virtual clock. Without data hooks of its own a site keeps its state in ``self.store``, or with -``@website(models=...)`` in an MCP server's database, which ``self.db`` opens sessions of (``website.database``). - -A store-backed site keeps a state per session, the CUA-Gym convention the website state verifier reads: a request's -session is its ``?sid=``, else the one its browser last named, which the site keeps in the ``agentenv_sid_`` -cookie, else the default one. ``self.store`` is the request's session's store, each session started as a copy of the -initial state; the default session, ``sid=default``, is the state the data plane loads, exports and resets, and the -one a browser that names no session uses, so a site browsed without ``sid`` works as it did before sessions. - -Two routes are built in, each replaced by a site's own ``@api`` route at its very path and answering before one whose -path parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the -acting user in when nobody is (GET), or to sign out (DELETE); and, for a store-backed site, ``/go?sid=``, that -session's ``{initial_state, current_state, state_diff}`` at the path ``website/v1``'s ``params.state.path`` names, -which a reader of a card without ``state`` falls back to, with the site's ``changed_sessions``, the named sessions whose -state changed, which data/reset would drop; a session no request has used yet answers the initial state, unchanged. A -store-backed site answers ``website/v1``'s ``state``, of the session its ``sid`` names, with the same ``WebsiteState``, -among agentenv's own routes, which is how the website state verifier reads it.""" +the gateway's virtual clock. Without data hooks of its own a site keeps its data in ``self.store``, or with +``@website(models=...)`` in an MCP server's database, which ``self.db`` opens sessions of (``website.database``); the +data plane loads, exports and resets it, and ``data/get`` is how agentenv reads what a task left a site with. + +One route is built in, replaced by a site's own ``@api`` route at its very path and answering before one whose path +parameters would also match it: ``/api/session``, which a frontend calls to learn who is signed in, signing the acting +user in when nobody is (GET), or to sign out (DELETE).""" from __future__ import annotations import inspect @@ -49,20 +39,18 @@ get_data, reset_data, ) -from ..types import DataPart, EnvironmentExtension +from ..types import DataPart from .clock import CLOCK_EXTENSION_URI, UNIVERSE_NOW_HEADER, UniverseClock, clock_patch_script, iso from .database import ModelState, Sessions, Tables, is_hooks, metadata_of from .errors import NotFound from .readiness import WEBSITE_EXTENSION_URI, WEBSITE_READY_OP, WebsiteReadiness from .routes import (ApiContext, ApiRoute, endpoint, error_response, forwarded_prefix, openapi, openapi_path, request_role, reserved_path, routes_of) -from .session import SessionSigner, expires_at, session_cookie_name, sid_cookie_name -from .state import WEBSITE_STATE_OP, WebsiteState -from .store import DEFAULT_SID, Collections, active_sid, session_id +from .session import SessionSigner, expires_at, session_cookie_name +from .store import Collections DEFAULT_HEALTH_PATH = "/api/health" SESSION_PATH = "/api/session" -STATE_PATH = "/go" _WEBSITE_ATTR = "_aee_website" _READY_ATTR = "_aee_website_ready" _DEFAULT_HOOK_ATTR = "_aee_default_store_hook" @@ -87,8 +75,8 @@ def website(_cls: type | None = None, *, domains: list[str] | tuple[str, ...] = health: str = DEFAULT_HEALTH_PATH, clock_patch: bool = False, data: str | None = None, models: Any = None) -> Any: """Declare a WebsiteEnvironment's site. ``data`` names a JSON file the default store starts with; ``models`` - keeps the site's state in a database instead, an MCP server's (see ``website.database``), and the site's - data plane and ``/go`` are that database's.""" + keeps the site's data in a database instead, an MCP server's (see ``website.database``), and the site's + data plane is that database's.""" if _cls is not None: raise TypeError("use @website(...) with parentheses") if not health.startswith("/") or health == "/" or reserved_path(health): @@ -123,16 +111,6 @@ class WebsiteEnvironment(AgentEnvEnvironment): app: ClassVar[Any] = None asgi: Any = None db: Any = None - _store: Collections - - @property - def store(self) -> Collections: - """The state store of the request's session: the default session's outside a request.""" - return self._store.session(active_sid.get()) - - @store.setter - def store(self, value: Collections) -> None: - self._store = value def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) @@ -148,7 +126,6 @@ def __init__(self) -> None: self.store = Collections() if declaration.data is not None: self.store.load(json.loads(self._site_path(declaration.data).read_text(encoding="utf-8"))) - self.store.mark_initial() self.clock = UniverseClock() self._signer = SessionSigner(cookie_name=session_cookie_name(self.environment_name())) models = declaration.models @@ -217,14 +194,11 @@ def create_app(self) -> Starlette: api_routes = self.routes() taken = {openapi_path(r.path) for r in api_routes} root = self._site_path(declaration.static) if declaration.static is not None else None - asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self), - Middleware(_Session, cookie=sid_cookie_name(self.environment_name()))]) + asgi = Starlette(middleware=[Middleware(_Forwarded), Middleware(_Universe, env=self)]) AgentEnvStarletteApplication(self._build_card(), self).add_routes_to_app(asgi) asgi.router.routes.append(Route(declaration.health, self._health, methods=["GET"])) if SESSION_PATH not in taken: asgi.router.routes.append(Route(SESSION_PATH, self._session, methods=["GET", "DELETE"])) - if self._serves_state_path(): - asgi.router.routes.append(Route(STATE_PATH, self._state, methods=["GET"])) asgi.router.routes.extend(Route(r.path, _unreserved(endpoint(getattr(self, r.fn.__name__), r, self.context)), methods=[r.method]) for r in api_routes) asgi.router.default = _Fallback(root, type(self).app) @@ -249,17 +223,9 @@ def extension_params(self, uri: str) -> dict: declaration = self.declaration() return {"domains": list(declaration.domains), "health": declaration.health, "routes": [r.advertised() for r in self.routes()], "session_cookie": self.session_cookie, - "session_path": SESSION_PATH, **({"state": {"path": STATE_PATH}} if self._serves_state_path() else {})} + "session_path": SESSION_PATH} return super().extension_params(uri) - def served_extensions(self, extensions: list[EnvironmentExtension]) -> list[EnvironmentExtension]: - """The extensions the base class serves, ``website/v1``'s ``state`` only for a store-backed site: one with data - hooks of its own keeps its state where ``self.store`` can't see it.""" - served = super().served_extensions(extensions) - if self.uses_store: - return served - return [_without_method(e, WEBSITE_STATE_OP) if e.uri == WEBSITE_EXTENSION_URI else e for e in served] - @extension(CLOCK_EXTENSION_URI, description="Sync this site to the gateway's virtual clock.") async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) -> dict: """Follow the gateway's virtual time at ``env_get_time_url`` from now on, in ``timezone`` when the task names @@ -270,9 +236,8 @@ async def sync_time(self, env_get_time_url: str, timezone: Optional[str] = None) @extension(WEBSITE_EXTENSION_URI, op="openapi", method="GET", description=( "A site's server: domains, its health path and its @api routes by name; openapi returns the OpenAPI document " - "of its routes, ready whether the site is usable now and why not, state the task state it is in (its " - "initial and current state and their diff), and session the cookies a browser adds to start signed in as user, " - "or signed out without one.")) + "of its routes, ready whether the site is usable now and why not, and session the cookies a browser adds to " + "start signed in as user, or signed out without one.")) async def _website_openapi(self) -> dict: name = self._application.environment_card.name if self._application is not None else type(self).__name__ return openapi(name, self.routes()) @@ -284,16 +249,6 @@ async def _website_ready(self) -> dict: problems = await self._not_ready() return WebsiteReadiness(ready=not problems, problems=problems).model_dump() - @extension(WEBSITE_EXTENSION_URI, op=WEBSITE_STATE_OP, method="GET") - async def _website_state(self, sid: Optional[str] = None) -> dict: - """The site's task state, a ``WebsiteState``: its state when the task started, now, and what changed between - them, of session ``sid``'s store (the default session's without one), with the site's ``changed_sessions`` as - ``/go`` names them, or of a models site's database, which keeps one.""" - if self._models is not None: - return WebsiteState.model_validate(await self._models.state()).model_dump() - return {**WebsiteState.model_validate(self._store.session(sid).state()).model_dump(), - "changed_sessions": self._store.changed_sessions()} - @extension(WEBSITE_EXTENSION_URI, op="session") async def _website_session(self, user: Optional[str] = None) -> dict: cookie = self._session_signer().cookie(user) @@ -303,18 +258,17 @@ async def _website_session(self, user: Optional[str] = None) -> dict: async def _store_reset(self) -> None: if self._models is not None: return await self._models.reset() - self._store.reset() + self.store.reset() @add_data async def _store_add(self, parts: list) -> None: if self._models is not None: return await self._models.load_parts(parts) - self._store.load_parts(parts) - self._store.mark_initial() + self.store.load_parts(parts) @get_data async def _store_get(self) -> list: - return [DataPart(data=await self._models.export() if self._models is not None else self._store.export())] + return [DataPart(data=await self._models.export() if self._models is not None else self.store.export())] async def _not_ready(self) -> list[str]: return [check.__name__ for check, _marker in _marked_members(self, _READY_ATTR) if not await _ready(check)] @@ -334,15 +288,6 @@ async def _session(self, request: Request) -> Response: return context.apply(JSONResponse({"user": context.user, "role": context.role, "signed_in": context.user is not None})) - def _serves_state_path(self) -> bool: - """Whether the site serves its state at ``STATE_PATH``: it is store-backed and no ``@api`` route takes the path.""" - return self.uses_store and STATE_PATH not in {openapi_path(r.path) for r in self.routes()} - - async def _state(self, request: Request) -> Response: - if self._models is not None: - return JSONResponse(await self._models.state()) - return JSONResponse({**self.store.state(), "changed_sessions": self._store.changed_sessions()}) - def _clock(self) -> UniverseClock: if not hasattr(self, "clock"): raise TypeError(f"{type(self).__qualname__}.__init__ must call super().__init__()") @@ -363,12 +308,6 @@ def _site_path(cls, path: str) -> Path: setattr(getattr(WebsiteEnvironment, _hook), _DEFAULT_HOOK_ATTR, True) -def _without_method(extension: EnvironmentExtension, method: str) -> EnvironmentExtension: - params = dict(extension.params or {}) - params["methods"] = {name: entry for name, entry in (params.get("methods") or {}).items() if name != method} - return extension.model_copy(update={"params": params}) - - async def _ready(check: Any) -> bool: try: return bool(await _maybe_await(check())) @@ -416,34 +355,6 @@ def _first(value: bytes | None) -> str: return value.decode("latin-1").split(",")[0].strip() if value else "" -class _Session: - """Runs the request in its session (``active_sid``): its ``sid`` query parameter's, else its ``cookie``'s; an - answer to a request naming a session other than its cookie's sets the cookie, so a page opened at ``?sid=`` keeps - its session for the requests it makes after.""" - - def __init__(self, app: Any, cookie: str) -> None: - self.app, self.cookie = app, cookie - - async def __call__(self, scope: dict, receive: Any, send: Any) -> None: - if scope["type"] != "http": - return await self.app(scope, receive, send) - request = Request(scope) - named, kept = request.query_params.get("sid"), request.cookies.get(self.cookie) - sid = session_id(named if named is not None else kept) - - async def remembered(message: dict) -> None: - if message["type"] == "http.response.start" and named is not None and session_id(kept) != sid: - MutableHeaders(scope=message).append( - "set-cookie", f"{self.cookie}={sid or DEFAULT_SID}; Path=/; HttpOnly; SameSite=Lax") - await send(message) - - token = active_sid.set(sid) - try: - await self.app(scope, receive, remembered) - finally: - active_sid.reset(token) - - class _Universe: """Stamps ``X-Universe-Now`` on every answer, and with ``clock_patch`` puts the clock script in its whole HTML (never a range of it), holding only the HTML up to its ```` (at most ``HEAD_SCAN_BYTES``), so a page streamed in chunks reaches the browser diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py index b798afcd4..b84b47be5 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/session.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/session.py @@ -17,7 +17,6 @@ import time SESSION_COOKIE = "agentenv_session" -SID_COOKIE = "agentenv_sid" SECRET_ENV = "WEBSITE_SESSION_SECRET" SESSION_TTL_S = 7 * 24 * 3600 SECRET_BYTES = 32 @@ -40,13 +39,6 @@ def session_cookie_name(environment_name: str | None) -> str: return f"{SESSION_COOKIE}_{_cookie_suffix(environment_name)}" if environment_name else SESSION_COOKIE -def sid_cookie_name(environment_name: str | None) -> str: - """The cookie in which the site named ``environment_name`` keeps the store session its browser last named: - ``agentenv_sid_``, made as ``session_cookie_name`` makes its name, ``agentenv_sid`` for a site with no name. - Its prefix is no session cookie's, so no site's store-selection cookie is another site's session cookie.""" - return f"{SID_COOKIE}_{_cookie_suffix(environment_name)}" if environment_name else SID_COOKIE - - class SessionSigner: def __init__(self, secret: str | bytes | None = None, cookie_name: str = SESSION_COOKIE) -> None: secret = secret if secret is not None else os.environ.get(SECRET_ENV) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py index 63c316a75..bf767efc3 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/store.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/store.py @@ -1,19 +1,11 @@ -"""The default state store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON -object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as. -It remembers the state it was last loaded with, so a verifier can read the initial state, the current one and the -difference between them. - -A store also keeps sessions, as CUA-Gym sites do, so browsers that each name a session id (``?sid=``) never see one -another's changes: ``session(sid)`` is a store of its own, started on first use as a copy of this store's initial -state, and the session ``default``, or none, is this store itself. ``reset`` drops every session; a session started -before a later ``load`` keeps the state it was started with.""" +"""The default data store of a simple site: named collections of JSON objects, each with an ``id``. Its data is a JSON +object of collections by name, beside which a string ``current_user`` names the user a browser starts signed in as; +the data plane loads, exports and resets it.""" from __future__ import annotations import base64 -import contextvars import copy import json -import re from pathlib import Path from typing import Any, Iterable from urllib.parse import unquote, urlparse @@ -21,17 +13,6 @@ from ..types import DataPart, FilePart, Part, TextPart CURRENT_USER = "current_user" -DEFAULT_SID = "default" -SID_MAX = 128 -_NOT_SID = re.compile(r"[^A-Za-z0-9_-]") -active_sid: contextvars.ContextVar[str | None] = contextvars.ContextVar("agentenv_website_sid", default=None) - - -def session_id(value: str | None) -> str | None: - """The session id ``value`` names, as CUA-Gym sites read ``sid``: its letters, digits, ``_`` and ``-``, at most - ``SID_MAX`` of them; None, the default session, for one that is empty or ``default``.""" - sid = _NOT_SID.sub("", value or "")[:SID_MAX] - return None if sid in ("", DEFAULT_SID) else sid class Collections: @@ -40,33 +21,12 @@ class Collections: def __init__(self, data: dict | None = None) -> None: self._collections: dict[str, list[dict]] = {} self.current_user: str | None = None - self.initial: dict = {} - self._sessions: dict[str, Collections] = {} if data: self.load(data) - self.mark_initial() def __getitem__(self, name: str) -> list[dict]: return self._collections.setdefault(name, []) - def session(self, sid: str | None) -> Collections: - """The store of session ``sid``, started as a copy of this one's initial state on first use; this store - itself for the default session.""" - sid = session_id(sid) - if sid is None: - return self - if sid not in self._sessions: - self._sessions[sid] = Collections(self.initial) - return self._sessions[sid] - - def sessions(self) -> list[str]: - """The ids of the sessions started since the store was last reset, the default one aside.""" - return list(self._sessions) - - def changed_sessions(self) -> list[str]: - """The ids of the sessions, the default one aside, whose state differs from the one they started with.""" - return [sid for sid, store in self._sessions.items() if store.diff()] - def names(self) -> list[str]: return list(self._collections) @@ -115,8 +75,6 @@ def load_parts(self, parts: Iterable[Part]) -> None: def reset(self) -> None: self._collections.clear() self.current_user = None - self.initial = {} - self._sessions.clear() def export(self) -> dict: data: dict = copy.deepcopy(self._collections) @@ -124,47 +82,6 @@ def export(self) -> dict: data[CURRENT_USER] = self.current_user return data - def mark_initial(self) -> None: - self.initial = self.export() - - def diff(self) -> dict: - return state_diff(self.initial, self.export()) - - def state(self) -> dict: - return verifier_state(self.initial, self.export()) - - -def state_diff(initial: dict, current: dict, key: Any = None) -> dict: - """Each collection that changed from ``initial`` to ``current``, as its ``added``, ``removed`` and ``changed`` - objects, matched by ``key(name, doc)``, their ``id`` by default; a changed one as its ``id``, ``before`` and - ``after``.""" - key = key or (lambda name, doc: _key(doc)) - changes: dict = {} - for name in dict.fromkeys([*initial, *current]): - if name == CURRENT_USER: - continue - before = {key(name, d): d for d in initial.get(name, [])} - after = {key(name, d): d for d in current.get(name, [])} - entry = { - "added": [d for k, d in after.items() if k not in before], - "removed": [d for k, d in before.items() if k not in after], - "changed": [{"id": d.get("id"), "before": before[k], "after": d} - for k, d in after.items() if k in before and before[k] != d], - } - if any(entry.values()): - changes[name] = entry - return changes - - -def verifier_state(initial: dict, current: dict, key: Any = None) -> dict: - """What ``/go`` answers: ``{initial_state, current_state, state_diff}``.""" - return {"initial_state": copy.deepcopy(initial), "current_state": current, - "state_diff": state_diff(initial, current, key)} - - -def _key(doc: dict) -> str: - return json.dumps(doc.get("id")) if doc.get("id") is not None else json.dumps(doc, sort_keys=True) - def part_data(part: Part) -> dict: """A part's JSON: a data part's data, a text part's text, or a file part's bytes or ``file://`` file.""" diff --git a/packages/agentenv-protocol/tests/test_website_conformance.py b/packages/agentenv-protocol/tests/test_website_conformance.py index af03260b3..9140ab39d 100644 --- a/packages/agentenv-protocol/tests/test_website_conformance.py +++ b/packages/agentenv-protocol/tests/test_website_conformance.py @@ -37,7 +37,6 @@ def notes(self) -> list: env = Notes() env.store.load({"current_user": "ann", "notes": [{"id": 1, "text": "hello"}]}) - env.store.mark_initial() return env @@ -58,12 +57,10 @@ def _handwritten(*, leak_host: bool = False, domain_cookie: bool = False, clock: clock_ops_first: tuple[str, ...] = (), leak_redirect: bool = False, stamp_rpc: bool = True, reset_keeps: bool = False, get_fails_after_reset: bool = False, rpc_reply: tuple[int, dict] | None = None, - ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None, - state_path: str | None = None, changed: bool = False) -> Starlette: + ready_answer: object = None, ready_status: int = 200, rpc_path: str | None = None) -> Starlette: """A server written without the SDK: the contract by hand, optionally broken. Its state is its ``state.data``; with ``ready_answer`` its card advertises ``website/v1``'s ``ready``, which answers it with ``ready_status``. With ``rpc_path`` its card's - ``url`` names its data plane there, not at ``/agentenv``, and with ``state_path`` its ``params.state.path`` names - where it serves its state, which shows changes since the initial state when ``changed``.""" + ``url`` names its data plane there, not at ``/agentenv``.""" state = {"items": ["a"], "url": None, "at": None, "reset": False} def now() -> datetime.datetime: @@ -98,8 +95,6 @@ async def card(request: Request) -> Response: "routes": [{"method": "GET", "path": "/items", "name": "items"}]}}] if ready_answer is not None: extensions[0]["params"]["methods"] = {"ready": {"method": "GET", "endpoint": "/ready"}} - if state_path is not None: - extensions[0]["params"]["state"] = {"path": state_path} if clock: extensions.append({"uri": "urn:agentenv:clock/v1", "params": { "endpoint": "/agentenv/ext/sync_time", "methods": { @@ -109,9 +104,6 @@ async def card(request: Request) -> Response: "capabilities": {"extensions": extensions, "operations": ["data/reset", "data/add", "data/get"]}})) - async def site_state(request: Request) -> Response: - return JSONResponse({"state_diff": {"items": {"added": ["b"]}} if changed else {}}) - async def rpc(request: Request) -> Response: body = await request.json() method = body["method"] @@ -147,8 +139,7 @@ async def unrouted(request: Request, exc: Exception) -> Response: app = Starlette(routes=[Route("/", page), Route("/healthz", health), Route("/ready", ready), Route("/items", items), Route("/items/", items_slash), Route("/.well-known/agent-env.json", card), Route(rpc_path or "/agentenv", rpc, methods=["POST"]), - Route("/agentenv/ext/sync_time", sync_time, methods=["POST"]), - *([Route(state_path, site_state)] if state_path else [])], + Route("/agentenv/ext/sync_time", sync_time, methods=["POST"])], exception_handlers={404: unrouted, 405: unrouted}) app.state.data = state return app @@ -234,28 +225,14 @@ def test_a_data_request_without_a_2xx_json_rpc_result_fails_the_data_plane(reply assert result.status == "failed" and "no JSON-RPC result" in result.detail -def test_an_sdk_site_keeps_its_initial_state_and_one_changed_since_skips_the_round_trip( - notes: WebsiteEnvironment, sdk_site: TestClient) -> None: - before = sdk_site.get("/go").json() - conformance.check("http://testserver", client=sdk_site) - assert sdk_site.get("/go").json() == before - notes.store.insert("notes", {"text": "new"}) - changed = sdk_site.get("/go").json() - assert changed["state_diff"] - result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] - assert result.status == "skipped" and "website/v1 state shows changes" in result.detail - assert sdk_site.get("/go").json() == changed - - -def test_a_named_session_with_changes_skips_the_round_trip_and_keeps_its_state( +def test_an_sdk_sites_data_round_trips_and_is_left_as_the_check_found_it_changed_or_not( notes: WebsiteEnvironment, sdk_site: TestClient) -> None: - sdk_site.get("/api/notes", params={"sid": "task-1"}) - assert {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"].status == "passed" - notes.store.session("task-2")["notes"].append({"id": 2, "text": "draft"}) - result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] - assert result.status == "skipped" and "named sessions" in result.detail - assert sdk_site.get("/go", params={"sid": "task-2"}).json()["state_diff"]["notes"]["added"] == [ - {"id": 2, "text": "draft"}] + for _ in range(2): + before = notes.store.export() + result = {r.name: r for r in conformance.check("http://testserver", client=sdk_site)}["data_plane"] + assert result.status == "passed", result + assert notes.store.export() == before + notes.store.insert("notes", {"text": "new"}) def test_the_data_plane_is_reached_at_the_cards_url() -> None: @@ -271,14 +248,6 @@ def test_the_clock_check_calls_sync_time_whichever_operation_the_card_lists_firs assert results["clock"] == "passed", results -@pytest.mark.parametrize(("changed", "status"), [(True, "skipped"), (False, "passed")]) -def test_a_card_without_state_is_read_at_the_state_path_it_names(changed: bool, status: str) -> None: - app = _handwritten(state_path="/grading/state", changed=changed) - result = {r.name: r for r in conformance.check("http://testserver", client=TestClient(app))}["data_plane"] - assert result.status == status, result - assert ("/grading/state shows changes" in result.detail) is changed - - def test_a_server_with_no_card_health_or_header_fails_the_contract_and_warns_of_the_rest() -> None: bare = Starlette(routes=[Route("/", lambda request: JSONResponse({"not": "html"}))]) results = conformance.check("http://testserver", client=TestClient(bare)) diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index e2690f233..2cc5c8b8e 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -1,6 +1,6 @@ """A site of ``@website(models=...)``: its state is a database's, SQLAlchemy models' tables on DATABASE_URL (sqlite -here) or an object's own load/export/reset, which its data plane and /go read and write, and its routes reach -through ``self.db``.""" +here) or an object's own load/export/reset, which its data plane reads and writes, and its routes reach through +``self.db``.""" from __future__ import annotations @@ -156,7 +156,7 @@ def enforce(connection, record) -> None: sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) -def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, database: str) -> None: +def test_a_row_the_gui_adds_is_the_mcp_servers_and_data_get_reports_it(dist: Path, database: str) -> None: env = helpdesk(dist, Base)() client = TestClient(env.create_app()) rpc(client, "data/add", {"parts": [{"kind": "data", "data": SEED}]}) @@ -166,12 +166,8 @@ def test_a_row_the_gui_adds_is_the_mcp_servers_and_go_reports_it(dist: Path, dat assert connection.execute(select(Ticket.title).order_by(Ticket.id)).scalars().all() == ["printer", "badge"] connection.execute(sqlalchemy.update(Ticket).where(Ticket.id == 1).values(title="printer jam")) connection.commit() - state = client.get("/go", params={"sid": "default"}).json() - assert state["initial_state"] == SEED - diff = state["state_diff"]["tickets"] - assert [t["title"] for t in diff["added"]] == ["badge"] and diff["removed"] == [] - assert diff["changed"][0]["before"]["title"] == "printer" and diff["changed"][0]["after"]["title"] == "printer jam" - assert client.get("/go", params={"sid": "other"}).json() == state + tickets = rpc(client, "data/get")["parts"][0]["data"]["tickets"] + assert [t["title"] for t in tickets] == ["printer jam", "badge"] def test_a_models_site_passes_the_conformance_check(dist: Path, database: str) -> None: @@ -232,28 +228,5 @@ def reset(self) -> None: rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"current_user": "bo", "rows": [{"id": 1}]}}]}) rows.rows.append({"id": 2}) assert rpc(client, "data/get")["parts"][0]["data"] == {"current_user": "bo", "rows": [{"id": 1}, {"id": 2}]} - assert client.get("/go").json()["state_diff"] == {"rows": {"added": [{"id": 2}], "removed": [], "changed": []}} assert env.acting_user() == "bo" assert env.db is None - - -def test_editing_a_row_a_hook_still_owns_leaves_the_initial_state(dist: Path) -> None: - class Rows: - def __init__(self) -> None: - self.rows: list = [] - - def load(self, data: dict) -> None: - self.rows += data["rows"] - - def export(self) -> dict: - return {"rows": list(self.rows)} - - def reset(self) -> None: - self.rows.clear() - - rows = Rows() - client = TestClient(helpdesk(dist, rows)().create_app()) - rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"rows": [{"id": 1, "status": "open"}]}}]}) - rows.rows[0]["status"] = "closed" - assert client.get("/go").json()["state_diff"] == {"rows": {"added": [], "removed": [], "changed": [ - {"id": 1, "before": {"id": 1, "status": "open"}, "after": {"id": 1, "status": "closed"}}]}} diff --git a/packages/agentenv-protocol/tests/test_website_environment.py b/packages/agentenv-protocol/tests/test_website_environment.py index e2ccfccbb..1cfb7c1d3 100644 --- a/packages/agentenv-protocol/tests/test_website_environment.py +++ b/packages/agentenv-protocol/tests/test_website_environment.py @@ -239,39 +239,6 @@ def test_a_sites_data_is_read_through_the_data_plane_alone(client: TestClient) - assert "state" not in website_params and "state" not in website_params.get("methods", {}) -def test_each_sid_keeps_a_state_of_its_own_started_from_the_initial_state(client: TestClient, - shop: WebsiteEnvironment) -> None: - client.post("/api/orders", json={"sku": "mug"}) - assert TestClient(shop.asgi).get("/go", params={"sid": "task-7"}).json() == { - "initial_state": SEED, "current_state": SEED, "state_diff": {}, "changed_sessions": []} - page = client.get("/", params={"sid": "task-7"}) - assert page.headers["set-cookie"].startswith("agentenv_sid_Shop=task-7;") - assert "set-cookie" not in client.get("/", params={"sid": "task-7"}).headers - client.post("/api/orders", json={"sku": "pen"}) - client.post("/api/orders", params={"sid": "other"}, json={"sku": "cup"}) - added = {sid: [o.get("sku") for d in [client.get("/go", params={"sid": sid}).json()["state_diff"]] - for o in d.get("orders", {}).get("added", [])] for sid in ("default", "task-7", "other")} - assert added == {"default": ["mug"], "task-7": ["pen"], "other": ["cup"]} - assert client.get("/go").json()["state_diff"]["orders"]["added"][0]["sku"] == "cup" - client.get("/", params={"sid": "default"}) - assert client.get("/go").json()["state_diff"]["orders"]["added"][0]["sku"] == "mug" - assert rpc(client, "data/get")["parts"][0]["data"]["orders"][-1]["sku"] == "mug" - assert sorted(shop._store.sessions()) == ["other", "task-7"] - assert sorted(client.get("/go").json()["changed_sessions"]) == ["other", "task-7"] - rpc(client, "data/reset") - assert shop._store.sessions() == [] and client.get("/go", params={"sid": "task-7"}).json()["current_state"] == {} - - -def test_a_session_id_is_sanitised_as_cua_gym_sites_do() -> None: - from agentenv_protocol.website.store import session_id - - assert [session_id(v) for v in (None, "", "default", "../a b", "x" * 200)] == [None, None, None, "ab", "x" * 128] - store = Collections({"items": [{"id": 1}]}) - store.session("s")["items"].append({"id": 2}) - assert store.session("default") is store and store.export() == {"items": [{"id": 1}]} - assert store.session("s").diff() == {"items": {"added": [{"id": 2}], "removed": [], "changed": []}} - - def test_broad_routes_leave_the_built_in_ones_answering_their_paths(dist: Path) -> None: class Broad(site(dist)): @api.get("/{slug}") @@ -379,20 +346,6 @@ def test_each_site_keeps_its_own_session_cookie_and_reads_the_legacy_one(dist: P assert TestClient(first.asgi, cookies={SESSION_COOKIE: legacy}).get("/api/orders").json() == [{"id": 1, "user": "bob"}] -def test_no_site_s_session_choice_is_another_site_s_sign_in(dist: Path, monkeypatch) -> None: - monkeypatch.setenv("ENVIRONMENT_NAME", "notes") - notes = site(dist)() - monkeypatch.setenv("ENVIRONMENT_NAME", "notes_sid") - notes_sid = site(dist)() - browser = TestClient(notes.create_app()) - browser.post("/api/orders", params={"sid": "task-1"}, json={"sku": "mug"}) - other = TestClient(notes_sid.create_app(), cookies=browser.cookies) - other.post("/api/sign-in", json={"user": "bob"}) - back = TestClient(notes.asgi, cookies=other.cookies) - assert [o["sku"] for o in back.get("/go").json()["current_state"]["orders"]] == ["mug"] - assert TestClient(notes_sid.asgi, cookies=back.cookies).get("/api/session").json()["user"] == "bob" - - def test_sites_whose_names_spell_alike_keep_their_own_cookies(dist: Path, monkeypatch) -> None: apps = [] for name in ("shop a/1", "shop_a_1"): @@ -402,14 +355,11 @@ def test_sites_whose_names_spell_alike_keep_their_own_cookies(dist: Path, monkey apps.append(env.create_app()) browser = TestClient(apps[0]) browser.post("/api/sign-in", json={"user": "bob"}) - browser.get("/", params={"sid": "task-1"}) other = TestClient(apps[1], cookies=browser.cookies) other.post("/api/sign-in", json={"user": "alice"}) - other.get("/", params={"sid": "task-2"}) - assert len(other.cookies.keys()) == 4 + assert len(other.cookies.keys()) == 2 back = TestClient(apps[0], cookies=other.cookies) assert back.get("/api/session").json()["user"] == "bob" - assert back.cookies.get("agentenv_sid_shop_a_1~030f5785d67a") == "task-1" def test_every_answer_carries_the_universe_time(client: TestClient) -> None: diff --git a/packages/agentenv-protocol/tests/test_website_notes_example.py b/packages/agentenv-protocol/tests/test_website_notes_example.py index 93eb51cc1..ad92a50df 100644 --- a/packages/agentenv-protocol/tests/test_website_notes_example.py +++ b/packages/agentenv-protocol/tests/test_website_notes_example.py @@ -197,14 +197,6 @@ def test_the_seed_loads_exports_and_resets_through_the_data_plane(client: TestCl -def test_a_page_opened_at_a_sid_keeps_that_sessions_notes_apart(client: TestClient, env: WebsiteEnvironment) -> None: - other = TestClient(env.asgi) - other.get("/", params={"sid": "task-1"}) - created = other.post("/api/notes", json={"title": "only in task-1"}).json() - assert client.get("/go", params={"sid": "task-1"}).json()["state_diff"]["notes"]["added"] == [created] - assert client.get("/go", params={"sid": "default"}).json()["state_diff"] == {} - assert all(note["title"] != "only in task-1" for note in client.get("/api/notes").json()) - GATEWAY_PREFIX = "/website/notes" diff --git a/packages/agentenv-protocol/tests/test_website_state.py b/packages/agentenv-protocol/tests/test_website_state.py deleted file mode 100644 index b4e4ec882..000000000 --- a/packages/agentenv-protocol/tests/test_website_state.py +++ /dev/null @@ -1,132 +0,0 @@ -"""What a website env's task state is: WebsiteState, the SDK's website/v1 state, and website_state and website_sites -reading it through cards.""" - -from __future__ import annotations - -import asyncio -from pathlib import Path -from unittest.mock import patch - -import httpx -import pytest -from pydantic import ValidationError -from starlette.testclient import TestClient - -from agentenv_protocol import DataPart -from agentenv_protocol.agent_env_environment import get_data -from agentenv_protocol.client import find_extension_method -from agentenv_protocol.website import ( - STATE_PATH, - WEBSITE_EXTENSION_URI, - WEBSITE_STATE_OP, - WebsiteEnvironment, - WebsiteState, - api, - website, - website_sites, - website_state, -) - -_GW = "http://gw" - - -def _site_card(name: str, *, state: bool = True) -> dict: - methods = {"state": {"method": "GET", "endpoint": f"/svc/{name}/agentenv/ext/website/state"}} if state else {} - return {"name": name, "url": f"/svc/{name}/agentenv", "capabilities": {"extensions": [ - {"uri": WEBSITE_EXTENSION_URI, "params": {"methods": methods}}]}} - - -def _routed(handler): - real = httpx.AsyncClient - return patch.object(httpx, "AsyncClient", lambda *a, **k: real(transport=httpx.MockTransport(handler))) - - -@pytest.fixture -def shop(tmp_path: Path) -> WebsiteEnvironment: - (tmp_path / "index.html").write_text("") - - @website(domains=["shop.test"], static=str(tmp_path)) - class Shop(WebsiteEnvironment): - @api.post("/api/orders") - def place(self, order: dict) -> dict: - return self.store.insert("orders", order) - - env = Shop() - env.store.load({"orders": [{"id": 1, "sku": "pen"}]}) - env.store.mark_initial() - return env - - -def test_a_website_state_ignores_fields_it_doesnt_know_and_refuses_a_state_that_isnt_an_object() -> None: - assert WebsiteState.model_validate({"current_state": {"a": 1}, "seen_at": "now"}).model_dump() == { - "initial_state": None, "current_state": {"a": 1}, "state_diff": None} - with pytest.raises(ValidationError): - WebsiteState.model_validate({"current_state": [1]}) - - -def test_an_sdk_site_advertises_state_and_answers_its_stores_state_there_and_at_the_path_it_names( - shop: WebsiteEnvironment) -> None: - client = TestClient(shop.create_app()) - card = client.get("/.well-known/agent-env.json").json() - op = find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) - params = next(e for e in card["capabilities"]["extensions"] if e["uri"] == WEBSITE_EXTENSION_URI)["params"] - assert (op["method"], op["endpoint"], params["state"]) == ("GET", "/agentenv/ext/website/state", {"path": STATE_PATH}) - client.post("/api/orders", json={"sku": "mug"}) - answer = client.get(op["endpoint"], params={"sid": "default"}).json() - assert answer == client.get(STATE_PATH).json() == {**shop.store.state(), "changed_sessions": []} - assert answer["state_diff"]["orders"]["added"][0]["sku"] == "mug" - - -def test_an_sdk_sites_state_op_answers_the_session_its_sid_names(shop: WebsiteEnvironment) -> None: - client = TestClient(shop.create_app()) - client.post("/api/orders", params={"sid": "task-1"}, json={"sku": "mug"}) - named = client.get("/agentenv/ext/website/state", params={"sid": "task-1"}).json() - assert named == client.get(STATE_PATH, params={"sid": "task-1"}).json() - assert named["state_diff"]["orders"]["added"][0]["sku"] == "mug" and named["changed_sessions"] == ["task-1"] - default = client.get("/agentenv/ext/website/state").json() - assert WebsiteState.model_validate(default).state_diff == shop.store.state()["state_diff"] - assert default["current_state"]["orders"] == [{"id": 1, "sku": "pen"}] - - -def test_a_site_with_data_hooks_of_its_own_advertises_no_state(tmp_path: Path) -> None: - @website(domains=["own.test"]) - class Own(WebsiteEnvironment): - @get_data - def export(self) -> list: - return [DataPart(data={"mine": True})] - - client = TestClient(Own().create_app()) - card = client.get("/.well-known/agent-env.json").json() - params = next(e for e in card["capabilities"]["extensions"] if e["uri"] == WEBSITE_EXTENSION_URI)["params"] - assert find_extension_method(card, WEBSITE_EXTENSION_URI, WEBSITE_STATE_OP) is None and "state" not in params - assert client.get("/agentenv/ext/website/state").status_code == 404 - - -def test_website_state_asks_the_cards_state_of_the_session_with_the_headers_given() -> None: - seen = [] - - def handler(request: httpx.Request) -> httpx.Response: - seen.append((request.url.path, dict(request.url.params), request.headers.get("authorization"))) - return httpx.Response(200, json={"initial_state": {}, "current_state": {"n": 1}, "state_diff": {"n": {}}}) - - with _routed(handler): - state = asyncio.run(website_state(_GW, _site_card("mail"), sid="s1", headers={"Authorization": "Bearer t"})) - assert asyncio.run(website_state(_GW, _site_card("mail", state=False))) is None - assert state == WebsiteState(initial_state={}, current_state={"n": 1}, state_diff={"n": {}}) - assert seen == [("/svc/mail/agentenv/ext/website/state", {"sid": "s1"}, "Bearer t")] - - -def test_an_answer_that_isnt_a_website_state_is_refused() -> None: - with _routed(lambda request: httpx.Response(200, json={"current_state": "full"})): - with pytest.raises(ValueError): - asyncio.run(website_state(_GW, _site_card("mail"))) - - -def test_website_sites_are_a_single_sites_own_card_or_a_composed_cards_website_children() -> None: - mail, calendar = _site_card("mail"), _site_card("calendar", state=False) - mcp = {"name": "mail", "url": "/svc/mcp-mail/agentenv", "capabilities": {"extensions": []}} - composed = {"name": "gw", "children_environments": [mail, mcp, calendar, "junk"]} - assert [(site.name, site.url) for site in website_sites(composed)] == [ - ("mail", "/svc/mail/agentenv"), ("calendar", "/svc/calendar/agentenv")] - assert [site.card for site in website_sites(mail)] == [mail] - assert website_sites({"name": "plain", "children_environments": [mcp]}) == [] From 3184f7940028ae1722ec998a75b19b5e28cdd47a Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 20:02:32 -0800 Subject: [PATCH 5/6] fix(website): a models= site's load matches a composite foreign key whole and restores enum columns as their members (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/database.py | 36 +++++++++--- .../tests/test_website_database.py | 56 +++++++++++++++++++ 2 files changed, 84 insertions(+), 8 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index dbc260f02..baed53719 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -17,6 +17,7 @@ import datetime import decimal +import enum import itertools import types import uuid @@ -136,20 +137,28 @@ def reset(self) -> None: def _parents_first(table: Any, rows: list[dict]) -> list[dict]: """``rows`` in their order, but each row that refers to another row of ``table`` through a foreign key of the table's - own after that row, so a constraint that isn't deferred holds at each insert; rows in a cycle keep their order.""" - links = [(fk.parent.name, fk.column.name) for fk in table.foreign_keys if fk.column.table is table] + own after that row, each key matched whole, every column of a composite one together, so a constraint that isn't + deferred holds at each insert; rows in a cycle keep their order.""" + links = [([element.parent.name for element in constraint.elements], + [element.column.name for element in constraint.elements]) + for constraint in table.foreign_key_constraints if constraint.referred_table is table] if not links: return rows - by_value = {(referenced, row.get(referenced)): index for index, row in enumerate(rows) - for _, referenced in links if row.get(referenced) is not None} + by_key: dict = {} + for index, row in enumerate(rows): + for n, (_, referenced) in enumerate(links): + key = tuple(row.get(column) for column in referenced) + if None not in key: + by_key[(n, key)] = index ordered, placed, visiting = [], set(), set() def place(index: int) -> None: if index in placed or index in visiting: return visiting.add(index) - for local, referenced in links: - parent = by_value.get((referenced, rows[index].get(local))) + for n, (local, _) in enumerate(links): + key = tuple(rows[index].get(column) for column in local) + parent = by_key.get((n, key)) if None not in key else None if parent is not None and parent != index: place(parent) visiting.discard(index) @@ -166,8 +175,9 @@ def place(index: int) -> None: def _row(table: Any, row: dict) -> dict: - """``row`` without its computed columns' values, and with each JSON string a date, time, decimal or UUID column - holds parsed to that type.""" + """``row`` without its computed columns' values, with each JSON string a date, time, decimal or UUID column holds + parsed to that type, and each an enum column holds made its member again, found by its value, as ``export`` writes + it, else by its name.""" parsed = dict(row) for column in table.columns: if column.computed is not None: @@ -178,9 +188,19 @@ def _row(table: Any, row: dict) -> dict: kind = _python_type(column) if kind in _PARSERS: parsed[column.name] = _PARSERS[kind](value.replace("Z", "+00:00") if kind is datetime.datetime else value) + elif isinstance(kind, type) and issubclass(kind, enum.Enum): + parsed[column.name] = _member(kind, value) return parsed +def _member(kind: type[enum.Enum], value: str) -> Any: + """The member of ``kind`` whose value is ``value``, else the one so named, else ``value`` as given.""" + try: + return kind(value) + except ValueError: + return kind.__members__.get(value, value) + + def _python_type(column: Any) -> type | None: try: return column.type.python_type diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index 2cc5c8b8e..36fab7fd3 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -5,6 +5,7 @@ from __future__ import annotations import datetime +import enum import sys import types from pathlib import Path @@ -39,6 +40,11 @@ class Ticket(Base): opened_at: Mapped[datetime.datetime] = mapped_column(DateTime) +class State(enum.Enum): + OPEN = "open" + CLOSED = "closed" + + SEED = {"current_user": "ann", "users": [{"id": "ann", "name": "Ann"}], "tickets": [{"id": 1, "owner": "ann", "title": "printer", "opened_at": "2031-01-02T03:04:05"}]} @@ -156,6 +162,56 @@ def enforce(connection, record) -> None: sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) +def test_a_composite_self_reference_loads_back_parents_first_matching_the_whole_key(dist: Path, database: str) -> None: + class Tenants(DeclarativeBase): + pass + + class Node(Tenants): + __tablename__ = "nodes" + tenant: Mapped[str] = mapped_column(String, primary_key=True) + id: Mapped[int] = mapped_column(Integer, primary_key=True) + parent_tenant: Mapped[str | None] = mapped_column(String, nullable=True) + parent_id: Mapped[int | None] = mapped_column(Integer, nullable=True) + __table_args__ = (sqlalchemy.ForeignKeyConstraint(["parent_tenant", "parent_id"], + ["nodes.tenant", "nodes.id"]),) + + def enforce(connection, record) -> None: + connection.execute("PRAGMA foreign_keys=ON") + + sqlalchemy.event.listen(sqlalchemy.engine.Engine, "connect", enforce) + try: + client = TestClient(helpdesk(dist, Tenants)().create_app()) + rows = [{"tenant": t, "id": i, "parent_tenant": None, "parent_id": None} for t, i in (("b", 2), ("b", 5), ("c", 2))] + rows.append({"tenant": "a", "id": 1, "parent_tenant": "b", "parent_id": 2}) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"nodes": rows}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert [(row["tenant"], row["id"]) for row in exported["nodes"]] == [("a", 1), ("b", 2), ("b", 5), ("c", 2)] + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + finally: + sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) + + +def test_an_export_of_an_enum_column_loads_back_as_its_members(dist: Path, database: str) -> None: + class Desk(DeclarativeBase): + pass + + class Issue(Desk): + __tablename__ = "issues" + id: Mapped[int] = mapped_column(Integer, primary_key=True) + state: Mapped[State] = mapped_column(sqlalchemy.Enum(State)) + + client = TestClient(helpdesk(dist, Desk)().create_app()) + rpc(client, "data/add", {"parts": [{"kind": "data", "data": {"issues": [{"id": 1, "state": "open"}, + {"id": 2, "state": "CLOSED"}]}}]}) + exported = rpc(client, "data/get")["parts"][0]["data"] + assert exported == {"issues": [{"id": 1, "state": "open"}, {"id": 2, "state": "closed"}]} + rpc(client, "data/reset") + rpc(client, "data/add", {"parts": [{"kind": "data", "data": exported}]}) + assert rpc(client, "data/get")["parts"][0]["data"] == exported + + def test_a_row_the_gui_adds_is_the_mcp_servers_and_data_get_reports_it(dist: Path, database: str) -> None: env = helpdesk(dist, Base)() client = TestClient(env.create_app()) From e7590246c2cf90d4667c892edebb95d3b9b46894 Mon Sep 17 00:00:00 2001 From: William Loo Date: Tue, 3 Nov 2026 20:54:13 -0800 Subject: [PATCH 6/6] fix(website): a models= site orders a self-referencing chain of any length parents first, without recursing (FD-4683) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/agentenv_protocol/website/database.py | 33 ++++++++++++------- .../tests/test_website_database.py | 16 +++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py index baed53719..2e18d002b 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/website/database.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/website/database.py @@ -138,7 +138,8 @@ def reset(self) -> None: def _parents_first(table: Any, rows: list[dict]) -> list[dict]: """``rows`` in their order, but each row that refers to another row of ``table`` through a foreign key of the table's own after that row, each key matched whole, every column of a composite one together, so a constraint that isn't - deferred holds at each insert; rows in a cycle keep their order.""" + deferred holds at each insert; rows in a cycle keep their order. The walk keeps its own stack, so a chain of any + length is ordered.""" links = [([element.parent.name for element in constraint.elements], [element.column.name for element in constraint.elements]) for constraint in table.foreign_key_constraints if constraint.referred_table is table] @@ -152,21 +153,31 @@ def _parents_first(table: Any, rows: list[dict]) -> list[dict]: by_key[(n, key)] = index ordered, placed, visiting = [], set(), set() - def place(index: int) -> None: - if index in placed or index in visiting: - return - visiting.add(index) + def parents(index: int) -> list[int]: + found = [] for n, (local, _) in enumerate(links): key = tuple(rows[index].get(column) for column in local) parent = by_key.get((n, key)) if None not in key else None if parent is not None and parent != index: - place(parent) - visiting.discard(index) - placed.add(index) - ordered.append(rows[index]) + found.append(parent) + return found - for index in range(len(rows)): - place(index) + for start in range(len(rows)): + if start in placed: + continue + visiting.add(start) + stack = [(start, iter(parents(start)))] + while stack: + index, pending = stack[-1] + parent = next((p for p in pending if p not in placed and p not in visiting), None) + if parent is not None: + visiting.add(parent) + stack.append((parent, iter(parents(parent)))) + continue + stack.pop() + visiting.discard(index) + placed.add(index) + ordered.append(rows[index]) return ordered diff --git a/packages/agentenv-protocol/tests/test_website_database.py b/packages/agentenv-protocol/tests/test_website_database.py index 36fab7fd3..1c623a482 100644 --- a/packages/agentenv-protocol/tests/test_website_database.py +++ b/packages/agentenv-protocol/tests/test_website_database.py @@ -162,6 +162,22 @@ def enforce(connection, record) -> None: sqlalchemy.event.remove(sqlalchemy.engine.Engine, "connect", enforce) +def test_a_chain_longer_than_the_recursion_limit_is_ordered_parents_first() -> None: + import sys + + from agentenv_protocol.website.database import _parents_first + + metadata = sqlalchemy.MetaData() + table = sqlalchemy.Table("notes", metadata, sqlalchemy.Column("id", Integer, primary_key=True), + sqlalchemy.Column("parent_id", Integer, ForeignKey("notes.id"), nullable=True)) + count = sys.getrecursionlimit() * 2 + rows = [{"id": i, "parent_id": i + 1 if i + 1 < count else None} for i in range(count)] + ordered = _parents_first(table, rows) + assert [row["id"] for row in ordered] == list(range(count - 1, -1, -1)) + cycle = [{"id": 1, "parent_id": 2}, {"id": 2, "parent_id": 1}] + assert sorted(row["id"] for row in _parents_first(table, cycle)) == [1, 2] + + def test_a_composite_self_reference_loads_back_parents_first_matching_the_whole_key(dist: Path, database: str) -> None: class Tenants(DeclarativeBase): pass