diff --git a/specs/projects/startup_sql/architecture.md b/specs/projects/startup_sql/architecture.md new file mode 100644 index 0000000..dadf20d --- /dev/null +++ b/specs/projects/startup_sql/architecture.md @@ -0,0 +1,287 @@ +--- +status: complete +--- + +# Architecture: Startup SQL + +`setup_sql: str | None` is a new `world.instance(...)` parameter. Instance creation runs it on one +writable connection to the root's file with every added node attached, after the files are in place +and before any node connection is opened. Everything else is plumbing the value through the entry +points and reporting it in the state document. + +The single-string design from the functional spec stands. A prototype against two copies of the +`agency` fixture (root plus one attached node, both with FTS5 and its triggers) confirmed every +risk the spec named: + +- Cross-node writes in one string work, FTS5 triggers fire in the attached file, and the search + index is correct in both files when reopened through `open_instance`. +- `Authorizer(read_only=False)` plus the `_harden` settings refuse every statement the spec refuses + (results under [Refusals](#refusals)). +- Running before `_open_node` means no second connection exists, so there is no locking question. + +No pushback to the fallback (per-node object) is needed. + +## Contents + +- [Flow in `InstanceManager.create`](#flow-in-instancemanagercreate) +- [New module `seahaven/setup_sql.py`](#new-module-seahavensetup_sqlpy) +- [Refusals](#refusals) +- [Error messages](#error-messages) +- [Plumbing](#plumbing) +- [State document](#state-document) +- [Docs](#docs) +- [Testing](#testing) + +## Flow in `InstanceManager.create` + +`src/seahaven/instances.py` `create(...)` gains `setup_sql: str | None = None`, keyword-only, after +`startup`. Changes, in the order the method runs: + +1. **Pre-copy checks** (beside `_check_startup_keywords`, ~:1125): `_check_setup_sql(setup_sql)` + refuses anything that is not `str` or `None` with a `WorldBug`: + `setup_sql takes a string of SQL statements separated by ';', not {type name}`. + Nothing is parsed here. +2. Clock, seed, fixture copy or blank build: unchanged. +3. **New step, directly after the copy/build and before the `_open_node` loop (~:1167):** + + ```py + if setup_sql is not None: + run_setup_sql( + setup_sql, + root=directory / composition.root.file_name, + attachments=[ + (node.schema_name, directory / node.file_name) + for node in composition.nodes + if node.depth > 0 + ], + clock=clock, + seed=node_seed(base, ROOT_PATH), + ) + ``` + + (`Composition.root` is the root `Node`. The attachment list + mirrors `Instance._attachments()` at :898, built from the composition because the runtime does + not exist yet. Factor a shared `_attachment_pairs(nodes, directory)` helper so both read the + same rule.) +4. `Instance(...)` gains `setup_sql=setup_sql`. +5. Startup hooks, `tracked_tables`: unchanged. Hooks see the committed setup rows. + +A failure in step 3 raises inside the existing `try`, so the existing `except BaseException` closes +nothing (the runtime is still empty) and removes the directory. No new cleanup code. + +`World.instance` (`src/seahaven/world.py:515`) gains the same parameter and forwards it. + +## New module `seahaven/setup_sql.py` + +One public function, two private helpers. Lives beside `sandbox.py` and reuses its `Authorizer`. + +```py +def run_setup_sql( + sql: str, + *, + root: Path, + attachments: Sequence[tuple[str, Path]], + clock: Clock, + seed: bytes, +) -> None: + """Run a caller's setup SQL against a new instance's files, in one transaction.""" +``` + +Steps: + +1. `statements = split_statements(sql)`. Empty list: return without opening anything. +2. Open `apsw.Connection(str(root))`. Apply the same settings `open_instance` applies, minus WAL + (the WAL switch belongs to `open_instance`, which runs next): + `_harden(conn)`, `pragma synchronous OFF`, `set_busy_timeout(0)`, + `register_random_functions(conn, seed, SETUP_STREAM)`, + `helper = register_clock_functions(conn, clock)`. +3. For each `(schema, path)`: `conn.execute("ATTACH DATABASE ? AS ?", (str(path), schema))`. Bound + parameters, as `open_inspection` does. Plain path, not the read-only URI. +4. `tables = _every_table(conn)`: + `SELECT name FROM pragma_table_list WHERE type IN ('table', 'virtual', 'shadow') AND name NOT LIKE 'sqlite\_%' ESCAPE '\'`. + Shadow tables must be listed: FTS5 reads and writes them through this connection, and its + virtual-table constructor reads `_config` while the caller's statement is prepared. A + direct write to a shadow table by the caller is still refused, by `SQLITE_DBCONFIG_DEFENSIVE` + (on via `_harden`): "table issues_fts_data may not be modified". +5. `conn.execute("BEGIN")`, *then* `conn.authorizer = authorizer`. The order matters: the + authorizer refuses transaction control, including the framework's own `BEGIN` and `COMMIT`. +6. For `index, statement` in `enumerate(statements, 1)`: `authorizer.reset()`, then + `for _ in conn.execute(statement): pass` (drain, so a `SELECT` runs to completion and its rows + are discarded). +7. On any exception: `conn.authorizer = None`, `ROLLBACK`, raise the `WorldBug` from + [Error messages](#error-messages) `from` the original. +8. On success: `conn.authorizer = None`, `COMMIT`. +9. `finally`: `conn.close()`, `helper.close()`. + +`SETUP_STREAM = b"setup"` is added to `src/seahaven/ids.py` beside the other streams. A stream of +its own keeps the instance's `INSTANCE_STREAM` draws identical whether or not setup SQL ran, and the +root's node seed makes it deterministic per `seed`. + +### `split_statements(sql: str) -> list[str]` + +Splits on `;` and joins pieces until `apsw.complete(piece)` is true, so a `;` inside a string +literal, a quoted identifier or a comment does not split. A trailing incomplete fragment is kept as +the last statement and fails with SQLite's own "incomplete input". Pieces that are only whitespace +are dropped; a piece that is only a comment runs as nothing. Verified in the prototype on +`"UPDATE a SET t='x;y'; -- c;\n CREATE TRIGGER ... BEGIN SELECT 1; SELECT 2; END; SELECT 3"` → three +statements. + +Splitting ourselves, rather than one `execute` of the whole string, is what makes the statement +number in an error exact: with a single `execute`, a refusal at prepare time and a failure at step +time are indistinguishable. + +### `SetupAuthorizer(Authorizer)` + +A subclass that records the first refused action as `(action, third, fourth)` on top of what the +base class does, so the error message can be written from the action code rather than by parsing +the base class's refusal text. `__call__` calls `super().__call__(...)`, and when the answer is +`SQLITE_DENY` and nothing is recorded yet, records it. `reset()` clears it. Constructed with +`tables` and `read_only=False`; functions default to `ALLOWED_FUNCTIONS`, the `run_sql` allowlist +(includes `random`, `randomblob`, `datetime`). + +`MAX_VALUE_BYTES` is not applied: it bounds an agent's statement, and setup SQL comes from the eval. + +## Refusals + +Prototype results with this design (refused action → what SQLite or the authorizer reported): + +| Statement | Refused as | +|---|---| +| `CREATE TABLE`, `DROP TABLE`, `CREATE INDEX` | write to `sqlite_master` | +| `ALTER TABLE` | `SQLITE_ALTER_TABLE` | +| `CREATE TRIGGER`, `CREATE VIEW` | `SQLITE_CREATE_TRIGGER`, `SQLITE_CREATE_VIEW` | +| `BEGIN`, `COMMIT`, `ROLLBACK` | `SQLITE_TRANSACTION` | +| `SAVEPOINT`, `RELEASE` | `SQLITE_SAVEPOINT` | +| `ATTACH`, `DETACH` | `SQLITE_ATTACH`, `SQLITE_DETACH` | +| `PRAGMA journal_mode = ...`, `PRAGMA foreign_keys = OFF` | `SQLITE_PRAGMA` not on the read-only list | +| `SELECT load_extension(...)` | `SQLITE_FUNCTION` not on the allowlist | +| `INSERT INTO issues_fts_data ...` | SQLite (defensive mode): "may not be modified" | +| `UPDATE sqlite_master ...` | SQLite: "may not be modified" | + +Allowed: `INSERT`, `UPDATE`, `DELETE`, `SELECT`, `INSERT ... SELECT` across nodes, FTS5 commands +such as `INSERT INTO issues_fts(issues_fts) VALUES('rebuild')`, read-only pragmas. Foreign keys are +on, so `DELETE FROM users` in `agency` fails with SQLite's constraint error. + +## Error messages + +Every failure is one `WorldBug`, raised from the original exception, with this shape: + +``` +setup_sql statement {i} of {n} {was refused|failed}: {reason} {fix} + {statement, whitespace collapsed, cut at 200 characters with "..."} +``` + +`_explain(recorded, error, schemas)` writes `reason` and `fix`: + +| Case | reason / fix | +|---|---| +| write to `sqlite_master`, or a `CREATE_*`, `DROP_*`, `ALTER_TABLE`, `REINDEX`, `ANALYZE` action | "it changes the schema." / "setup_sql may only read and write rows (INSERT, UPDATE, DELETE, SELECT). Change the schema in the world and bump its version." | +| `TRANSACTION`, `SAVEPOINT` | "it controls the transaction." / "setup_sql already runs in one transaction that Seahaven opens and commits; remove BEGIN, COMMIT and SAVEPOINT." | +| `ATTACH`, `DETACH` | "it attaches or detaches a database." / "Every node is already attached: name the root's tables unqualified and an added node's as ." + `" (this world: payments, payments__tax)."` or `" (this world adds none)."` | +| `PRAGMA` | "PRAGMA {name} can change the connection." / "Only read-only pragmas such as table_info are allowed." | +| `FUNCTION` | "function {name}() is not allowed." / "setup_sql has the functions run_sql has." | +| any other recorded action | "{the base class's description} is not allowed." / "setup_sql may only read and write rows." | +| nothing recorded (a plain SQLite error) | SQLite's own message. / If it starts with "no such table" and the world adds nodes: "An added node's tables are named .
(this world: ...)." Otherwise no fix. | + +"was refused" for a recorded refusal, "failed" for a plain SQLite error. + +## Plumbing + +From the touchpoint survey; each item is a one-line change unless noted. + +**In process** +- `world.py:515` `World.instance`: parameter (after `startup`), docstring line, forward. +- `instances.py:1074` `InstanceManager.create`: as above. +- `instances.py:290` `Instance.__init__`: `setup_sql: str | None`; store `self.setup_sql` beside + `self.startup` (:334). Public read-only attribute, listed in `reference/api.md`'s `inst.*` table. +- `pytest_plugin.py:47` `_MARKER_SIGNATURE`: add `setup_sql=None` in `World.instance` order. + +**OpenEnv** (`src/seahaven/openenv/env.py`) +- `RESET_ORDER` (:342): add `"setup_sql"` after `"startup"`. This also orders the console form. +- `SeahavenResetRequest` (:353): `setup_sql: str | None = Field(default=None, description=...)`. + `for_world` needs no change. +- `SeahavenEnv.reset` (:480): parameter before `**unknown`; forward at :581; docstring. +- `SeahavenState` (:317): `setup_sql: str | None = Field(default=None, description=...)` before + `call_count`; update the count in the comment at :275 ("the six it answers null for"). + +**HTTP and MCP**: no code change. `RESET_OPTION_KEYS` (`cli/mcp.py:90`) is derived from +`inspect.signature(World.instance)`, and `check_reset_options`, `_make` and the PUT handler all use +it or pass `**kwargs`. Update the stale comment at `cli/mcp.py:88` ("pins the five names"). + +**Console**: no code change. A `str | None` field titled "Setup Sql" unwraps to a string, and +`kindOf` maps a title matching `/sql/` to a multi-line text box. Add the field to the mock reset +schema in `tools/console/mock/server.mjs` and the form order in `tools/console/e2e/smoke.mjs`. + +## State document + +`src/seahaven/state.py` `envelope()` (:80): add +`"setup_sql": instance.setup_sql if instance is not None else None` directly after `startup`. The +value is the string exactly as given (an empty string stays `""`), `None` when none was given and +before the first `reset`. It is not run through `serialise`: it is already a string. + +The change log needs no code: sessions open per call (`instances.py:745`), and setup runs before the +instance exists. A test pins it (below). + +## Docs + +Per the AGENTS.md docs style, short and checked against the code: + +- `serving_and_openenv.md`: a row in the reset table (:127); a row in the `SeahavenState` table + (:318); add `setup_sql` to the pre-reset null list (:339); one example reset message with + `setup_sql`. +- `db_schema_and_fixtures.md`: a short section, "Adjusting a fixture per episode": what + `setup_sql` is for, rows only, the composed-world table names, and when a new fixture or a startup + hook is the better tool. One `python` example that runs (projecttracker-free, on a tiny inline + world, so `test_docs_examples.py` executes it). +- `state.md`: example JSON (:61) and envelope table row (:79). +- `composition.md`: one sentence and a fragment showing `payments.charges` in `setup_sql`. +- `reference/api.md`: `world.instance` row (:123), marker signature (:628), `inst.setup_sql` row. +- `reference/cli.md`: mention under `--reset-options` (:204). + +## Testing + +New file `tests/test_setup_sql.py`. Each test drives `world.instance(...)` unless it says otherwise, +per the AGENTS.md rule to cover the real entry point. Worlds are small inline worlds, one leaf and +one composed (host plus an added `payments` world, plus one with an FTS5 table and sync triggers), +in the style of `tests/test_composite_instance.py`. + +Behaviour: +- rows written by `setup_sql` are visible to the first tool call and to `inst.inspect()`; +- several statements run in order (a later statement sees an earlier one's row); +- `;` inside a string literal does not split; +- a composed world: an unqualified name writes the root, `payments.charges` writes the node, and + `INSERT INTO payments.charges ... SELECT ... FROM ` works; +- FTS5: an `UPDATE` through `setup_sql` is found by an FTS `MATCH` in the first tool call; +- a startup hook sees setup rows (hook reads a row the SQL inserted); +- same `seed` + same `setup_sql` → identical `random()` results; and an instance's own + `random()` draws in a tool call are the same with and without `setup_sql` (the separate stream); +- `setup_sql` rows are not in `inst.change_log()`, and `call_count` is 0 after creation; +- `inst.state()["setup_sql"]` is the string as given; `None` when omitted; +- `""`, whitespace and a comment-only string create the instance and change nothing. + +Refusals and errors (each asserts the `WorldBug` message: statement number, reason, fix): +- one test per row of the [Refusals](#refusals) table, parametrized; +- a refusal in statement 2 rolls back statement 1, and no instance directory is left behind; +- syntax error, constraint failure, and "no such table" with the composed-world hint; +- a non-string `setup_sql` is refused before anything is copied (no directory created). + +Unit tests: `split_statements` cases (literal `;`, quoted identifier, comment, trigger body, +trailing fragment, empty); `_explain` per row of the error table. + +Entry points: +- OpenEnv: `SeahavenEnv.reset(fixture=..., setup_sql=...)` in process, and one WebSocket round trip + through the served app (as `tests/test_server.py` does), asserting the row and the state field; + a failing `setup_sql` leaves the session fresh and open for another `reset`. +- HTTP: a `PUT` body with `setup_sql` creates an instance with the row. +- MCP: `--reset-options '{"setup_sql": "..."}'` accepted (in the mcp environment's tests). +- projecttracker: one test in `worlds/projecttracker/tests` resets `agency` with `setup_sql` + inserting a user and `startup={"user_id": }`, then `create_issue` is attributed to them. + +Pins to update (from the survey): `tests/test_env.py` `RESET_PARAMETERS` (:256) and +`DECLARED_DESCRIPTIONS` (:1010); `tests/test_reset_schema.py` property order (:86); +`tests/test_cli_mcp.py` key list (:285) and messages (:316); `tests/test_mcp_process.py:587`; +`tests/state_v1.schema.json` (`required` and `properties`, `["string", "null"]`); +`tests/test_state.py` key order (:66) and no-instance nulls (:346); `tests/test_server.py` +`expected_document` (:445); `tests/test_client.py` `DOCUMENT_FRAME` (:112); +`tests/test_pytest_plugin.py:629` (automatic once the marker signature is updated); +`tests/test_world.py:791` framework-parameter list. diff --git a/specs/projects/startup_sql/functional_spec.md b/specs/projects/startup_sql/functional_spec.md new file mode 100644 index 0000000..43b894d --- /dev/null +++ b/specs/projects/startup_sql/functional_spec.md @@ -0,0 +1,159 @@ +--- +status: complete +--- + +# Functional Spec: Startup SQL + +## Summary + +`setup_sql` is a new parameter of `world.instance(...)`, and therefore of OpenEnv `reset`, of the +HTTP API's instance creation and of `seahaven mcp --reset-options`. It holds SQL that the framework +runs against the new instance after the fixture is copied (or the blank schema is built) and before +any startup hook. An eval uses it to adjust the starting state of a served world without writing +code. + +It reuses the startup step of instance creation. It is not a startup keyword and not an +`instance_startup` hook: `startup` stays the world's own keyword namespace. + +## The interface + +The single string below is the preferred design. It depends on a new writable connection that +attaches every node. If the architecture step finds that connection adds disproportionate +complexity or risk (locking, triggers, seeding, attach limits), it may push back to the fallback: +`setup_sql: str | Mapping[str, str] | None`, where a string is the root's SQL and an object maps a +canonical node path to that node's SQL, run on each node's own connection. The fallback needs a +per-world reset schema and a console change to render well; the architecture must cover both if +it is chosen. + +```py +world.instance( + fixture=None, *, seed=None, now=None, clock_mode=None, state_format=None, + control_tools=False, startup=None, + setup_sql: str | None = None, +) +``` + +Over OpenEnv: + +```jsonc +{"type": "reset", "data": { + "fixture": "agency", + "seed": 7, + "setup_sql": "UPDATE issues SET due_at = '2026-10-01T00:00:00.000Z' WHERE key = 'ENG-12'; DELETE FROM comments WHERE issue_id = (SELECT id FROM issues WHERE key = 'ENG-12');", + "startup": {"user_id": "a0b45f75-0917-49b9-9efd-d0279f2dd73d"} +}} +``` + +`setup_sql` is one string. It may hold several statements separated by `;`, and they run in the +order written. `null` or omitted means no setup SQL. An empty or whitespace-only string is accepted +and runs nothing. + +### Table names in a composed world + +The SQL runs on one connection that sees every node, named as `inst.inspect()` names them: + +- the root's tables unqualified (`issues`), or as `main.issues`; +- an added node's tables under a schema named by its canonical path, with `/` replaced by `__`: + `payments.charges`, `payments__tax.rates`. + +A statement may read and write across nodes, for example +`INSERT INTO payments.charges (...) SELECT ... FROM orders`. A world that adds no other world only +ever uses unqualified names. + +## Behaviour + +### When it runs + +Instance creation, in order: + +1. Copy the fixture, or build the blank schema. +2. **Run `setup_sql`** in its own transaction, on a writable connection to the root's file with + every other node attached writable under its schema name. Commit, then close the connection. +3. Open every node's transaction and run the startup hooks, as today. A hook sees the rows + `setup_sql` wrote: for example, SQL that inserts a user and `startup={"user_id": }` + together work. +4. Commit, and the instance is ready for its first call. + +Step 2 commits before step 3 so that its write locks on the attached files cannot block a hook. +Creation is still all or nothing: a failure at any step removes the whole instance directory, as +a failing startup hook does today. + +### The change log and the state document + +- Rows `setup_sql` writes are **not** in the change log. Change-log sessions are opened only per + tool call; creation runs before any exists. They are part of the starting state, as rows a startup + hook writes are today. A grader folding the log sees only what the agent's calls changed. + `call_count` is unchanged. +- The state document's envelope gains a field, `setup_sql`, beside `startup`: the string exactly as + given, or `null` when there was none (and before the first `reset` over a server). +- `fixture.nodes[path].file_sha256` keeps naming the fixture file each node was copied from. A + reader reconstructs the starting state from `fixture` plus `setup_sql` plus `startup`. + +### What SQL is allowed + +The docs tell an author to use `INSERT`, `UPDATE` and `DELETE`, and to change the schema with a new +world version, not with setup SQL. + +Runtime enforcement is part of v1, through `seahaven.sandbox.Authorizer`, which already +default-denies everything but row reads, row writes and allowlisted functions. It refuses schema +changes (`CREATE`, `DROP`, `ALTER`), `ATTACH`/`DETACH`, `PRAGMA` writes, transaction control +(`BEGIN`, `COMMIT`, `SAVEPOINT`) and functions outside the `run_sql` allowlist. Read and write +access covers every table of every node. + +A refusal fails loudly: instance creation aborts with a `WorldBug` whose message names the +statement's position, what was refused, and what to do instead. For example: "setup_sql statement +2 was refused: CREATE TABLE is a schema change. setup_sql may only read and write rows (INSERT, +UPDATE, DELETE); change the schema in the world and bump its version." + +Transaction control would end the setup transaction early, and `ATTACH` and `DETACH` would change +the setup connection's attachments, which are the framework's. + +`random()` and `randomblob()` draw from the root node's seeded stream, whichever node the row lands +in, so the same `seed` and the same `setup_sql` give the same starting state. + +### Errors + +A failure aborts instance creation with `seahaven.WorldBug` and nothing is left behind, exactly as +when a startup hook raises. Over OpenEnv the session is +left fresh (no instance, no episode) and open for another `reset`; the frame is `EXECUTION_ERROR`. + +Refused before anything is copied (the checks `world.instance` already runs first): + +- `setup_sql` is not a string or `null`. + +Refused while running: + +- A statement SQLite rejects (syntax, constraint, missing table). The message names the + statement's position in the string (1-based) and SQLite's own text. +- A statement the enforcement refuses, named the same way. + +## Every entry point + +`setup_sql` is a `world.instance` parameter, so each place that forwards instance options takes it: + +| Entry point | How it is passed | +|---|---| +| `world.instance(...)` | `setup_sql=` | +| OpenEnv `reset` (WebSocket and `SeahavenEnv.reset`) | `"setup_sql"` key; published in the reset schema from `GET /schema` | +| HTTP API instance creation (`PUT` body) and `seahaven serve` reset options | `"setup_sql"` key | +| `seahaven mcp --reset-options` | `"setup_sql"` key | +| `/ui` reset form | rendered from the reset schema as a multi-line text box; no console change | + +It is always available. There is no operator flag to turn it off. + +## Out of scope + +- Bound parameters (`:now`, `:seed`). Statements are plain literal SQL. +- Setup SQL on `add_world(...)` (a host fixing a child's SQL at composition time). +- A per-world opt-out. +- Turning setup SQL into a fixture (freezing). Authors who reuse the same setup across many evals + should make a fixture instead; the docs say so. + +## Documentation + +- `serving_and_openenv.md`: a row in the reset argument table and one example. +- `db_schema_and_fixtures.md` or `authoring.md`: when to use `setup_sql` versus a fixture or a + startup hook, and the advice to write rows only. +- `state.md`: the new envelope field. +- `reference/api.md`: the `world.instance` signature. +- `composition.md`: the schema names an added node's tables have in `setup_sql`. diff --git a/specs/projects/startup_sql/implementation_plan.md b/specs/projects/startup_sql/implementation_plan.md new file mode 100644 index 0000000..41985b2 --- /dev/null +++ b/specs/projects/startup_sql/implementation_plan.md @@ -0,0 +1,23 @@ +--- +status: complete +--- + +# Implementation Plan: Startup SQL + +## Phases + +- [x] Phase 1: Core. `seahaven/setup_sql.py` (`run_setup_sql`, `split_statements`, + `SetupAuthorizer`, `_explain`), `SETUP_STREAM`, the `setup_sql` parameter on `World.instance`, + `InstanceManager.create` and `Instance` (stored as `inst.setup_sql`), the pre-copy type check, and + the pytest marker signature. Tests in `tests/test_setup_sql.py` through `world.instance(...)`: + behaviour, composed worlds, FTS5, startup hooks seeing setup rows, seeding, change log, every + refusal and error message, rollback leaving no directory, and the unit tests. +- [x] Phase 2: Entry points and state document. `setup_sql` in the state document envelope and + `SeahavenState`; OpenEnv `reset`, `SeahavenResetRequest` and `RESET_ORDER`; the stale + `cli/mcp.py` comment; console mock schema and smoke order; every pinned test and + `tests/state_v1.schema.json`. Tests for OpenEnv (in process and over the WebSocket), HTTP `PUT`, + `seahaven mcp --reset-options`, and the projecttracker test (SQL inserts a user, `startup` names + them, `create_issue` is attributed to them). +- [x] Phase 3: Docs. Updates to six existing pages, no new page: `serving_and_openenv.md`, `db_schema_and_fixtures.md` (new short section with + a runnable example), `state.md`, `composition.md`, `reference/api.md`, `reference/cli.md`, per + the AGENTS.md docs style. diff --git a/specs/projects/startup_sql/phase_plans/phase_1.md b/specs/projects/startup_sql/phase_plans/phase_1.md new file mode 100644 index 0000000..ad4988f --- /dev/null +++ b/specs/projects/startup_sql/phase_plans/phase_1.md @@ -0,0 +1,127 @@ +--- +status: complete +--- + +# Phase 1: Core + +## Overview + +`setup_sql: str | None` becomes a parameter of `world.instance(...)`. Instance creation runs it on +one writable connection to the root's file with every added node attached, after the files are in +place and before any node connection is opened, in one transaction, under an authorizer that allows +only row reads and row writes. Startup hooks then see the committed rows. Any failure is a +`WorldBug` naming the statement, and creation leaves no directory behind. + +The entry points (OpenEnv, the state document) and the docs are phases 2 and 3. HTTP and MCP derive +their keys from the `World.instance` signature, so they take the key from this phase on; the one +test that pins those keys is updated here so the suite stays green. + +## Steps + +1. `src/seahaven/ids.py`: add `SETUP_STREAM = b"setup"` beside the other streams, to `__all__`, and + one sentence in the stream comment saying it is the setup connection's door. +2. New `src/seahaven/setup_sql.py`: + + ```py + __all__ = ["SetupAuthorizer", "run_setup_sql", "split_statements"] + + def run_setup_sql( + sql: str, *, root: Path, attachments: Sequence[tuple[str, Path]], clock: Clock, seed: bytes + ) -> None: ... + + def split_statements(sql: str) -> list[str]: ... + + @dataclass(frozen=True) + class Refused: + action: int + third: str | None + fourth: str | None + description: str # the base class's refusal text + + class SetupAuthorizer(Authorizer): + refused: Refused | None + def reset(self) -> None: ... + def __call__(self, action, third, fourth, database, trigger, /) -> int: ... + + def _explain(refused: Refused | None, error: apsw.Error, schemas: Sequence[str]) -> tuple[str, str] + def _every_table(conn: apsw.Connection) -> list[str] + def _failure(index, count, statement, refused, error, schemas) -> WorldBug + ``` + + - `split_statements`: split on `;`, join pieces until `apsw.complete` is true, drop + whitespace-only pieces, keep a trailing fragment as the last statement. + - `run_setup_sql`: per architecture — open, `_harden`, `synchronous OFF`, busy timeout 0, random + functions on `SETUP_STREAM`, clock functions, `ATTACH ... AS ?` with bound parameters, list every + table (including FTS5 shadow tables), `BEGIN`, install the authorizer, run each statement on + one cursor and drain it, remove the authorizer, `COMMIT`; on an `apsw.Error` roll back and raise + the `WorldBug` from it; always close the connection and the clock helper. + - `SetupAuthorizer`: `read_only=False`, `ALLOWED_FUNCTIONS`; additionally allows the read-only + pragmas `db._READ_ONLY_PRAGMAS` (the list the inspection connection uses), so the error + message "Only read-only pragmas such as table_info are allowed" is true. Records the first + refusal of a statement; `reset()` clears it. + - `_explain`: the reason and fix per the architecture's error table; message shape + `setup_sql statement {i} of {n} {was refused|failed}: {reason} {fix}\n {statement}` with the + statement whitespace-collapsed and cut at 200 characters with `...`. +3. `src/seahaven/instances.py`: + - `_attachment_pairs(nodes: Iterable[Node], directory: Path) -> list[tuple[str, Path]]`, used by + `Instance._attachments()` and by `create`. + - `_check_setup_sql(setup_sql: object) -> None`: `WorldBug("setup_sql takes a string of SQL + statements separated by ';', not {type}")` unless `str` or `None`; called beside + `_check_startup_keywords`, before any directory exists. + - `create(..., setup_sql: str | None = None)`: after the copy or blank build and before the + `_open_node` loop, `run_setup_sql(...)` when `setup_sql is not None`, seeded with + `node_seed(base, ROOT_PATH)`. Docstring line. + - `Instance.__init__(..., setup_sql: str | None)`: stored as `self.setup_sql`. +4. `src/seahaven/world.py` `World.instance`: `setup_sql: str | None = None` after `startup`, + docstring sentence, forwarded. +5. `src/seahaven/pytest_plugin.py` `_MARKER_SIGNATURE`: add `setup_sql=None` after `startup=None`. +6. Pins the new parameter breaks: `tests/test_cli_mcp.py` key list; `tests/test_world.py` + framework-parameter list; `SETUP_STREAM` added to the door lists in `tests/test_ids.py` and + `tests/test_composite_instance.py`. + +## Tests + +New `tests/test_setup_sql.py`, every behaviour test through `world.instance(...)`, on small inline +worlds (a leaf `notes`-style world, a host adding `payments` which adds `tax`, and an FTS5 world). + +Behaviour: +- `test_rows_written_by_setup_sql_are_there_for_the_first_call`: inserted row read by the first + tool call and by `inst.inspect()`. +- `test_statements_run_in_the_order_written`: a later `UPDATE` sees an earlier `INSERT`'s row. +- `test_a_semicolon_inside_a_literal_does_not_split`. +- `test_unqualified_names_write_the_root_and_qualified_names_write_a_node`: `payments.charges` and + `payments__tax.rates`. +- `test_one_statement_reads_one_node_and_writes_another`: `INSERT INTO payments.charges ... + SELECT ... FROM `. +- `test_setup_sql_runs_on_a_copied_fixture`: a frozen fixture plus setup SQL. +- `test_an_update_through_setup_sql_is_found_by_fts5`: `UPDATE` then a `MATCH` in the first call; + the old term no longer matches. +- `test_a_startup_hook_sees_the_rows_setup_sql_wrote`: SQL inserts a user, `startup={"user_id": + ...}` names them, the hook reads the row. +- `test_the_same_seed_and_setup_sql_give_the_same_random_rows`, and + `test_setup_sql_draws_from_a_stream_of_its_own`: an instance's own `random()` in a tool call is + the same with and without setup SQL that drew from `random()`; and different seeds differ. +- `test_the_clock_functions_read_the_instances_clock`: `datetime('now')` is the instance's `now`. +- `test_setup_sql_rows_are_not_in_the_change_log`: `change_log()` empty and `call_count == 0`. +- `test_setup_sql_is_kept_on_the_instance`: `inst.setup_sql` is the string as given, `None` when + omitted. +- `test_an_empty_string_whitespace_or_a_comment_changes_nothing` (parametrized). +- `test_a_select_runs_and_changes_nothing`, and `test_read_only_pragmas_and_fts5_commands_are_allowed`. + +Refusals and errors (each asserts the whole `WorldBug` message shape: statement number of count, +"was refused"/"failed", reason, fix): +- `test_a_refused_statement_names_what_was_refused` parametrized over every row of the + architecture's Refusals table. +- `test_a_refusal_rolls_back_earlier_statements_and_leaves_no_directory`. +- `test_a_syntax_error_names_the_statement_and_sqlites_message`, + `test_a_constraint_failure_fails_the_instance`, + `test_a_missing_table_in_a_composed_world_names_the_schemas`, and the same in a leaf world with no + hint. +- `test_a_non_string_setup_sql_is_refused_before_anything_is_copied` (no directory created). +- `test_a_long_statement_is_cut_in_the_message`. + +Unit tests: +- `split_statements` cases: literal `;`, quoted identifier, comment, trigger body, trailing fragment, + empty and whitespace, comment-only piece. +- `_explain` per row of the error table, including the ATTACH fix with and without added nodes. +- `SetupAuthorizer.reset` clears the recorded refusal and records only the first. diff --git a/specs/projects/startup_sql/phase_plans/phase_2.md b/specs/projects/startup_sql/phase_plans/phase_2.md new file mode 100644 index 0000000..324ed7e --- /dev/null +++ b/specs/projects/startup_sql/phase_plans/phase_2.md @@ -0,0 +1,61 @@ +--- +status: complete +--- + +# Phase 2: Entry points and state document + +## Overview + +Phase 1 made `setup_sql` a `world.instance(...)` parameter. HTTP `PUT` and +`seahaven mcp --reset-options` already take it, because both read their keys off that signature. +This phase puts it on the OpenEnv wire (the `reset` signature and the published reset schema), +reports it in the state document's envelope and in `SeahavenState`, updates the console mock and +smoke test, and updates every test that pins the envelope or the reset keys. Each entry point gets +a test that drives `setup_sql` through it. + +## Steps + +1. `src/seahaven/state.py` `envelope()`: add + `"setup_sql": instance.setup_sql if instance is not None else None` directly after `startup`. +2. `src/seahaven/openenv/env.py`: + - `SeahavenState`: `setup_sql: str | None = Field(default=None, description=...)` after + `startup`; the field comment's count of null-defaulted fields becomes seven. + - `RESET_ORDER = ("fixture", "startup", "setup_sql", "now", "clock_mode", "state_format")`. + - `SeahavenResetRequest`: `setup_sql: str | None = Field(default=None, description=...)`. + `for_world` is unchanged. + - `SeahavenEnv.reset`: `setup_sql: str | None = None` after `startup`, before `**unknown`; + forwarded to `InstanceManager.create`; one docstring clause. +3. `src/seahaven/cli/mcp.py`: the comment over `RESET_OPTION_KEYS` no longer says "five names". +4. Console: `tools/console/mock/server.mjs` reset schema gains `setup_sql` (titled "Setup Sql") + after `startup`; `tools/console/e2e/smoke.mjs` form order includes "Setup Sql" between the + startup keywords and "Now", and checks it renders as a `textarea`. No console source change. +5. Pins: `tests/test_env.py` `RESET_PARAMETERS` and `DECLARED_DESCRIPTIONS`; + `tests/test_reset_schema.py` property order; `tests/state_v1.schema.json` (`required`, + `properties` as `["string", "null"]`); `tests/test_state.py` key order and no-instance nulls; + `tests/test_server.py` `EPISODE` and `expected_document`; `tests/test_client.py` + `DOCUMENT_FRAME` and the pre-reset nulls. + +## Tests + +- `tests/test_state.py::test_the_setup_sql_is_reported_exactly_as_given`: the string as given, + `""` stays `""`, `None` when omitted, and the setup row is not in the log. +- `tests/test_state.py::test_a_built_in_document_validates_against_the_published_schema`: now with + `setup_sql`, so the string branch of the schema is exercised. +- `tests/test_env.py::test_setup_sql_reaches_the_instance_through_reset`: in process, on a fixture; + rows visible to the first call, `state.setup_sql`, empty log. +- `tests/test_env.py::test_a_failing_setup_sql_leaves_the_session_as_a_fresh_one`: `WorldBug`, no + instance, old directory gone, `setup_sql` null, another `reset` works. +- `tests/test_server.py::test_setup_sql_reaches_the_instance_over_the_wire`: a refused statement is + an `EXECUTION_ERROR` frame; the next reset with good SQL has the row and the state field. +- `tests/test_server.py` whole-document tests: `setup_sql` is in `EPISODE`, so the document over the + WebSocket (typed and stock client) carries it, and its row is not in the log. +- `tests/test_reset_schema.py`: `setup_sql` in the published order, as a nullable string titled + "Setup Sql". +- `tests/test_http_server.py::test_a_put_body_with_setup_sql_starts_the_instance_with_its_rows`, and + two new rows in the refused-`PUT` table (non-string, refused statement) answering 400. +- `tests/test_cli_mcp.py::test_setup_sql_is_a_reset_option` and + `tests/test_mcp_process.py::test_reset_options_carry_setup_sql_to_the_instance` (a real MCP + process; the first call reads the row). +- `worlds/projecttracker/tests/test_issues.py::test_a_user_setup_sql_inserts_can_be_the_viewer_who_files_an_issue`: + `agency` plus SQL inserting a user plus `startup={"user_id": ...}`; `create_issue` is attributed + to that user, and `users` is not in the change log. diff --git a/specs/projects/startup_sql/phase_plans/phase_3.md b/specs/projects/startup_sql/phase_plans/phase_3.md new file mode 100644 index 0000000..6c1fe46 --- /dev/null +++ b/specs/projects/startup_sql/phase_plans/phase_3.md @@ -0,0 +1,52 @@ +--- +status: complete +--- + +# Phase 3: Docs + +## Overview + +Phases 1 and 2 built `setup_sql` and put it on every entry point and in the state document. This +phase documents it in the six existing pages the plan names, with no new page, per the AGENTS.md +docs style: short, plain sentences, every name checked against the code, examples tested. + +## Steps + +1. `db_schema_and_fixtures.md`: a new section, "Adjusting a fixture per episode", after "Building a + fixture" and before "The generator is committed", with a row in the page's table of contents. + Content: what `setup_sql` is (SQL run on the new instance after the copy and before the startup + hooks, in one transaction), write rows only and change the schema with a new world version, a + composed world's table names (link to `composition.md`), and when a fixture or a startup hook is + the better tool. One runnable `python` example on an inline world: `world.instance(..., + setup_sql=...)` with two statements, the rows visible through `inst.inspect()`, + `inst.change_log() == []`, and `inst.state()["setup_sql"]` as given. +2. `serving_and_openenv.md`: + - the reset argument table gains a `setup_sql=` row after `startup=`; + - the paragraph after it ("One served world covers every scenario a fixture and a startup hook + can express") names setup SQL too; + - one example reset message with `setup_sql` beside the existing `startup` example; + - the `SeahavenState` table gains a `setup_sql` row after `startup`; + - the pre-reset null list gains `setup_sql`. +3. `state.md`: + - the example envelope gains `"setup_sql": null` after `startup`; + - the envelope table gains a `setup_sql` row; + - "Startup writes are not in the log" covers `setup_sql` rows too; + - "The state the episode started from" names `setup_sql` as part of the starting state, beside + `startup`. +4. `composition.md`: one sentence and a `py` fragment in "One instance, many stores" or "What an eval + sees": `setup_sql` names tables as `inst.inspect()` does, `payments.charges` for an added node. +5. `reference/api.md`: + - the `world.instance` row gains `setup_sql=None` and one clause; + - the `inst.*` table gains `inst.setup_sql`; + - the marker signature gains `setup_sql=None`; + - a short list of what `setup_sql` refuses (the reference home for the refusal list). +6. `reference/cli.md`: under `seahaven mcp`, one example `--reset-options` line or sentence naming + the `"setup_sql"` key. + +## Tests + +- `tests/test_docs_examples.py` runs the new `python` example and parses the new `py` fragment; it + also resolves `inst.setup_sql` in the api table against a live instance. +- `tests/test_docs.py` checks every new anchor link resolves (the new table-of-contents row and any + cross-page link to the new section). +- No new test file: the behaviour is covered by phases 1 and 2. diff --git a/specs/projects/startup_sql/project_overview.md b/specs/projects/startup_sql/project_overview.md new file mode 100644 index 0000000..9a6fd0c --- /dev/null +++ b/specs/projects/startup_sql/project_overview.md @@ -0,0 +1,23 @@ +--- +status: complete +--- + +# Startup SQL + +Let a caller using OpenEnv run setup on a world when an instance starts, without writing code. Today +the only per-episode setup is a startup hook, and a hook is code the world author writes: an eval +can pass a hook data through `reset(startup={...})`, but it cannot change the world in a way the +author did not write a hook for. + +The caller sends SQL with `reset`, and the framework runs it against the new instance before the +first tool call. One served world then covers any starting state a fixture plus some SQL can +express, without a new fixture or a new hook for each scenario. + +This is a framework feature that every world gets for free, not something each world opts in to. + +## Design decision + +Same machinery as startup hooks, new key. The SQL runs inside the startup step of instance creation +(same transaction, same failure handling, not in the change log), but it is a framework parameter, +`setup_sql`, beside `fixture` and `startup`, and not a startup keyword or a default +`instance_startup` hook. `startup` stays the world's own namespace. diff --git a/src/seahaven/cli/mcp.py b/src/seahaven/cli/mcp.py index 9032f7a..8381fca 100644 --- a/src/seahaven/cli/mcp.py +++ b/src/seahaven/cli/mcp.py @@ -85,7 +85,7 @@ # The keys a `--reset-options` object may name. Read off the signature of # `world.instance()` rather than written out, so a keyword argument the # framework adds is one this command takes without an edit here. -# `tests/test_cli_mcp.py` pins the five names, so a parameter added to +# `tests/test_cli_mcp.py` pins the names, so a parameter added to # `world.instance()` fails a test rather than reaching an MCP client unreviewed. RESET_OPTION_KEYS = frozenset( name diff --git a/src/seahaven/docs/composition.md b/src/seahaven/docs/composition.md index b6763ee..263ae32 100644 --- a/src/seahaven/docs/composition.md +++ b/src/seahaven/docs/composition.md @@ -609,6 +609,16 @@ Nothing agent-facing says a world is composed. Everything eval-facing does. the first `reset`, so a session can say what tree it is running against. `state` is not an observation, and nothing agent-facing carries any of it. +`setup_sql` ([db_schema_and_fixtures.md](db_schema_and_fixtures.md#adjusting-a-fixture-per-episode)) +names tables as `inst.inspect()` does, on a connection that can write every store. One statement +can read the root's tables and write an added node's: + +```py +setup_sql = "INSERT INTO payments.charges (id, amount) SELECT 'c-' || id, 500 FROM staff" +with company.instance("acme", setup_sql=setup_sql) as inst: + charged = inst.inspect().rows("SELECT id, amount FROM payments.charges") +``` + ## Typed access A tool is a plain typed function, so calling it **by the function itself** gives a type checker diff --git a/src/seahaven/docs/db_schema_and_fixtures.md b/src/seahaven/docs/db_schema_and_fixtures.md index 42353b7..428a5fc 100644 --- a/src/seahaven/docs/db_schema_and_fixtures.md +++ b/src/seahaven/docs/db_schema_and_fixtures.md @@ -9,6 +9,7 @@ runs on, the schema that shapes it, and the fixtures that hold the starting stat | [The schema](#the-schema) | Where the tables are declared, the three rules, and the schema hash | | [What a fixture is](#what-a-fixture-is) | The directory, the sidecar file, and the rules that keep a fixture trustworthy | | [Building a fixture](#building-a-fixture) | Blank instances, bulk loading, and the commands | +| [Adjusting a fixture per episode](#adjusting-a-fixture-per-episode) | `setup_sql`, and when a fixture or a startup hook is the better tool | | [The generator is committed](#the-generator-is-committed) | Why the script that builds a fixture ships beside it | | [Writing a description](#writing-a-description) | The one field written for a person | | [When the schema changes](#when-the-schema-changes) | Regenerating every fixture, and why nothing migrates | @@ -237,6 +238,70 @@ with world.instance("seeded") as inst: `freeze` cannot run inside a `bulk()` block, because the rows are not committed yet. Leave the block first. +## Adjusting a fixture per episode + +An eval often needs a fixture with one small change: a due date moved, a comment deleted, one more +user. `setup_sql` makes that change without a new fixture and without code. It is a parameter of +`world.instance(...)`, and so of `reset` over a server +([serving_and_openenv.md](serving_and_openenv.md#sessions-and-instances)). + +`setup_sql` is one string of SQL statements separated by `;`. Seahaven runs them in order on the new +instance, after the fixture is copied (or the blank schema is built) and before the startup hooks +run, in one transaction. A startup hook therefore sees the rows `setup_sql` wrote. Those rows are +starting state, so they are not in the change log. `inst.state()["setup_sql"]` records the string +as given. + +```python +from pathlib import Path + +import seahaven + +world = seahaven.World( + name="notes", + version="1.0.0", + schema=( + "CREATE TABLE notes " + "(id TEXT PRIMARY KEY, body TEXT NOT NULL, done INTEGER NOT NULL) STRICT;" + ), + fixtures_dir=Path("fixtures"), + state_format="seahaven.state/1", +) + +with world.instance(now="2026-06-01T09:00:00.000Z", clock_mode="fixed") as inst: + with inst.bulk() as ctx: + ctx.db.execute("INSERT INTO notes (id, body, done) VALUES ('n1', 'Buy milk', 0)") + inst.freeze("one_note", "One open note.") + +setup_sql = """ + UPDATE notes SET done = 1 WHERE id = 'n1'; + INSERT INTO notes (id, body, done) VALUES ('n2', 'Call the bank; ask about fees', 0); +""" + +with world.instance("one_note", setup_sql=setup_sql) as inst: + rows = inst.inspect().rows("SELECT id, done FROM notes ORDER BY id") + assert rows == [{"id": "n1", "done": 1}, {"id": "n2", "done": 0}] + assert inst.change_log() == [] + assert inst.state()["setup_sql"] == setup_sql +``` + +**Write rows only.** Use `INSERT`, `UPDATE` and `DELETE`. A statement that changes the schema, +controls the transaction or attaches a database is refused, and instance creation fails with a +`WorldBug` that names the statement. [reference/api.md](reference/api.md#setup_sql) lists what is +refused. To change the schema, change the world and bump its version. + +In a world that adds other worlds, an added node's tables are named `.
`, as in +`inst.inspect()` ([composition.md](composition.md#what-an-eval-sees)). `random()` and +`randomblob()` draw from the instance's seed, so the same `seed` and the same `setup_sql` write the +same rows. SQL's clock functions read the instance's clock. + +Choose the tool by what the change is: + +| Use | When | +|---|---| +| a new fixture | many evals start from the same state, or the change is large. Freeze it once, as above | +| `setup_sql` | one eval needs a small change to the rows of a fixture | +| a startup hook | the setup needs the world's own logic or `ctx.state`, such as who the agent acts as ([authoring.md](authoring.md#instance-startup)) | + ## The generator is committed **Commit every fixture together with the script that generates it.** A binary file with no source is diff --git a/src/seahaven/docs/reference/api.md b/src/seahaven/docs/reference/api.md index 59bb444..8ec2ffe 100644 --- a/src/seahaven/docs/reference/api.md +++ b/src/seahaven/docs/reference/api.md @@ -120,7 +120,7 @@ a NUL, an accent, a non-Latin script. A **fixture id** follows the same rule, an | `world.add_world(other, *, name=None, store=None, tool_prefix=None, tool_allow_list=None, tool_block_list=None, startup=None)` | add another world: its tools join this world's surface, its store becomes a node of every instance. Call it; there is nothing to decorate | | `world.state_format(name)` | register a state format of this world's own, as a decorator. The name is `/` and may not begin with `seahaven.` | | `world.resolve_state_format(name)` | the formatter a name answers to: a built-in, or one this world registered. Asked of the root of an instance and of nothing else | -| `world.instance(fixture=None, *, seed=None, now=None, clock_mode=None, state_format=None, control_tools=False, startup=None)` | make an instance; a context manager. `now` is where the clock starts, no earlier than the fixture's `now`. `clock_mode` is how the clock moves, `None` for `world.default_clock_mode`. `state_format` answers in another of this world's formats, in place of the pin. `control_tools=True` makes the framework's own control tool callable on the instance. `startup` is the world's own keywords, passed to the startup hooks that name them | +| `world.instance(fixture=None, *, seed=None, now=None, clock_mode=None, state_format=None, control_tools=False, startup=None, setup_sql=None)` | make an instance; a context manager. `now` is where the clock starts, no earlier than the fixture's `now`. `clock_mode` is how the clock moves, `None` for `world.default_clock_mode`. `state_format` answers in another of this world's formats, in place of the pin. `control_tools=True` makes the framework's own control tool callable on the instance. `startup` is the world's own keywords, passed to the startup hooks that name them. `setup_sql` is SQL run on the new instance before the startup hooks; see [`setup_sql`](#setup_sql) | | `world.fixtures()` | every fixture in the fixtures directory, as a list sorted by id. A world with no fixtures directory has none, which is not an error | | `copy.copy(world)` | this world with the same registrations and its own instances: set `fixtures_dir` on the copy to freeze somewhere else without moving the imported world's | | `world.tools` | the registry, in registration order. Read-only, and this world's **own** tools: the composite surface an agent sees is `inst.tools()`, or `world.composition().tools` | @@ -187,6 +187,7 @@ Made by `world.instance(...)`, never by hand. A context manager; leaving the blo | `inst.destroy()` | close everything and remove the working directory. Idempotent, and waits for a call in flight | | `inst.id`, `inst.fixture`, `inst.seed` | the instance id, the fixture id (or `None`), the derived seed bytes | | `inst.state_format` | the format `inst.state()` answers in, fixed for the instance's life | +| `inst.setup_sql` | the `setup_sql` the instance was created with, as given, or `None` | | `inst.control_tools` | whether the framework's own control tool is callable on this instance, fixed for the instance's life | | `inst.clock`, `inst.world`, `inst.state_path` | the clock, the world, and the instance's own database file | @@ -198,6 +199,28 @@ a name the world does not have raises, and no instance lists it either way. The read `inst.state()` instead. Over a server, `serve --include-control-tools` is what makes each session's instance one that can call it, and [cli.md](cli.md) names the tool. +### `setup_sql` + +`world.instance(setup_sql=...)` takes a string of SQL statements separated by `;`, or `None`. The +statements run in order, after the fixture is copied (or the blank schema is built) and before the +startup hooks, in one transaction. The connection has the root's tables unqualified and every added +node's tables under its schema name, as `inst.inspect()` does. A statement may read and write the +rows of every table, call the functions in `seahaven.sandbox.ALLOWED_FUNCTIONS`, and run a pragma +that only reports, such as `PRAGMA table_info(issues)`. It is refused when it: + +- changes the schema: `CREATE`, `DROP`, `ALTER`, `REINDEX` or `ANALYZE`; +- controls the transaction: `BEGIN`, `COMMIT`, `ROLLBACK`, `SAVEPOINT` or `RELEASE`; +- runs `ATTACH` or `DETACH`; +- runs any other pragma, with or without a value, such as `PRAGMA foreign_keys = OFF` or + `PRAGMA user_version`; +- calls a function outside the allowlist, `MATCH` and `load_extension()` included; +- reads a table-valued function, such as `json_each()` or `pragma_table_info()`; +- writes an FTS5 shadow table or one of SQLite's own tables. + +A refused or failing statement raises `WorldBug` naming its position, and every statement before it +is rolled back. No instance is left behind. See +[../db_schema_and_fixtures.md](../db_schema_and_fixtures.md#adjusting-a-fixture-per-episode). + ## `Ctx` What every tool, middleware and startup hook receives. Nothing else is exposed: no working @@ -626,8 +649,8 @@ it to use Seahaven. See [../http_apis.md](../http_apis.md) for more. Installing `seahaven` activates the plugin. It adds two fixtures, `world` (session-scoped) and `instance` (one per test); one marker, `@pytest.mark.seahaven(fixture, seed=None, now=None, -clock_mode=None, state_format=None, control_tools=False, startup=None)`; and one option, -`--seahaven-world module:attr`. See [../testing.md](../testing.md). +clock_mode=None, state_format=None, control_tools=False, startup=None, setup_sql=None)`; and one +option, `--seahaven-world module:attr`. See [../testing.md](../testing.md). ## The concurrency gate diff --git a/src/seahaven/docs/reference/cli.md b/src/seahaven/docs/reference/cli.md index d144b4f..b0db98a 100644 --- a/src/seahaven/docs/reference/cli.md +++ b/src/seahaven/docs/reference/cli.md @@ -209,14 +209,16 @@ uv run --extra mcp seahaven mcp uv run --extra mcp seahaven mcp --fixture small_startup --seed 7 uv run --extra mcp seahaven mcp --fixture small_startup --clock-mode tick uv run --extra mcp seahaven mcp --reset-options '{"startup": {"reviewer": "ada"}}' +uv run --extra mcp seahaven mcp --reset-options '{"setup_sql": "UPDATE issues SET due_at = NULL"}' ``` -`--reset-options` is the whole of what `world.instance()` is called with, and the only way to reach -a world's own startup keywords, which go inside `"startup"`. `--fixture`, `--seed`, `--now` and -`--clock-mode` are convenience spellings of four of its keys. Giving `--reset-options` together with -one of those four flags is refused rather than merged, and so is a key `world.instance()` does not -take. An instance this command makes never has control tools, and `--reset-options` refuses the -`"control_tools"` key that would ask for them. +`--reset-options` is the whole of what `world.instance()` is called with. It is the only way to +pass `"startup"`, which holds a world's own startup keywords, and `"setup_sql"`, the SQL that +changes the fixture's rows before the startup hooks run ([api.md](api.md#setup_sql)). `--fixture`, +`--seed`, `--now` and `--clock-mode` are convenience spellings of four of its keys. Giving +`--reset-options` together with one of those four flags is refused rather than merged, and so is a +key `world.instance()` does not take. An instance this command makes never has control tools, and +`--reset-options` refuses the `"control_tools"` key that would ask for them. Read [../serving_and_openenv.md](../serving_and_openenv.md#serving-one-world-to-an-mcp-client) for the `.mcp.json` that launches the command, and for what a client is and is not given. diff --git a/src/seahaven/docs/serving_and_openenv.md b/src/seahaven/docs/serving_and_openenv.md index 4c7ae5a..07515ff 100644 --- a/src/seahaven/docs/serving_and_openenv.md +++ b/src/seahaven/docs/serving_and_openenv.md @@ -133,9 +133,10 @@ A WebSocket connection is one session, and one session holds one instance. | `episode_id=` | your own id for the episode, echoed back on `state` so a trajectory ties to your run | | `state_format=` | the format the `state` message answers in, in place of the world's pin ([state.md](state.md)) | | `startup=` | an object of the world's own startup keywords, passed to its startup hooks, so a world can be set up per episode | +| `setup_sql=` | a string of SQL statements, separated by `;`, run on the new instance before its startup hooks, to change the fixture's rows for this episode ([db_schema_and_fixtures.md](db_schema_and_fixtures.md#adjusting-a-fixture-per-episode)) | -`reset` is therefore where a run is customised. One served world covers every scenario a fixture and -a startup hook can express. +`reset` is therefore where a run is customised. One served world covers every scenario a fixture, +`setup_sql` and a startup hook can express. The list above is every key that changes what a reset does. A key outside the list is refused before the session is touched: the error names the stray keys and the keys `reset` does take, the session @@ -150,6 +151,15 @@ refused rather than delivered: {"fixture": "agency", "seed": 7, "startup": {"user_id": "u_12"}} ``` +`setup_sql` is one string, which may hold several statements: + +```jsonc +{ + "fixture": "agency", + "setup_sql": "UPDATE issues SET due_at = NULL; UPDATE issues SET priority = 1 WHERE key = 'ENG-3'" +} +``` + `reset` answers a plain OpenEnv `Observation`, not the shape of a tool call, because no tool was called. Its `metadata` carries `fixture`, `now`, `clock_mode` and `tools` (the fixture the instance was made from, the clock's reading and its mode, and how many tools the instance lists), and the @@ -326,6 +336,7 @@ envelope field of the document is a typed field on the model: | `clock_mode` | `ClockMode \| None` | how the instance's clock moves: `fixed`, `tick`, `running` or `wall` | | `seed` | `int \| None` | the seed `reset` was given | | `startup` | `dict[str, Any] \| None` | the world's own startup keywords, reset's `startup`, rendered as JSON at instance creation | +| `setup_sql` | `str \| None` | the SQL reset's `setup_sql` ran on the instance, as given | | `call_count` | `int` | how many calls were dispatched | | `state` | `dict[str, Any]` | the formatter's output, left untyped because its shape is the format's | | `step_count` | `int` | OpenEnv's count of everything the session asked for, tool listings included. Not `call_count` | @@ -337,9 +348,9 @@ from `seahaven.openenv` beside `SeahavenState`. The model keeps OpenEnv's `extra newer server can talk to an older client: read the fields you know and ignore the rest. Before the first `reset` there is no instance, so `composition`, `fixture`, `episode_id`, `seed`, -`now`, `clock_mode` and `startup` are `null`, `call_count` is 0, and the world's pinned formatter -runs with no instance. A blank instance is still told apart from no instance, because a blank -instance has a `now` and a `composition`. +`now`, `clock_mode`, `startup` and `setup_sql` are `null`, `call_count` is 0, and the world's pinned +formatter runs with no instance. A blank instance is still told apart from no instance, because a +blank instance has a `now` and a `composition`. **Choosing the format.** `reset(state_format="…")` selects the format for that episode, in place of the root world's pin. The `state` message itself carries no arguments, so one episode answers in one diff --git a/src/seahaven/docs/state.md b/src/seahaven/docs/state.md index b971a4b..b14432a 100644 --- a/src/seahaven/docs/state.md +++ b/src/seahaven/docs/state.md @@ -59,6 +59,7 @@ A document has two halves, and each half has its own owner. "now": "2026-06-01T09:00:00.004Z", "clock_mode": "running", "startup": {}, // the startup keywords, if any + "setup_sql": null, // the setup SQL, if any "call_count": 2, "state": {"db": {"log": [ … ]}} // the formatter's output } @@ -79,6 +80,7 @@ every format, built in or your own. | `now` | The instance's clock reading when the document was produced, as an ISO-8601 instant, the same string `inst.clock.iso()` answers. Producing a document reads the clock and never moves it | | `clock_mode` | How the instance's clock moves: `fixed`, `tick`, `running` or `wall` ([clock.md](clock.md)). `null` before the first `reset` over a server | | `startup` | The world's own startup keywords — `world.instance(startup={...})`, or the same `startup` on `reset` — rendered as JSON at instance creation. A hook receives the value the caller passed; the document carries that value's JSON rendering, so a `datetime` or a model is text or an object here. Empty when there were none | +| `setup_sql` | The SQL that ran on the instance before its startup hooks — `world.instance(setup_sql=...)`, or the same `setup_sql` on `reset` — exactly as given ([db_schema_and_fixtures.md](db_schema_and_fixtures.md#adjusting-a-fixture-per-episode)). `null` when there was none | | `call_count` | How many calls have been dispatched to the instance. The last call's ordinal is one less. Over a server this is **not** `step_count`, which counts tool listings as well | | `state` | The formatter's output, and the only part of the document `format` describes | @@ -419,9 +421,9 @@ Four more rules are worth knowing: records for both, under the same `i`, told apart by `world`. A tool that reaches another node through `ctx.worlds..call(...)` takes no ordinal of its own: the caller made one call, and the rows that tool wrote carry that call's `i`. -- **Startup writes are not in the log.** Startup hooks run when the instance is created, before the - first call, so what they wrote is starting state rather than the agent's work. Every write after - that is in the log, `inst.bulk()` writes included, which carry `i: null`. +- **Startup writes are not in the log.** `setup_sql` and the startup hooks run when the instance is + created, before the first call, so what they wrote is starting state rather than the agent's work. + Every write after that is in the log, `inst.bulk()` writes included, which carry `i: null`. What is tracked is every table of every node, except the tables that node's world named in `World(untracked_tables=...)`, FTS5's shadow tables, and virtual tables. A table with no explicit @@ -522,12 +524,12 @@ with projecttracker.world.instance() as blank: assert blank.state()["fixture"] is None # built from the schema ``` -For a blank instance, `fixture` is `null` and the starting state is the world's schema plus -whatever the startup hooks wrote, with `startup` recording the startup keywords it was created -with. **The format does not promise that starting state is reproducible.** A startup hook is -ordinary world code and may read anything, a blank instance's clock starts at wall time unless -`now=` was given, and only the `fixed` and `tick` clocks replay. -Grade against a frozen fixture when the starting state has to be pinned down. +The starting state is the fixture, then the rows `setup_sql` wrote, then whatever the startup hooks +wrote. `setup_sql` and `startup` record what the instance was created with. For a blank instance, +`fixture` is `null` and the world's schema takes the fixture's place. **The format does not promise +that starting state is reproducible.** A startup hook is ordinary world code and may read anything, +a blank instance's clock starts at wall time unless `now=` was given, and only the `fixed` and +`tick` clocks replay. Grade against a frozen fixture when the starting state has to be pinned down. ## The compatibility contract diff --git a/src/seahaven/ids.py b/src/seahaven/ids.py index d70dcb3..7c39ff3 100644 --- a/src/seahaven/ids.py +++ b/src/seahaven/ids.py @@ -34,6 +34,7 @@ "CONTROL_STREAM", "INSPECTION_STREAM", "INSTANCE_STREAM", + "SETUP_STREAM", "Ids", "instance_seed", "register_random_functions", @@ -98,10 +99,13 @@ def uuid(self) -> str: # door redraws. Every door draws from the start of its stream each time it is # opened, so a build that drew from `INSTANCE_STREAM` would hand a schema-seeded # row exactly the bytes the world's own first draw is about to take. +# `SETUP_STREAM` is the same kind of door for the caller's `setup_sql` +# (`setup_sql.py`): an instance's own draws are the same whether or not it ran. INSTANCE_STREAM = b"instance" INSPECTION_STREAM = b"inspection" CONTROL_STREAM = b"control" BUILD_STREAM = b"build" +SETUP_STREAM = b"setup" # Registered on connections that have SQLITE_DBCONFIG_TRUSTED_SCHEMA off, where # INNOCUOUS is what keeps them callable from a DEFAULT clause or a trigger. diff --git a/src/seahaven/instances.py b/src/seahaven/instances.py index 767ea62..b620db1 100644 --- a/src/seahaven/instances.py +++ b/src/seahaven/instances.py @@ -51,7 +51,7 @@ import threading import time import uuid -from collections.abc import Callable, Iterator, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping from contextlib import AbstractContextManager, ExitStack, closing, contextmanager, suppress from dataclasses import dataclass, field from datetime import datetime @@ -83,6 +83,7 @@ from seahaven.fixtures import Fixture, check_composition, check_id, freeze, load, verify from seahaven.handles import Frame, unbound from seahaven.ids import CONTROL_STREAM, INSPECTION_STREAM, Ids, instance_seed +from seahaven.setup_sql import run_setup_sql from seahaven.state import Formatter, document from seahaven.tool import Tool @@ -305,6 +306,7 @@ def __init__( caller_seed: int | None, fixture_files: Mapping[str, str] | None, startup: Mapping[str, Any], + setup_sql: str | None, control_tools: bool, ) -> None: self.id = id @@ -332,6 +334,7 @@ def __init__( # serialised at creation, so a keyword no document could carry is refused # there rather than at `state()` time. self.startup = dict(startup) + self.setup_sql = setup_sql # The format this instance answers in, fixed for its life, and the # formatter the root resolved it to when the instance was made. self.state_format = state_format @@ -899,14 +902,9 @@ def _attachments(self) -> list[tuple[str, Path]]: """Every node but the root, as `open_inspection` attaches them. Built from the instance's own runtime, which is the set of files that - exist on disk, and not from the world's current seal. By depth rather - than by position, so the invariant is read off the node itself. + exist on disk, and not from the world's current seal. """ - return [ - (runtime.node.schema_name, self.dir / runtime.node.file_name) - for runtime in self._runtime.values() - if runtime.node.depth > 0 - ] + return _attachment_pairs((runtime.node for runtime in self._runtime.values()), self.dir) def _nodes(self) -> tuple[NodeRuntime, ...]: """Every node's runtime, root first, in the composition's canonical order. @@ -1082,6 +1080,7 @@ def create( episode_id: str | None = None, control_tools: bool = False, startup: Mapping[str, Any] | None = None, + setup_sql: str | None = None, ) -> Instance: """Materialise an instance from a fixture, or from the world's DDL. @@ -1106,6 +1105,9 @@ def create( `startup` is the world's own keyword namespace, kept apart from the parameters above so that the framework can add a parameter without taking a name a world was already using. + + `setup_sql` is SQL that runs against the new files, every node attached, + before any node is opened and so before any startup hook (`setup_sql.py`). """ world = self._world # `None` is the only spelling of "no startup keywords". Anything else is @@ -1123,6 +1125,7 @@ def create( # fixture's own. A creation that cannot succeed copies nothing and leaves # nothing behind. _check_startup_keywords(composition, keywords) + _check_setup_sql(setup_sql) serialised_startup = _serialised_startup(keywords) # On the root, and only the root: an added world's registrations are # never consulted (`functional_spec.md` §6). @@ -1164,6 +1167,16 @@ def create( ).close() else: _copy_fixture(fixture, composition, directory) + if setup_sql is not None: + # Committed and closed before any node is opened, so its write + # locks on the attached files cannot block a startup hook. + run_setup_sql( + setup_sql, + root=directory / composition.root.file_name, + attachments=_attachment_pairs(composition.nodes, directory), + clock=clock, + seed=node_seed(base, ROOT_PATH), + ) info = InstanceInfo(id=instance_id, fixture=fixture_id, seed=base) for node in composition.nodes: runtime[node.key] = _open_node(node, directory, clock, base, info) @@ -1183,6 +1196,7 @@ def create( caller_seed=seed, fixture_files=_fixture_files(fixture), startup=serialised_startup, + setup_sql=setup_sql, control_tools=control_tools, ) # One activation for the whole of creation, so a root hook's handles @@ -1560,6 +1574,23 @@ def _copy_fixture(fixture: Fixture, composition: Composition, directory: Path) - shutil.copyfile(fixture.file_of(node), directory / by_path[node.path].file_name) +def _check_setup_sql(setup_sql: object) -> None: + """Refuse a `setup_sql=` that is not a string. Its SQL is not looked at until it runs.""" + if setup_sql is not None and not isinstance(setup_sql, str): + raise WorldBug( + "setup_sql takes a string of SQL statements separated by ';', not " + f"{type(setup_sql).__name__}" + ) + + +def _attachment_pairs(nodes: Iterable[Node], directory: Path) -> list[tuple[str, Path]]: + """Every node but the root, as `(schema name, file)`: how a connection attaches them. + + By depth rather than by position, so the invariant is read off the node itself. + """ + return [(node.schema_name, directory / node.file_name) for node in nodes if node.depth > 0] + + def _check_startup_keywords(composition: Composition, keywords: Mapping[str, Any]) -> None: """Refuse a `startup=` that is not a mapping, and a keyword no hook in the tree asked for. diff --git a/src/seahaven/openenv/env.py b/src/seahaven/openenv/env.py index fdb51a8..a74f457 100644 --- a/src/seahaven/openenv/env.py +++ b/src/seahaven/openenv/env.py @@ -273,7 +273,7 @@ class SeahavenState(State): # subclass's fields were published at all. # # Declared in `functional_spec.md` §3.1's order, less `episode_id`, which the - # base carries. A field the document always writes is required here; the six + # base carries. A field the document always writes is required here; the seven # it answers `null` for default to `None`, so reading one off a frame that # left it out answers `None` rather than raising. format: str = Field( @@ -322,6 +322,13 @@ class SeahavenState(State): "JSON rendering. Empty when there were none, null before any reset." ), ) + setup_sql: str | None = Field( + default=None, + description=( + "The SQL that reset's `setup_sql` ran against the instance when it was created, " + "exactly as given. Null when there was none, and before any reset." + ), + ) call_count: int = Field( description=( "How many calls have been dispatched to the instance. Not `step_count`, which also " @@ -339,7 +346,7 @@ class SeahavenState(State): # The console lays its reset form out in `properties` order, and pydantic puts # the inherited OpenEnv fields first. The fields not named here keep their own # order, after these. -RESET_ORDER = ("fixture", "startup", "now", "clock_mode", "state_format") +RESET_ORDER = ("fixture", "startup", "setup_sql", "now", "clock_mode", "state_format") def _order_properties(schema: dict[str, Any], leading: tuple[str, ...]) -> None: @@ -392,6 +399,13 @@ class SeahavenResetRequest(ResetRequest): startup: dict[str, Any] | None = Field( default=None, description="The world's own startup keywords, passed to its startup hooks." ) + setup_sql: str | None = Field( + default=None, + description=( + "SQL statements, separated by ';', run against the new instance before its startup " + "hooks. Rows only: INSERT, UPDATE, DELETE and SELECT." + ), + ) @classmethod def for_world(cls, world: World) -> type[SeahavenResetRequest]: @@ -490,6 +504,7 @@ def reset( state_format: str | None = None, control_tools: Any = None, startup: Mapping[str, Any] | None = None, + setup_sql: str | None = None, **unknown: Any, ) -> Observation: """Start an episode: a fresh instance, and the session's previous one gone. @@ -513,7 +528,8 @@ def reset( unregistered format is refused before anything is copied. Those are the instance's own rules and not a second set: everything here is passed straight through, including `startup`, the world's own keyword - namespace. + namespace, and `setup_sql`, the SQL the new instance runs before its + startup hooks. This signature is the reset wire schema. OpenEnv has no schema of its own for a reset message: its server introspects this method to bind the @@ -587,6 +603,7 @@ def reset( episode_id=episode_id or str(uuid.uuid4()), control_tools=self.include_control_tools, startup=startup, + setup_sql=setup_sql, ) self._instance = instance return Observation( diff --git a/src/seahaven/pytest_plugin.py b/src/seahaven/pytest_plugin.py index c1fe752..80d053b 100644 --- a/src/seahaven/pytest_plugin.py +++ b/src/seahaven/pytest_plugin.py @@ -46,8 +46,8 @@ def test_an_issue_can_be_created(instance: seahaven.Instance) -> None: _MARKER_SIGNATURE = ( f"{MARKER}(fixture, seed=None, now=None, clock_mode=None, state_format=None, " - "control_tools=False, startup=None): the fixture the `instance` fixture is created from; " - "fixture=None is a blank instance" + "control_tools=False, startup=None, setup_sql=None): the fixture the `instance` fixture is " + "created from; fixture=None is a blank instance" ) _EXAMPLE = ( diff --git a/src/seahaven/setup_sql.py b/src/seahaven/setup_sql.py new file mode 100644 index 0000000..e98d13a --- /dev/null +++ b/src/seahaven/setup_sql.py @@ -0,0 +1,343 @@ +"""Running a caller's `setup_sql` against a new instance, before its first call. + +`world.instance(setup_sql=...)` hands this module a string of statements. They +run after the fixture is copied (or the blank schema is built) and before any +node connection is opened, on one writable connection to the root's file with +every added node attached under its schema name, in one transaction. Nothing +else is open on those files yet, so nothing can be waiting on a lock. + +The statements run under `SetupAuthorizer`, the sandbox's default-deny +authorizer with every table of every node allowed for reading and writing. So +setup SQL reads and writes rows and does nothing else: no schema change, no +transaction control, no attach, no pragma that sets anything, and only the +functions `run_sql` has. These checks catch an author's mistakes and are not a +boundary: `SetupAuthorizer` says where they stop. +""" + +import re +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from pathlib import Path + +import apsw + +from seahaven.clock import Clock, register_clock_functions +from seahaven.db import _READ_ONLY_PRAGMAS, _fold_ascii, _harden +from seahaven.errors import WorldBug +from seahaven.ids import SETUP_STREAM, register_random_functions +from seahaven.sandbox import _ROW_WRITES, Authorizer + +__all__ = ["Refused", "SetupAuthorizer", "run_setup_sql", "split_statements"] + +_QUOTED_CHARACTERS = 200 + +_SCHEMA_ACTIONS = frozenset( + { + apsw.SQLITE_ALTER_TABLE, + apsw.SQLITE_ANALYZE, + apsw.SQLITE_CREATE_INDEX, + apsw.SQLITE_CREATE_TABLE, + apsw.SQLITE_CREATE_TEMP_INDEX, + apsw.SQLITE_CREATE_TEMP_TABLE, + apsw.SQLITE_CREATE_TEMP_TRIGGER, + apsw.SQLITE_CREATE_TEMP_VIEW, + apsw.SQLITE_CREATE_TRIGGER, + apsw.SQLITE_CREATE_VIEW, + apsw.SQLITE_CREATE_VTABLE, + apsw.SQLITE_DROP_INDEX, + apsw.SQLITE_DROP_TABLE, + apsw.SQLITE_DROP_TEMP_INDEX, + apsw.SQLITE_DROP_TEMP_TABLE, + apsw.SQLITE_DROP_TEMP_TRIGGER, + apsw.SQLITE_DROP_TEMP_VIEW, + apsw.SQLITE_DROP_TRIGGER, + apsw.SQLITE_DROP_VIEW, + apsw.SQLITE_DROP_VTABLE, + apsw.SQLITE_REINDEX, + } +) +# SQLite's own schema tables. A row write to one of these is the schema +# changing; a write to another `sqlite_` table, such as `sqlite_sequence`, is +# refused for the sandbox's own reason. +_SCHEMA_TABLES = frozenset({"sqlite_master", "sqlite_schema", "sqlite_temp_master"}) + +_SCHEMA_FIX = ( + "setup_sql may only read and write rows (INSERT, UPDATE, DELETE, SELECT). Change the schema " + "in the world and bump its version." +) + + +@dataclass(frozen=True) +class Refused: + """The first action `SetupAuthorizer` refused in one statement.""" + + action: int + third: str | None + fourth: str | None + # The base class's own words for it, for an action this module has no reason for. + description: str + + +class SetupAuthorizer(Authorizer): + """The sandbox's authorizer, writable over every table, recording what it refuses. + + It also allows the pragmas the read-only inspection connection allows, which + report and set nothing, and any read or function call SQLite attributes to + one of `triggers`, the world's own triggers. That attribution is by name + only, so it is a guard against mistakes and not a boundary: see `__init__`. + """ + + def __init__(self, tables: Sequence[str], triggers: Iterable[str] = ()) -> None: + super().__init__(tables, read_only=False) + # SQLite's auth context names the innermost trigger, view or named FROM + # item, a CTE included, and it names a view while it resolves the + # caller's own WHERE clause in DML on a view with an INSTEAD OF trigger. + # So a name is trusted only as a trigger, and never when a table or a + # view shares it (SQLite lets a trigger and a view have one name). A CTE + # the caller names after a world trigger still shares that trigger's + # trust for function calls and reads; that residual is accepted, because + # setup SQL comes from the eval's author and these checks catch mistakes. + shared = {_fold_ascii(name) for name in tables} + self._triggers = frozenset(_fold_ascii(name) for name in triggers) - shared + self.refused: Refused | None = None + + def reset(self) -> None: + super().reset() + self.refused = None + + def __call__( + self, + action: int, + third: str | None, + fourth: str | None, + database: str | None, + trigger: str | None, + /, + ) -> int: + if ( + action == apsw.SQLITE_PRAGMA + and third is not None + and _fold_ascii(third) in _READ_ONLY_PRAGMAS + ): + return apsw.SQLITE_OK + # A world trigger's body is the world's own SQL, run as world code runs it. + if ( + trigger is not None + and _fold_ascii(trigger) in self._triggers + and action in (apsw.SQLITE_FUNCTION, apsw.SQLITE_READ) + ): + return apsw.SQLITE_OK + answer = super().__call__(action, third, fourth, database, trigger) + if answer == apsw.SQLITE_DENY and self.refused is None: + self.refused = Refused(action, third, fourth, self.refusals[-1]) + return answer + + +def run_setup_sql( + sql: str, + *, + root: Path, + attachments: Sequence[tuple[str, Path]], + clock: Clock, + seed: bytes, +) -> None: + """Run a caller's setup SQL against a new instance's files, in one transaction. + + `attachments` are `(schema name, file)` for every node but the root. `seed` + is the root node's: `random()` draws from it on `SETUP_STREAM`, whichever + node a row lands in. Raises `WorldBug` naming the statement that failed, + with every statement before it rolled back. + """ + statements = split_statements(sql) + if not statements: + return + conn = apsw.Connection(str(root)) + try: + _harden(conn) + conn.set_busy_timeout(0) + register_random_functions(conn, seed, SETUP_STREAM) + helper = register_clock_functions(conn, clock) + try: + for schema, path in attachments: + conn.execute("ATTACH DATABASE ? AS ?", (str(path), schema)) + # An instance directory does not survive a crash (the sweep deletes + # it), so no file of it needs an fsync. Per schema: an attached + # file keeps SQLite's default otherwise. + for schema in ("main", *(schema for schema, _ in attachments)): + conn.pragma("synchronous", "OFF", schema=schema) + schemas = [schema for schema, _ in attachments] + authorizer = SetupAuthorizer( + _every_table(conn), _every_trigger(conn, ["main", *schemas]) + ) + _run(conn, statements, authorizer, schemas) + finally: + helper.close() + finally: + conn.close() + + +def _run( + conn: apsw.Connection, + statements: Sequence[str], + authorizer: SetupAuthorizer, + schemas: Sequence[str], +) -> None: + """Run every statement in one transaction, committed only if all of them succeed.""" + # `BEGIN` before the authorizer and `COMMIT` after it: it refuses transaction + # control, the framework's own included. + conn.execute("BEGIN") + cursor = conn.cursor() + try: + conn.authorizer = authorizer + try: + for index, statement in enumerate(statements, 1): + authorizer.reset() + try: + # Drained, so a statement that returns rows runs to its end. + for _ in cursor.execute(statement): + pass + except apsw.Error as error: + raise _failure( + index, len(statements), statement, authorizer.refused, error, schemas + ) from error + finally: + conn.authorizer = None + except BaseException: + cursor.close(force=True) + # A failed statement may already have ended the transaction itself (an + # `ON CONFLICT ROLLBACK`), and a second ROLLBACK would hide the error. + if conn.in_transaction: + conn.execute("ROLLBACK") + raise + cursor.close() + try: + conn.execute("COMMIT") + except apsw.Error as error: + if conn.in_transaction: + conn.execute("ROLLBACK") + raise WorldBug( + f"setup_sql failed at commit: {error}. A deferred constraint is checked once every " + "statement has run." + ) from error + + +def split_statements(sql: str) -> list[str]: + """`sql` as its separate statements, in order, without their trailing `;`. + + A `;` inside a string literal, a quoted identifier, a comment or a trigger + body does not split, because a piece is only a statement once SQLite says + it is complete. Whitespace between statements is dropped. Text after the last + complete statement is kept as a statement of its own, so SQLite reports it. + """ + statements: list[str] = [] + pending = "" + *pieces, last = sql.split(";") + for piece in pieces: + pending += piece + ";" + if apsw.complete(pending): + _keep(statements, pending.removesuffix(";")) + pending = "" + _keep(statements, pending + last) + return statements + + +def _keep(statements: list[str], text: str) -> None: + if text.strip(): + statements.append(text.strip()) + + +def _every_trigger(conn: apsw.Connection, schemas: Sequence[str]) -> list[str]: + """The name of every trigger the world's schemas hold.""" + names: list[str] = [] + for schema in schemas: + quoted = '"' + schema.replace('"', '""') + '"' + rows = conn.execute(f"SELECT name FROM {quoted}.sqlite_schema WHERE type = 'trigger'") + names.extend(str(name) for (name,) in rows) + return names + + +def _every_table(conn: apsw.Connection) -> list[str]: + """Every table and view on every schema of `conn`, FTS5's shadow tables included. + + FTS5 reads and writes its shadow tables through this connection while a + caller's statement runs, so they are allowed here. A caller's own write to + one is still refused, by SQLite's defensive mode (`db._harden`). + """ + rows = conn.execute( + "SELECT name FROM pragma_table_list WHERE type IN ('table', 'view', 'virtual', 'shadow') " + "AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\'" + ).fetchall() + return [str(name) for (name,) in rows] + + +def _failure( + index: int, + count: int, + statement: str, + refused: Refused | None, + error: apsw.Error, + schemas: Sequence[str], +) -> WorldBug: + reason, fix = _explain(refused, error, schemas) + outcome = "failed" if refused is None else "was refused" + explanation = f"{reason} {fix}" if fix else reason + return WorldBug( + f"setup_sql statement {index} of {count} {outcome}: {explanation}\n {_quoted(statement)}" + ) + + +def _explain(refused: Refused | None, error: apsw.Error, schemas: Sequence[str]) -> tuple[str, str]: + """Why a statement failed, and what to do instead. The fix may be empty.""" + if refused is None: + message = str(error) + if message.startswith("no such table") and schemas: + return ( + f"{message}.", + f"An added node's tables are named .
{_named(schemas)}", + ) + return f"{message}.", "" + action = refused.action + if action in _SCHEMA_ACTIONS or ( + action in _ROW_WRITES and _fold_ascii(refused.third or "") in _SCHEMA_TABLES + ): + return "it changes the schema.", _SCHEMA_FIX + if action in (apsw.SQLITE_TRANSACTION, apsw.SQLITE_SAVEPOINT): + return ( + "it controls the transaction.", + "setup_sql already runs in one transaction that Seahaven opens and commits; remove " + "BEGIN, COMMIT and SAVEPOINT.", + ) + if action in (apsw.SQLITE_ATTACH, apsw.SQLITE_DETACH): + return ( + "it attaches or detaches a database.", + "Every node is already attached: name the root's tables unqualified and an added " + f"node's as .
{_named(schemas)}", + ) + if action == apsw.SQLITE_PRAGMA: + return ( + f"PRAGMA {refused.third} can change the connection.", + "Only read-only pragmas such as table_info are allowed.", + ) + if action == apsw.SQLITE_FUNCTION: + return ( + f"function {refused.fourth}() is not allowed.", + "setup_sql has the functions run_sql has.", + ) + return ( + f"{refused.description} is not allowed.", + "setup_sql may only read and write the world's own tables.", + ) + + +def _named(schemas: Sequence[str]) -> str: + """The schema names this world's added nodes have, as the end of a sentence.""" + if not schemas: + return " (this world adds none)." + return f" (this world: {', '.join(schemas)})." + + +def _quoted(statement: str) -> str: + collapsed = re.sub(r"\s+", " ", statement).strip() + if len(collapsed) <= _QUOTED_CHARACTERS: + return collapsed + return collapsed[:_QUOTED_CHARACTERS] + "..." diff --git a/src/seahaven/state.py b/src/seahaven/state.py index b9108c2..5d1df29 100644 --- a/src/seahaven/state.py +++ b/src/seahaven/state.py @@ -103,6 +103,7 @@ def envelope(world: World, instance: Instance | None, format: str) -> dict[str, # Copied, not handed out: a caller that edits the document it was given # must not edit the instance, and every later read with it. "startup": copy.deepcopy(instance.startup) if instance is not None else None, + "setup_sql": instance.setup_sql if instance is not None else None, "call_count": instance.call_count if instance is not None else 0, } diff --git a/src/seahaven/world.py b/src/seahaven/world.py index 851db02..5b2be8f 100644 --- a/src/seahaven/world.py +++ b/src/seahaven/world.py @@ -522,6 +522,7 @@ def instance( state_format: str | None = None, control_tools: bool = False, startup: Mapping[str, Any] | None = None, + setup_sql: str | None = None, ) -> Instance: """Make a live instance: a private copy of a fixture, or a blank one. @@ -535,8 +536,10 @@ def instance( they are not callable at all, and their names answer `UnknownTool` like any name the world does not have. `startup` is the world's own namespace: every keyword in it is passed to the startup hooks that named - it, and a keyword no hook names is refused. The instance is a context - manager and leaving the block destroys it. + it, and a keyword no hook names is refused. `setup_sql` is SQL that + reads and writes rows of the new instance before any startup hook runs, + every node's tables in reach. The instance is a context manager and + leaving the block destroys it. Never from inside a tool call: a handler that wants another world reaches it through `ctx.worlds`, and a world that made its own instance would be @@ -555,6 +558,7 @@ def instance( state_format=state_format, control_tools=control_tools, startup=startup, + setup_sql=setup_sql, ) def fixtures(self) -> list[Fixture]: diff --git a/tests/state_v1.schema.json b/tests/state_v1.schema.json index c165af3..94dad26 100644 --- a/tests/state_v1.schema.json +++ b/tests/state_v1.schema.json @@ -14,6 +14,7 @@ "now", "clock_mode", "startup", + "setup_sql", "call_count", "state" ], @@ -79,6 +80,7 @@ "now": {"type": ["string", "null"]}, "clock_mode": {"enum": ["fixed", "tick", "running", "wall", null]}, "startup": {"type": ["object", "null"]}, + "setup_sql": {"type": ["string", "null"]}, "call_count": {"type": "integer", "minimum": 0}, "state": { "type": "object", diff --git a/tests/test_cli_mcp.py b/tests/test_cli_mcp.py index 3d62ca1..5f33ee1 100644 --- a/tests/test_cli_mcp.py +++ b/tests/test_cli_mcp.py @@ -165,6 +165,15 @@ def test_reset_options_are_passed_whole( } +def test_setup_sql_is_a_reset_option(calls: list[Call], capsys: pytest.CaptureFixture[str]) -> None: + given = '{"setup_sql": "UPDATE notes SET n = 1; DELETE FROM notes WHERE n = 2"}' + + assert serve_mcp(capsys, "--reset-options", given).code == 0 + assert calls[0].reset_options["setup_sql"] == ( + "UPDATE notes SET n = 1; DELETE FROM notes WHERE n = 2" + ) + + def test_the_general_variable_is_passed_whole( calls: list[Call], monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: @@ -289,6 +298,7 @@ def test_reset_options_take_the_keyword_arguments_of_world_instance() -> None: "fixture", "now", "seed", + "setup_sql", "startup", "state_format", ] @@ -314,9 +324,9 @@ def test_a_key_world_instance_does_not_take_is_refused( assert (result.code, calls) == (1, []) assert result.err.strip() == ( '--reset-options does not take "region"; --reset-options takes "clock_mode", "fixture", ' - '"now", "seed", "startup" and "state_format", and a world\'s own startup keywords go ' - 'inside "startup": --reset-options \'{"fixture": "small_startup", "startup": ' - '{"user_id": "u_12"}}\'' + '"now", "seed", "setup_sql", "startup" and "state_format", and a world\'s own startup ' + 'keywords go inside "startup": --reset-options \'{"fixture": "small_startup", ' + '"startup": {"user_id": "u_12"}}\'' ) diff --git a/tests/test_client.py b/tests/test_client.py index b8183be..4d587b7 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -141,6 +141,7 @@ def test_an_observation_with_no_seahaven_error_answers_none(client: SeahavenClie "now": INSTANT_ISO, "clock_mode": "tick", "startup": {"tenant": "globex"}, + "setup_sql": "UPDATE notes SET n = 1", "call_count": 2, "state": { "db": { @@ -177,6 +178,7 @@ def test_parse_state_answers_a_typed_state(client: SeahavenClient) -> None: assert (state.seed, state.now, state.call_count) == (7, INSTANT_ISO, 2) assert state.clock_mode == "tick" assert state.startup == {"tenant": "globex"} + assert state.setup_sql == "UPDATE notes SET n = 1" # `state` is untyped on purpose: the format owns its shape, so it arrives as # the dict the formatter produced and nothing validates it here. assert state.state["db"]["log"][0]["key"] == {"id": "n1"} @@ -193,7 +195,7 @@ def test_the_document_is_the_state_without_the_step_count(client: SeahavenClient def test_state_answers_none_for_the_fields_a_pre_reset_frame_leaves_null( client: SeahavenClient, ) -> None: - """Before the first reset the document answers `null` for five envelope fields. + """Before the first reset the document answers `null` for seven envelope fields. They are declared with a default for that reason, so a harness that logs `state.fixture` every step does not fail on step zero -- and nor does a frame @@ -210,13 +212,15 @@ def test_state_answers_none_for_the_fields_a_pre_reset_frame_leaves_null( "state": {"db": {"log": []}}, } ) - assert (state.composition, state.fixture, state.seed, state.now, state.startup) == ( - None, - None, - None, - None, - None, - ) + assert ( + state.composition, + state.fixture, + state.seed, + state.now, + state.clock_mode, + state.startup, + state.setup_sql, + ) == (None, None, None, None, None, None, None) def test_state_keeps_an_envelope_field_it_does_not_know(client: SeahavenClient) -> None: diff --git a/tests/test_composite_instance.py b/tests/test_composite_instance.py index 20814dd..0a07c6a 100644 --- a/tests/test_composite_instance.py +++ b/tests/test_composite_instance.py @@ -21,7 +21,14 @@ from seahaven.ctx import Ctx from seahaven.db import open_instance from seahaven.errors import DbError, WorldBug -from seahaven.ids import BUILD_STREAM, CONTROL_STREAM, INSPECTION_STREAM, INSTANCE_STREAM, Ids +from seahaven.ids import ( + BUILD_STREAM, + CONTROL_STREAM, + INSPECTION_STREAM, + INSTANCE_STREAM, + SETUP_STREAM, + Ids, +) from seahaven.instances import node_seed from seahaven.world import World from tests.conftest import INSTANT_ISO, composable_world @@ -226,7 +233,7 @@ def test_a_node_named_after_a_sql_door_does_not_draw_that_doors_stream(tmp_path: separates the two. """ host = rooted("host", tmp_path) - labels = (INSTANCE_STREAM, INSPECTION_STREAM, CONTROL_STREAM, BUILD_STREAM) + labels = (INSTANCE_STREAM, INSPECTION_STREAM, CONTROL_STREAM, BUILD_STREAM, SETUP_STREAM) named = [label.decode() for label in labels] for name in named: host.add_world(composable_world(name), name=name) diff --git a/tests/test_env.py b/tests/test_env.py index 80f4a91..e1053de 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -253,6 +253,35 @@ def test_a_failed_reset_leaves_the_session_as_a_fresh_one(env: SeahavenEnv) -> N assert env.state.episode_id == "second" +def test_setup_sql_reaches_the_instance_through_reset(env: SeahavenEnv, world: World) -> None: + """The SQL runs on the fixture's copy, and the document reports it without logging its rows.""" + fixture_id = make_fixture(world) + setup_sql = "UPDATE notes SET n = 5 WHERE id = 'n0'; INSERT INTO notes VALUES ('n1', 'b', 1)" + env.reset(fixture=fixture_id, setup_sql=setup_sql) + + rows = call(env, "rows", sql="SELECT id, n FROM notes ORDER BY id").result + assert rows == [{"id": "n0", "n": 5}, {"id": "n1", "n": 1}] + state = env.state + assert state.setup_sql == setup_sql + assert state.state == {"db": {"log": []}} + assert state.call_count == 1 + + +def test_a_failing_setup_sql_leaves_the_session_as_a_fresh_one(env: SeahavenEnv) -> None: + env.reset(setup_sql="INSERT INTO notes VALUES ('n0', 'a', 0)") + first = env.instance + assert first is not None + + with pytest.raises(WorldBug, match=r"^setup_sql statement 2 of 2 failed: no such table: nope"): + env.reset(setup_sql="INSERT INTO notes VALUES ('n1', 'a', 0); DELETE FROM nope") + assert env.instance is None + assert not first.dir.exists() + assert (env.state.episode_id, env.state.setup_sql) == (None, None) + + env.reset() + assert call(env, "rows", sql="SELECT id FROM notes").result == [] + + RESET_PARAMETERS = ( "self", "seed", @@ -263,6 +292,7 @@ def test_a_failed_reset_leaves_the_session_as_a_fresh_one(env: SeahavenEnv) -> N "state_format", "control_tools", "startup", + "setup_sql", ) @@ -1044,6 +1074,10 @@ def test_a_closed_session_resets_again(env: SeahavenEnv) -> None: "the instance was created: a hook receives the caller's value and this carries its " "JSON rendering. Empty when there were none, null before any reset." ), + "setup_sql": ( + "The SQL that reset's `setup_sql` ran against the instance when it was created, " + "exactly as given. Null when there was none, and before any reset." + ), "call_count": ( "How many calls have been dispatched to the instance. Not `step_count`, which also " "counts tool listings. The last call's ordinal is one less than this." diff --git a/tests/test_http_server.py b/tests/test_http_server.py index 64726f2..3bb5272 100644 --- a/tests/test_http_server.py +++ b/tests/test_http_server.py @@ -306,6 +306,14 @@ def test_a_put_body_is_laid_over_the_servers_options(notes_world: World) -> None assert len(client.get("/worlds/c/notes").json()) == 1 +def test_a_put_body_with_setup_sql_starts_the_instance_with_its_rows( + client: httpx.Client, +) -> None: + setup_sql = "INSERT INTO notes VALUES ('n1', 'seeded', '2024-03-05T12:00:00.000Z')" + assert client.put("/worlds/a", json={"setup_sql": setup_sql}).status_code == 201 + assert [note["body"] for note in client.get("/worlds/a/notes").json()] == ["seeded"] + + @pytest.mark.parametrize( ("body", "words"), [ @@ -320,6 +328,8 @@ def test_a_put_body_is_laid_over_the_servers_options(notes_world: World) -> None (b'{"fixture": "nope"}', "world 'notes_api' has no fixture 'nope'"), (b'{"clock_mode": "nope"}', "unknown clock mode 'nope'"), (b'{"startup": {"nope": 1}}', "unknown startup keyword(s): ['nope']"), + (b'{"setup_sql": 1}', "setup_sql takes a string of SQL statements"), + (b'{"setup_sql": "DROP TABLE notes"}', "setup_sql statement 1 of 1 was refused"), ], ) def test_a_put_body_that_is_refused_is_400( diff --git a/tests/test_ids.py b/tests/test_ids.py index 94a4bba..4738f2e 100644 --- a/tests/test_ids.py +++ b/tests/test_ids.py @@ -15,6 +15,7 @@ CONTROL_STREAM, INSPECTION_STREAM, INSTANCE_STREAM, + SETUP_STREAM, Ids, instance_seed, register_random_functions, @@ -153,7 +154,7 @@ def test_no_sql_stream_is_the_one_ctx_ids_draws_from() -> None: mirror = Ids(seed) from_ids = [mirror.random.randbytes(8) for _ in range(5)] - for stream in (INSTANCE_STREAM, INSPECTION_STREAM, CONTROL_STREAM, BUILD_STREAM): + for stream in (INSTANCE_STREAM, INSPECTION_STREAM, CONTROL_STREAM, BUILD_STREAM, SETUP_STREAM): with seeded_connection(seed, stream) as conn: assert [conn.execute("SELECT randomblob(8)").get for _ in range(5)] != from_ids @@ -175,7 +176,7 @@ def test_a_door_replays_its_own_stream_and_echoes_no_other(seeded: apsw.Connecti # And the door it replays is where it was left, not back at its start. assert seeded.execute("SELECT random()").get not in drawn - for other in (INSPECTION_STREAM, CONTROL_STREAM, BUILD_STREAM): + for other in (INSPECTION_STREAM, CONTROL_STREAM, BUILD_STREAM, SETUP_STREAM): with seeded_connection(seed, other) as door: assert [door.execute("SELECT random()").get for _ in range(3)] != drawn diff --git a/tests/test_mcp_process.py b/tests/test_mcp_process.py index 11e5124..457f0c3 100644 --- a/tests/test_mcp_process.py +++ b/tests/test_mcp_process.py @@ -375,6 +375,17 @@ async def work(client: Client) -> Any: assert talk(project, work, "--reset-options", given)["region"] == "eu" +def test_reset_options_carry_setup_sql_to_the_instance(project: Project) -> None: + """The instance starts with the rows the SQL wrote, before the first call.""" + + async def work(client: Client) -> Any: + return answered(await client.call_tool("notes", {})) + + given = json.dumps({"setup_sql": "INSERT INTO notes (id, body) VALUES ('n1', 'seeded')"}) + + assert talk(project, work, "--reset-options", given) == ["seeded"] + + def test_two_calls_in_flight_serialise(project: Project) -> None: """Calls into one instance queue on the instance's lock, so no update is lost.""" @@ -585,9 +596,9 @@ def test_a_startup_keyword_at_the_top_level_is_refused_before_anything_is_served assert finished.code == 1 assert finished.err.splitlines() == [ '--reset-options does not take "region"; --reset-options takes "clock_mode", "fixture", ' - '"now", "seed", "startup" and "state_format", and a world\'s own startup keywords go ' - 'inside "startup": --reset-options \'{"fixture": "small_startup", "startup": ' - '{"user_id": "u_12"}}\'' + '"now", "seed", "setup_sql", "startup" and "state_format", and a world\'s own startup ' + 'keywords go inside "startup": --reset-options \'{"fixture": "small_startup", ' + '"startup": {"user_id": "u_12"}}\'' ] assert finished.out == "" assert project.living() == [] diff --git a/tests/test_reset_schema.py b/tests/test_reset_schema.py index 22a9d0a..5cf42dd 100644 --- a/tests/test_reset_schema.py +++ b/tests/test_reset_schema.py @@ -86,6 +86,7 @@ def test_the_reference_world_publishes_its_reset_message() -> None: assert list(reset["properties"]) == [ "fixture", "startup", + "setup_sql", "now", "clock_mode", "state_format", @@ -98,6 +99,9 @@ def test_the_reference_world_publishes_its_reset_message() -> None: {"$ref": "#/$defs/SeahavenStartup"}, {"type": "null"}, ] + # The console renders a string field titled like SQL as a multi-line box. + assert reset["properties"]["setup_sql"]["anyOf"] == [{"type": "string"}, {"type": "null"}] + assert reset["properties"]["setup_sql"]["title"] == "Setup Sql" startup = reset["$defs"]["SeahavenStartup"] assert list(startup["properties"]) == ["user_id"] assert startup["additionalProperties"] is False diff --git a/tests/test_server.py b/tests/test_server.py index 609368c..80f179f 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -419,13 +419,14 @@ def misuse(ctx: Ctx) -> None: SCHEMA = json.loads((Path(__file__).parent / "state_v1.schema.json").read_text()) # The one episode both clients drive: a fixture, a seed, a named episode, a -# startup keyword and one write, so no envelope field below is left at its -# default. +# startup keyword, setup SQL and one write, so no envelope field below is left +# at its default. The setup row is not in the log: only calls are. EPISODE: dict[str, Any] = { "seed": 7, "episode_id": "ep-1", "clock_mode": "fixed", "startup": {"tenant": "globex"}, + "setup_sql": "INSERT INTO notes VALUES ('n2', 'seeded', 0)", } WRITE = CallToolAction(tool_name="execute", arguments={"sql": insert("n1")}) @@ -465,6 +466,7 @@ def expected_document(world: World, fixture_id: str) -> dict[str, Any]: "now": INSTANT_ISO, "clock_mode": "fixed", "startup": {"tenant": "globex"}, + "setup_sql": "INSERT INTO notes VALUES ('n2', 'seeded', 0)", "call_count": 1, "state": { "db": { @@ -610,6 +612,21 @@ def test_an_unknown_startup_keyword_is_an_error_frame_and_the_session_survives( assert env.call("rows", sql="SELECT 1 AS n").result == [{"n": 1}] +def test_setup_sql_reaches_the_instance_over_the_wire(world: World) -> None: + """A refused statement is an error frame, and the session takes the next reset.""" + setup_sql = "INSERT INTO notes VALUES ('n1', 'a body', 0)" + with serving(world) as url, SeahavenClient(base_url=url) as env: + with pytest.raises( + RuntimeError, + match=r"(?s)setup_sql statement 1 of 1 was refused: it changes the schema\..*" + r"\(code: EXECUTION_ERROR\)", + ): + env.reset(setup_sql="DROP TABLE notes") + env.reset(setup_sql=setup_sql) + assert ids(env.call("rows", sql="SELECT id FROM notes")) == ["n1"] + assert env.state().setup_sql == setup_sql + + def test_a_second_reset_over_the_wire_starts_from_the_fixture_again(world: World) -> None: fixture_id = _freeze(world) with serving(world) as url, SeahavenClient(base_url=url) as env: diff --git a/tests/test_setup_sql.py b/tests/test_setup_sql.py new file mode 100644 index 0000000..037c158 --- /dev/null +++ b/tests/test_setup_sql.py @@ -0,0 +1,862 @@ +"""`world.instance(setup_sql=...)`: a caller's SQL, run on a new instance before its first call. + +Driven through `world.instance(...)` throughout, on small inline worlds: a leaf +with a full-text index, and a host that adds `payments`, which adds `tax`. The +unit tests at the end cover the splitter and the error text on their own. +""" + +import re +import uuid +from contextlib import closing +from pathlib import Path +from typing import Any + +import apsw +import pytest + +from seahaven.clock import Clock +from seahaven.ctx import Ctx +from seahaven.db import build_blank +from seahaven.errors import WorldBug +from seahaven.instances import Instance +from seahaven.setup_sql import ( + Refused, + SetupAuthorizer, + _explain, + run_setup_sql, + split_statements, +) +from seahaven.world import World +from tests.conftest import INSTANT_ISO, build_world + +SCHEMA = """ +CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT NOT NULL) STRICT; + +CREATE TABLE issues ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + owner_id TEXT REFERENCES users (id) +) STRICT; + +CREATE VIRTUAL TABLE issues_fts USING fts5(title, content='issues', content_rowid='rowid'); + +CREATE TRIGGER issues_ai AFTER INSERT ON issues BEGIN + INSERT INTO issues_fts (rowid, title) VALUES (new.rowid, new.title); +END; + +CREATE TRIGGER issues_au AFTER UPDATE ON issues BEGIN + INSERT INTO issues_fts (issues_fts, rowid, title) VALUES ('delete', old.rowid, old.title); + INSERT INTO issues_fts (rowid, title) VALUES (new.rowid, new.title); +END; +""" + +SCHEMA_FIX = ( + "setup_sql may only read and write rows (INSERT, UPDATE, DELETE, SELECT). Change the schema " + "in the world and bump its version." +) +TRANSACTION_FIX = ( + "setup_sql already runs in one transaction that Seahaven opens and commits; remove BEGIN, " + "COMMIT and SAVEPOINT." +) +ATTACH_FIX = ( + "Every node is already attached: name the root's tables unqualified and an added node's as " + ".
" +) +PRAGMA_FIX = "Only read-only pragmas such as table_info are allowed." + + +def leaf(tmp_path: Path) -> World: + """Users and issues, with a full-text index kept in step by triggers.""" + world = build_world(tmp_path, SCHEMA, name="tracker") + + @world.tool + def search(ctx: Ctx, query: str) -> list[str]: + """The ids of the issues whose title matches.""" + return [ + row["id"] + for row in ctx.db.rows( + "SELECT issues.id FROM issues_fts JOIN issues ON issues.rowid = issues_fts.rowid " + "WHERE issues_fts MATCH ? ORDER BY issues.id", + query, + ) + ] + + return world + + +def composed(tmp_path: Path) -> World: + """`orders` at the root, which adds `payments` (`charges`), which adds `tax` (`rates`).""" + host = build_world( + tmp_path, + "CREATE TABLE orders (id TEXT PRIMARY KEY, total INTEGER NOT NULL) STRICT;", + name="shop", + ) + payments = World( + "payments", + "1.0.0", + "CREATE TABLE charges (id TEXT PRIMARY KEY, amount INTEGER NOT NULL) STRICT;", + state_format="seahaven.state/1", + ) + tax = World( + "tax", + "1.0.0", + "CREATE TABLE rates (region TEXT PRIMARY KEY, percent INTEGER NOT NULL) STRICT;", + state_format="seahaven.state/1", + ) + payments.add_world(tax, name="tax") + host.add_world(payments, name="payments") + return host + + +def instance_dirs(tmp_path: Path) -> list[Path]: + """Every instance directory under the test's working root.""" + return [ + path for path in (tmp_path / "work").rglob("*") if path.is_dir() and _is_uuid(path.name) + ] + + +def _is_uuid(name: str) -> bool: + try: + uuid.UUID(name) + except ValueError: + return False + return True + + +def first_line(error: pytest.ExceptionInfo[WorldBug]) -> str: + return str(error.value).splitlines()[0] + + +def rows(live: Instance, sql: str) -> list[dict[str, Any]]: + return live.inspect().rows(sql) + + +# ------------------------------------------------------------------ behaviour + + +def test_rows_written_by_setup_sql_are_there_for_the_first_call(tmp_path: Path) -> None: + sql = "INSERT INTO users (id, name) VALUES ('u1', 'Ada')" + with leaf(tmp_path).instance(None, setup_sql=sql) as live: + assert live.call("rows", sql="SELECT id, name FROM users") == [{"id": "u1", "name": "Ada"}] + assert rows(live, "SELECT id FROM users") == [{"id": "u1"}] + + +def test_statements_run_in_the_order_written(tmp_path: Path) -> None: + sql = """ + INSERT INTO users (id, name) VALUES ('u1', 'Ada'); + UPDATE users SET name = name || ' Lovelace' WHERE id = 'u1'; + INSERT INTO issues (id, title, owner_id) SELECT 'i1', name, id FROM users; + """ + with leaf(tmp_path).instance(None, setup_sql=sql) as live: + assert rows(live, "SELECT title, owner_id FROM issues") == [ + {"title": "Ada Lovelace", "owner_id": "u1"} + ] + + +def test_a_semicolon_inside_a_literal_does_not_split(tmp_path: Path) -> None: + sql = "INSERT INTO users (id, name) VALUES ('u1', 'a;b'); INSERT INTO users VALUES ('u2', ';')" + with leaf(tmp_path).instance(None, setup_sql=sql) as live: + assert rows(live, "SELECT name FROM users ORDER BY id") == [{"name": "a;b"}, {"name": ";"}] + + +def test_unqualified_names_write_the_root_and_qualified_names_write_a_node( + tmp_path: Path, +) -> None: + sql = """ + INSERT INTO orders (id, total) VALUES ('o1', 100); + INSERT INTO main.orders (id, total) VALUES ('o2', 50); + INSERT INTO payments.charges (id, amount) VALUES ('c1', 100); + INSERT INTO payments__tax.rates (region, percent) VALUES ('eu', 20); + """ + with composed(tmp_path).instance(None, setup_sql=sql) as live, live.bulk() as ctx: + assert ctx.db.rows("SELECT id FROM orders ORDER BY id") == [{"id": "o1"}, {"id": "o2"}] + assert ctx.worlds.payments.db.rows("SELECT id FROM charges") == [{"id": "c1"}] + assert ctx.worlds.payments.worlds.tax.db.rows("SELECT region FROM rates") == [ + {"region": "eu"} + ] + + +def test_one_statement_reads_one_node_and_writes_another(tmp_path: Path) -> None: + sql = """ + INSERT INTO orders (id, total) VALUES ('o1', 100), ('o2', 40); + INSERT INTO payments.charges (id, amount) SELECT 'c-' || id, total FROM orders; + """ + with composed(tmp_path).instance(None, setup_sql=sql) as live: + assert rows(live, "SELECT id, amount FROM payments.charges ORDER BY id") == [ + {"id": "c-o1", "amount": 100}, + {"id": "c-o2", "amount": 40}, + ] + + +def test_setup_sql_runs_on_a_copied_fixture_and_fts5_follows_it(tmp_path: Path) -> None: + world = leaf(tmp_path) + with world.instance(None, now=INSTANT_ISO, clock_mode="fixed") as live: + live.call("execute", sql="INSERT INTO issues (id, title) VALUES ('i1', 'Login fails')") + live.freeze("seeded", "one issue") + + sql = "UPDATE issues SET title = 'Export is slow' WHERE id = 'i1'" + with world.instance("seeded", setup_sql=sql) as live: + assert live.call("search", query="export") == ["i1"] + assert live.call("search", query="login") == [] + + with world.instance("seeded") as live: + assert live.call("search", query="login") == ["i1"] + + +def test_fts5_commands_select_and_read_only_pragmas_are_allowed(tmp_path: Path) -> None: + sql = """ + INSERT INTO issues (id, title) VALUES ('i1', 'Dark mode'); + INSERT INTO issues_fts (issues_fts) VALUES ('rebuild'); + SELECT count(*) FROM issues; + PRAGMA table_info(issues); + """ + with leaf(tmp_path).instance(None, setup_sql=sql) as live: + assert live.call("search", query="dark") == ["i1"] + + +def test_setup_sql_may_read_a_view(tmp_path: Path) -> None: + world = build_world( + tmp_path, + SCHEMA + "CREATE VIEW named AS SELECT id, upper(name) AS shout FROM users;", + ) + sql = """ + INSERT INTO users (id, name) VALUES ('u1', 'Ada'); + INSERT INTO issues (id, title, owner_id) SELECT 'i1', shout, id FROM named; + """ + with world.instance(None, setup_sql=sql) as live: + assert rows(live, "SELECT title FROM issues") == [{"title": "ADA"}] + + +TRIGGERED = """ +CREATE TABLE items (id TEXT PRIMARY KEY, tags TEXT NOT NULL DEFAULT '[]') STRICT; +CREATE TABLE audit ( + id INTEGER PRIMARY KEY, item_id TEXT NOT NULL, changed INTEGER, tag TEXT +) STRICT; + +CREATE TRIGGER items_counted AFTER INSERT ON items BEGIN + INSERT INTO audit (item_id, changed) VALUES (new.id, changes()); +END; + +CREATE TRIGGER items_tagged AFTER UPDATE OF tags ON items BEGIN + INSERT INTO audit (item_id, tag) SELECT new.id, value FROM json_each(new.tags); +END; + +CREATE VIEW item_changes AS SELECT id, changes() AS changed FROM items; + +CREATE VIEW live_items AS SELECT id, tags FROM items; + +CREATE TRIGGER live_items_update INSTEAD OF UPDATE ON live_items BEGIN + UPDATE items SET tags = new.tags WHERE id = old.id; +END; + +CREATE TRIGGER live_items_delete INSTEAD OF DELETE ON live_items BEGIN + DELETE FROM items WHERE id = old.id; +END; +""" + +# A trigger that shares the view's name, which SQLite allows: a name is trusted +# only as a trigger, so this one's body is held to the allowlists. +NAMESAKE = """ +CREATE TRIGGER live_items AFTER DELETE ON items BEGIN + INSERT INTO audit (item_id, changed) VALUES (old.id, changes()); +END; +""" + + +def test_a_worlds_trigger_may_call_a_function_setup_sql_may_not(tmp_path: Path) -> None: + world = build_world(tmp_path, TRIGGERED) + with world.instance(None, setup_sql="INSERT INTO items (id) VALUES ('a')") as live: + assert rows(live, "SELECT item_id, changed FROM audit") == [{"item_id": "a", "changed": 0}] + + +def test_a_worlds_trigger_may_read_json_each(tmp_path: Path) -> None: + sql = """ + INSERT INTO items (id) VALUES ('a'); + UPDATE items SET tags = '["red", "blue"]' WHERE id = 'a'; + """ + with build_world(tmp_path, TRIGGERED).instance(None, setup_sql=sql) as live: + assert rows(live, "SELECT tag FROM audit WHERE tag IS NOT NULL ORDER BY tag") == [ + {"tag": "blue"}, + {"tag": "red"}, + ] + + +def test_an_added_nodes_trigger_is_trusted_too(tmp_path: Path) -> None: + host = build_world(tmp_path, "CREATE TABLE orders (id TEXT PRIMARY KEY) STRICT;", name="shop") + payments = World( + "payments", + "1.0.0", + """ + CREATE TABLE charges (id TEXT PRIMARY KEY, changed INTEGER) STRICT; + CREATE TRIGGER charges_counted AFTER INSERT ON charges BEGIN + UPDATE charges SET changed = changes() WHERE id = new.id; + END; + """, + state_format="seahaven.state/1", + ) + host.add_world(payments, name="payments") + + sql = "INSERT INTO payments.charges (id) VALUES ('c1')" + with host.instance(None, setup_sql=sql) as live: + assert rows(live, "SELECT id, changed FROM payments.charges") == [ + {"id": "c1", "changed": 0} + ] + + +def test_a_worlds_view_is_held_to_the_allowlists(tmp_path: Path) -> None: + with pytest.raises(WorldBug) as error: + build_world(tmp_path, TRIGGERED).instance(None, setup_sql="SELECT * FROM item_changes") + + assert first_line(error) == ( + "setup_sql statement 1 of 1 was refused: function changes() is not allowed. setup_sql " + "has the functions run_sql has." + ) + + +def test_dml_on_a_view_runs_its_instead_of_triggers(tmp_path: Path) -> None: + sql = """ + INSERT INTO items (id) VALUES ('a'), ('b'); + UPDATE live_items SET tags = '["red"]' WHERE id = 'a'; + DELETE FROM live_items WHERE id = 'b'; + """ + with build_world(tmp_path, TRIGGERED).instance(None, setup_sql=sql) as live: + assert rows(live, "SELECT id, tags FROM items") == [{"id": "a", "tags": '["red"]'}] + + +@pytest.mark.parametrize("schema", [TRIGGERED, TRIGGERED + NAMESAKE], ids=["view", "namesake"]) +@pytest.mark.parametrize( + ("statement", "message"), + [ + ( + "UPDATE live_items SET tags = (SELECT count(*) FROM sqlite_master)", + "read of table 'sqlite_master' is not allowed. setup_sql may only read and write the " + "world's own tables.", + ), + ( + "DELETE FROM live_items WHERE sqlite_version() IS NOT NULL", + "function sqlite_version() is not allowed. setup_sql has the functions run_sql has.", + ), + ], +) +def test_the_callers_own_sql_in_dml_on_a_view_is_not_trusted( + tmp_path: Path, schema: str, statement: str, message: str +) -> None: + with pytest.raises(WorldBug) as error: + build_world(tmp_path, schema).instance(None, setup_sql=statement) + + assert first_line(error) == f"setup_sql statement 1 of 1 was refused: {message}" + + +def test_a_cte_named_after_a_world_trigger_is_a_documented_residual(tmp_path: Path) -> None: + """SQLite's auth context names a CTE as it names a trigger, by name alone. + + So a CTE the caller names after a world trigger shares that trigger's trust + for function calls and reads. Accepted: setup SQL comes from the eval's + author, and these checks catch mistakes rather than stop an attacker. Writes, + schema changes, attaches and pragmas stay refused whatever the CTE is called. + """ + sql = """ + WITH items_counted AS ( + SELECT name || '-' || typeof(sqlite_version()) AS v + FROM sqlite_master WHERE name = 'items' + ) + INSERT INTO items (id) SELECT v FROM items_counted + """ + with build_world(tmp_path, TRIGGERED).instance(None, setup_sql=sql) as live: + # It ran and was not refused: nothing here tells the CTE from the trigger. + assert rows(live, "SELECT id FROM items") == [{"id": "items-text"}] + + with pytest.raises(WorldBug, match="was refused: function sqlite_version"): + build_world(tmp_path, TRIGGERED).instance( + None, + setup_sql="WITH named AS (SELECT sqlite_version() AS v) " + "INSERT INTO items (id) SELECT v FROM named", + ) + + +def test_a_trigger_that_shares_a_views_name_is_not_trusted(tmp_path: Path) -> None: + sql = "INSERT INTO items (id) VALUES ('a'); DELETE FROM items" + with pytest.raises(WorldBug) as error: + build_world(tmp_path, TRIGGERED + NAMESAKE).instance(None, setup_sql=sql) + + assert first_line(error) == ( + "setup_sql statement 2 of 2 was refused: function changes() is not allowed. setup_sql " + "has the functions run_sql has." + ) + + +@pytest.mark.parametrize( + ("statement", "message"), + [ + ( + "INSERT INTO items (id) VALUES (changes())", + "function changes() is not allowed. setup_sql has the functions run_sql has.", + ), + ( + "SELECT * FROM item_changes WHERE changes() >= 0", + "function changes() is not allowed. setup_sql has the functions run_sql has.", + ), + ( + "UPDATE items SET tags = (SELECT json_group_array(value) FROM json_each('[1]'))", + "read of table 'json_each' is not allowed. setup_sql may only read and write the " + "world's own tables.", + ), + ( + "INSERT INTO items (id) SELECT name FROM sqlite_master", + "read of table 'sqlite_master' is not allowed. setup_sql may only read and write the " + "world's own tables.", + ), + ], +) +def test_the_callers_own_sql_is_still_refused_beside_a_worlds_trigger( + tmp_path: Path, statement: str, message: str +) -> None: + with pytest.raises(WorldBug) as error: + build_world(tmp_path, TRIGGERED).instance(None, setup_sql=statement) + + assert first_line(error) == f"setup_sql statement 1 of 1 was refused: {message}" + + +def test_a_startup_hook_sees_the_rows_setup_sql_wrote(tmp_path: Path) -> None: + world = leaf(tmp_path) + + @world.instance_startup + def sign_in(ctx: Ctx, *, user_id: str) -> None: + ctx.state["user"] = ctx.db.one("SELECT name FROM users WHERE id = ?", user_id) + + sql = "INSERT INTO users (id, name) VALUES ('u9', 'Grace')" + with world.instance(None, setup_sql=sql, startup={"user_id": "u9"}) as live: + assert live.ctx.state["user"] == {"name": "Grace"} + + +def test_the_same_seed_and_setup_sql_give_the_same_rows(tmp_path: Path) -> None: + sql = "INSERT INTO users (id, name) VALUES (hex(randomblob(8)), random())" + + def drawn(seed: int) -> list[dict[str, Any]]: + with leaf(tmp_path).instance(None, seed=seed, setup_sql=sql) as live: + return rows(live, "SELECT id, name FROM users") + + assert drawn(7) == drawn(7) + assert drawn(7) != drawn(8) + + +def test_random_draws_from_the_roots_stream_whichever_node_the_row_lands_in( + tmp_path: Path, +) -> None: + world = composed(tmp_path) + into_root = "INSERT INTO orders (id, total) VALUES (hex(randomblob(8)), 1)" + into_node = "INSERT INTO payments.charges (id, amount) VALUES (hex(randomblob(8)), 1)" + + with world.instance(None, seed=7, setup_sql=into_root) as live: + expected = rows(live, "SELECT id FROM orders") + with world.instance(None, seed=7, setup_sql=into_node) as live: + assert rows(live, "SELECT id FROM payments.charges") == expected + + +def test_setup_sql_draws_from_a_stream_of_its_own(tmp_path: Path) -> None: + """An instance's own draws are the same whether or not setup SQL drew first.""" + world = leaf(tmp_path) + draw = "SELECT random() AS r, hex(randomblob(4)) AS b" + + with world.instance(None, seed=7) as live: + expected = live.call("rows", sql=draw) + with world.instance(None, seed=7, setup_sql="SELECT random(), randomblob(4)") as live: + assert live.call("rows", sql=draw) == expected + with world.instance( + None, seed=7, setup_sql="INSERT INTO users VALUES (random(), hex(randomblob(4)))" + ) as live: + assert rows(live, "SELECT CAST(id AS INTEGER) AS r, name AS b FROM users") != expected + + +def test_the_clock_functions_read_the_instances_clock(tmp_path: Path) -> None: + sql = "INSERT INTO users (id, name) VALUES ('u1', datetime('now'))" + with leaf(tmp_path).instance(None, now=INSTANT_ISO, clock_mode="fixed", setup_sql=sql) as live: + assert rows(live, "SELECT name FROM users") == [{"name": INSTANT_ISO}] + + +def test_setup_sql_rows_are_not_in_the_change_log(tmp_path: Path) -> None: + sql = "INSERT INTO users (id, name) VALUES ('u1', 'Ada')" + with leaf(tmp_path).instance(None, setup_sql=sql) as live: + assert (live.change_log(), live.call_count) == ([], 0) + + live.call("execute", sql="UPDATE users SET name = 'Grace' WHERE id = 'u1'") + + assert [(r.i, r.table, r.op, r.before, r.after) for r in live.change_log()] == [ + (0, "users", "update", {"name": "Ada"}, {"name": "Grace"}) + ] + + +def test_setup_sql_is_kept_on_the_instance(tmp_path: Path) -> None: + world = leaf(tmp_path) + sql = " INSERT INTO users VALUES ('u1', 'Ada'); " + with world.instance(None, setup_sql=sql) as live: + assert live.setup_sql == sql + with world.instance(None) as live: + assert live.setup_sql is None + + +@pytest.mark.parametrize("sql", ["", " \n\t ", "-- nothing to do", ";;", "/* a */ ; -- b"]) +def test_an_empty_string_whitespace_or_a_comment_changes_nothing(tmp_path: Path, sql: str) -> None: + with leaf(tmp_path).instance(None, setup_sql=sql) as live: + assert rows(live, "SELECT count(*) AS n FROM users") == [{"n": 0}] + assert live.setup_sql == sql + + +# ------------------------------------------------------------------- refusals + + +@pytest.mark.parametrize( + ("statement", "message"), + [ + ("CREATE TABLE extra (x)", f"was refused: it changes the schema. {SCHEMA_FIX}"), + ("DROP TABLE users", f"was refused: it changes the schema. {SCHEMA_FIX}"), + ( + "CREATE INDEX by_name ON users (name)", + f"was refused: it changes the schema. {SCHEMA_FIX}", + ), + ("ALTER TABLE users ADD COLUMN age", f"was refused: it changes the schema. {SCHEMA_FIX}"), + ( + "CREATE TRIGGER t AFTER INSERT ON users BEGIN SELECT 1; END", + f"was refused: it changes the schema. {SCHEMA_FIX}", + ), + ("CREATE VIEW v AS SELECT 1", f"was refused: it changes the schema. {SCHEMA_FIX}"), + ("REINDEX", f"was refused: it changes the schema. {SCHEMA_FIX}"), + ("ANALYZE", f"was refused: it changes the schema. {SCHEMA_FIX}"), + ("BEGIN", f"was refused: it controls the transaction. {TRANSACTION_FIX}"), + ("COMMIT", f"was refused: it controls the transaction. {TRANSACTION_FIX}"), + ("ROLLBACK", f"was refused: it controls the transaction. {TRANSACTION_FIX}"), + ("SAVEPOINT s", f"was refused: it controls the transaction. {TRANSACTION_FIX}"), + ("RELEASE s", f"was refused: it controls the transaction. {TRANSACTION_FIX}"), + ( + "ATTACH 'elsewhere.sqlite' AS elsewhere", + f"was refused: it attaches or detaches a database. {ATTACH_FIX} " + "(this world adds none).", + ), + ( + "DETACH main", + f"was refused: it attaches or detaches a database. {ATTACH_FIX} " + "(this world adds none).", + ), + ( + "PRAGMA journal_mode = DELETE", + f"was refused: PRAGMA journal_mode can change the connection. {PRAGMA_FIX}", + ), + ( + "PRAGMA foreign_keys = OFF", + f"was refused: PRAGMA foreign_keys can change the connection. {PRAGMA_FIX}", + ), + ( + "SELECT load_extension('evil')", + "was refused: function load_extension() is not allowed. setup_sql has the functions " + "run_sql has.", + ), + ( + "SELECT * FROM sqlite_master", + "was refused: read of table 'sqlite_master' is not allowed. setup_sql may only read " + "and write the world's own tables.", + ), + ( + "INSERT INTO issues_fts_data (id, block) VALUES (99, x'00')", + "failed: table issues_fts_data may not be modified.", + ), + ("UPDATE sqlite_master SET sql = ''", "failed: table sqlite_master may not be modified."), + ], +) +def test_a_refused_statement_names_what_was_refused( + tmp_path: Path, statement: str, message: str +) -> None: + sql = f"INSERT INTO users (id, name) VALUES ('u1', 'Ada'); {statement}" + with pytest.raises(WorldBug) as error: + leaf(tmp_path).instance(None, setup_sql=sql) + + assert first_line(error) == f"setup_sql statement 2 of 2 {message}" + assert str(error.value).splitlines()[1] == f" {statement}" + assert isinstance(error.value.__cause__, apsw.Error) + + +def test_a_refusal_leaves_no_instance_directory(tmp_path: Path) -> None: + world = leaf(tmp_path) + with pytest.raises(WorldBug, match="statement 2 of 2 was refused"): + world.instance(None, setup_sql="INSERT INTO users VALUES ('u1', 'Ada'); DROP TABLE users") + + assert instance_dirs(tmp_path) == [] + with world.instance(None) as live: + assert rows(live, "SELECT count(*) AS n FROM users") == [{"n": 0}] + assert len(instance_dirs(tmp_path)) == 1 + + +def test_an_attach_in_a_composed_world_names_its_schemas(tmp_path: Path) -> None: + with pytest.raises(WorldBug) as error: + composed(tmp_path).instance(None, setup_sql="DETACH payments") + + assert first_line(error) == ( + "setup_sql statement 1 of 1 was refused: it attaches or detaches a database. " + f"{ATTACH_FIX} (this world: payments, payments__tax)." + ) + + +def test_a_syntax_error_names_the_statement_and_sqlites_message(tmp_path: Path) -> None: + with pytest.raises(WorldBug) as error: + leaf(tmp_path).instance(None, setup_sql="SELECT 1; SELEC 2; SELECT 3") + + assert first_line(error) == 'setup_sql statement 2 of 3 failed: near "SELEC": syntax error.' + + +def test_a_trailing_fragment_is_reported_by_sqlite(tmp_path: Path) -> None: + sql = "SELECT 1; INSERT INTO users VALUES ('u1', ';'" + with pytest.raises(WorldBug) as error: + leaf(tmp_path).instance(None, setup_sql=sql) + + assert first_line(error) == "setup_sql statement 2 of 2 failed: incomplete input." + + +def test_a_constraint_failure_fails_the_instance(tmp_path: Path) -> None: + sql = "INSERT INTO issues (id, title, owner_id) VALUES ('i1', 'x', 'nobody')" + with pytest.raises(WorldBug) as error: + leaf(tmp_path).instance(None, setup_sql=sql) + + assert first_line(error) == ( + "setup_sql statement 1 of 1 failed: FOREIGN KEY constraint failed." + ) + assert isinstance(error.value.__cause__, apsw.ConstraintError) + assert instance_dirs(tmp_path) == [] + + +def test_a_deferred_constraint_that_fails_at_commit_is_a_world_bug(tmp_path: Path) -> None: + world = build_world( + tmp_path, + """ + CREATE TABLE parents (id TEXT PRIMARY KEY) STRICT; + CREATE TABLE children ( + id TEXT PRIMARY KEY, + parent_id TEXT REFERENCES parents (id) DEFERRABLE INITIALLY DEFERRED + ) STRICT; + """, + ) + with pytest.raises(WorldBug) as error: + world.instance(None, setup_sql="INSERT INTO children VALUES ('c1', 'nobody')") + + assert str(error.value) == ( + "setup_sql failed at commit: FOREIGN KEY constraint failed. A deferred constraint is " + "checked once every statement has run." + ) + assert isinstance(error.value.__cause__, apsw.ConstraintError) + assert instance_dirs(tmp_path) == [] + + sql = "INSERT INTO children VALUES ('c1', 'p1'); INSERT INTO parents VALUES ('p1')" + with world.instance(None, setup_sql=sql) as live: + assert rows(live, "SELECT parent_id FROM children") == [{"parent_id": "p1"}] + + +def test_a_missing_table_in_a_composed_world_names_the_schemas(tmp_path: Path) -> None: + with pytest.raises(WorldBug) as error: + composed(tmp_path).instance(None, setup_sql="DELETE FROM refunds") + + assert first_line(error) == ( + "setup_sql statement 1 of 1 failed: no such table: refunds. An added node's tables are " + "named .
(this world: payments, payments__tax)." + ) + + +def test_a_missing_table_in_a_leaf_world_has_no_hint(tmp_path: Path) -> None: + with pytest.raises(WorldBug) as error: + leaf(tmp_path).instance(None, setup_sql="DELETE FROM charges") + + assert first_line(error) == "setup_sql statement 1 of 1 failed: no such table: charges." + + +def test_a_long_statement_is_collapsed_and_cut_in_the_message(tmp_path: Path) -> None: + statement = "INSERT INTO nowhere\n VALUES ('" + "x" * 300 + "')" + with pytest.raises(WorldBug) as error: + leaf(tmp_path).instance(None, setup_sql=statement) + + quoted = str(error.value).splitlines()[1] + assert quoted == " INSERT INTO nowhere VALUES ('" + "x" * 171 + "..." + + +@pytest.mark.parametrize("value", [["DELETE FROM users"], b"DELETE FROM users", 3]) +def test_a_non_string_setup_sql_is_refused_before_anything_is_built( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, value: object +) -> None: + def no_build(*_: object, **__: object) -> None: + raise AssertionError("built an instance before refusing setup_sql") + + monkeypatch.setattr("seahaven.instances.build_blank", no_build) + with pytest.raises(WorldBug) as error: + leaf(tmp_path).instance(None, setup_sql=value) # ty: ignore[invalid-argument-type] + + assert str(error.value) == ( + f"setup_sql takes a string of SQL statements separated by ';', not {type(value).__name__}" + ) + assert instance_dirs(tmp_path) == [] + + +# ---------------------------------------------------------- run_setup_sql alone + + +@pytest.fixture +def files(tmp_path: Path) -> tuple[Path, Path]: + """A root file and one node's file, as a new composed instance has them.""" + clock = Clock.from_iso(INSTANT_ISO) + root, node = tmp_path / "state.sqlite", tmp_path / "state.payments.sqlite" + build_blank(root, "CREATE TABLE orders (id TEXT PRIMARY KEY);", clock=clock, seed=b"s").close() + build_blank(node, "CREATE TABLE charges (id TEXT PRIMARY KEY);", clock=clock, seed=b"s").close() + return root, node + + +def test_a_failure_rolls_back_every_file(files: tuple[Path, Path]) -> None: + root, node = files + sql = """ + INSERT INTO orders VALUES ('o1'); + INSERT INTO payments.charges VALUES ('c1'); + INSERT INTO orders VALUES ('o1'); + """ + with pytest.raises(WorldBug, match="statement 3 of 3 failed: UNIQUE constraint failed"): + run_setup_sql( + sql, + root=root, + attachments=[("payments", node)], + clock=Clock.from_iso(INSTANT_ISO), + seed=b"s", + ) + + assert _count(root, "orders") == 0 + assert _count(node, "charges") == 0 + + +def test_an_on_conflict_rollback_that_ends_the_transaction_is_still_a_clean_failure( + files: tuple[Path, Path], +) -> None: + root, _ = files + sql = "INSERT INTO orders VALUES ('o1'); INSERT OR ROLLBACK INTO orders VALUES ('o1')" + with pytest.raises(WorldBug, match="statement 2 of 2 failed: UNIQUE constraint failed"): + run_setup_sql(sql, root=root, attachments=[], clock=Clock.from_iso(INSTANT_ISO), seed=b"s") + + assert _count(root, "orders") == 0 + + +def test_success_commits_every_file(files: tuple[Path, Path]) -> None: + root, node = files + run_setup_sql( + "INSERT INTO orders VALUES ('o1'); INSERT INTO payments.charges VALUES ('c1')", + root=root, + attachments=[("payments", node)], + clock=Clock.from_iso(INSTANT_ISO), + seed=b"s", + ) + + assert (_count(root, "orders"), _count(node, "charges")) == (1, 1) + + +def _count(path: Path, table: str) -> int: + with closing(apsw.Connection(str(path))) as conn: + return int(conn.execute(f"SELECT count(*) FROM {table}").get) + + +# -------------------------------------------------------------- the splitter + + +@pytest.mark.parametrize( + ("sql", "statements"), + [ + ("SELECT 1; SELECT 2;", ["SELECT 1", "SELECT 2"]), + ("SELECT 1; SELECT 2", ["SELECT 1", "SELECT 2"]), + ("SELECT 'a;b'; SELECT 2", ["SELECT 'a;b'", "SELECT 2"]), + ('SELECT 1 AS "x;y"; SELECT 2', ['SELECT 1 AS "x;y"', "SELECT 2"]), + ("SELECT 1 /* ; */; SELECT 2", ["SELECT 1 /* ; */", "SELECT 2"]), + ( + "UPDATE a SET t = 'x;y'; -- c;\n CREATE TRIGGER r AFTER INSERT ON a BEGIN " + "SELECT 1; SELECT 2; END; SELECT 3", + [ + "UPDATE a SET t = 'x;y'", + "-- c;\n CREATE TRIGGER r AFTER INSERT ON a BEGIN SELECT 1; SELECT 2; END", + "SELECT 3", + ], + ), + ("SELECT 1; SELECT 'open", ["SELECT 1", "SELECT 'open"]), + ("SELECT 1; -- done", ["SELECT 1", "-- done"]), + ("", []), + (" \n ", []), + (" ; ;\n;", []), + ], +) +def test_split_statements(sql: str, statements: list[str]) -> None: + assert split_statements(sql) == statements + + +# ------------------------------------------------------------- the error text + + +def test_an_action_with_no_reason_of_its_own_is_described_by_the_sandbox() -> None: + refused = Refused(apsw.SQLITE_READ, "secrets", "value", "read of table 'secrets'") + + assert _explain(refused, apsw.AuthError("not authorized"), []) == ( + "read of table 'secrets' is not allowed.", + "setup_sql may only read and write the world's own tables.", + ) + + +@pytest.mark.parametrize("table", ["sqlite_master", "sqlite_schema", "SQLITE_TEMP_MASTER"]) +def test_a_write_to_a_schema_table_is_a_schema_change(table: str) -> None: + refused = Refused(apsw.SQLITE_UPDATE, table, None, f"action UPDATE {table!r}") + + assert _explain(refused, apsw.AuthError("not authorized"), []) == ( + "it changes the schema.", + SCHEMA_FIX, + ) + + +def test_a_write_to_another_sqlite_table_is_refused_but_not_as_a_schema_change( + tmp_path: Path, +) -> None: + world = build_world( + tmp_path, "CREATE TABLE counted (n INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT) STRICT;" + ) + with pytest.raises(WorldBug) as error: + world.instance(None, setup_sql="DELETE FROM sqlite_sequence") + + assert first_line(error) == ( + "setup_sql statement 1 of 1 was refused: action DELETE 'sqlite_sequence' is not " + "allowed. setup_sql may only read and write the world's own tables." + ) + + +def test_a_plain_sqlite_error_other_than_a_missing_table_has_no_fix() -> None: + error = apsw.SQLError('near "x": syntax error') + + assert _explain(None, error, ["payments"]) == ('near "x": syntax error.', "") + + +def test_the_authorizer_records_the_first_refusal_until_reset() -> None: + authorizer = SetupAuthorizer(["users"]) + + assert authorizer(apsw.SQLITE_INSERT, "users", None, "main", None) == apsw.SQLITE_OK + assert authorizer(apsw.SQLITE_READ, "other", "x", "main", None) == apsw.SQLITE_DENY + assert authorizer(apsw.SQLITE_TRANSACTION, "BEGIN", None, None, None) == apsw.SQLITE_DENY + assert authorizer.refused == Refused(apsw.SQLITE_READ, "other", "x", "read of table 'other'") + + authorizer.reset() + + assert authorizer.refused is None + assert authorizer.refusals == () + + +@pytest.mark.parametrize("pragma", ["table_info", "TABLE_LIST", "foreign_key_check"]) +def test_the_authorizer_allows_the_read_only_pragmas(pragma: str) -> None: + authorizer = SetupAuthorizer([]) + + assert authorizer(apsw.SQLITE_PRAGMA, pragma, "users", "main", None) == apsw.SQLITE_OK + assert authorizer.refused is None + + +def test_the_quoted_statement_collapses_whitespace() -> None: + with pytest.raises(WorldBug) as error: + run_setup_sql( + "SELECT\n\n nope FROM\tnowhere", + root=Path(":memory:"), + attachments=[], + clock=Clock.from_iso(INSTANT_ISO), + seed=b"s", + ) + + assert re.fullmatch(r".*\n SELECT nope FROM nowhere", str(error.value), re.DOTALL) diff --git a/tests/test_state.py b/tests/test_state.py index 556192b..adde064 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -77,6 +77,7 @@ def test_the_envelope_reports_the_format_the_versions_and_the_world(instance: In "now", "clock_mode", "startup", + "setup_sql", "call_count", "state", ] @@ -175,6 +176,20 @@ def remember(ctx: Ctx, *, tier: str = "free", limits: dict[str, Any] | None = No assert live.state()["startup"] == {} +def test_the_setup_sql_is_reported_exactly_as_given(tmp_path: Path) -> None: + """The string itself, not the statements it was split into, so a reader can run it again.""" + world = build_world(tmp_path) + given = "INSERT INTO notes VALUES ('n1', 'a; b', 0);\n -- seeded\n" + + with world.instance(None, setup_sql=given) as live: + assert live.state()["setup_sql"] == given + assert live.state()["state"]["db"]["log"] == [] + with world.instance(None, setup_sql="") as live: + assert live.state()["setup_sql"] == "" + with world.instance(None) as live: + assert live.state()["setup_sql"] is None + + def test_a_startup_keyword_a_document_could_not_carry_is_refused_at_creation( tmp_path: Path, ) -> None: @@ -355,6 +370,7 @@ def test_every_built_in_answers_with_no_instance(world: World) -> None: assert (answered["episode_id"], answered["seed"], answered["now"]) == (None, None, None) assert answered["clock_mode"] is None assert answered["startup"] is None + assert answered["setup_sql"] is None assert answered["call_count"] == 0 assert log_of(answered) == [] if name == SEAHAVEN_STATE_CALLS_V1: @@ -363,7 +379,7 @@ def test_every_built_in_answers_with_no_instance(world: World) -> None: def test_a_built_in_document_validates_against_the_published_schema(tmp_path: Path) -> None: live_world = emporium_world(tmp_path) - with live_world.instance(None, now=INSTANT_ISO) as live: + with live_world.instance(None, now=INSTANT_ISO, setup_sql="SELECT 1") as live: live.call("settle_order", total=250) live.call("pay_create_charge", amount=100) diff --git a/tests/test_world.py b/tests/test_world.py index d5855e5..5b18268 100644 --- a/tests/test_world.py +++ b/tests/test_world.py @@ -788,7 +788,9 @@ def test_a_world_with_no_startup_hooks_accepts_no_startup_keywords(world: World) assert world.accepted_startup_kwargs == frozenset() -@pytest.mark.parametrize("name", ["fixture", "seed", "now", "state_format", "control_tools"]) +@pytest.mark.parametrize( + "name", ["fixture", "seed", "now", "state_format", "control_tools", "setup_sql"] +) def test_a_startup_hook_may_take_a_framework_parameters_name(world: World, name: str) -> None: """Startup keywords have their own namespace, so no name is spent by the framework.""" namespace: dict[str, Any] = {} diff --git a/tools/console/e2e/smoke.mjs b/tools/console/e2e/smoke.mjs index d80a625..6b49a36 100644 --- a/tools/console/e2e/smoke.mjs +++ b/tools/console/e2e/smoke.mjs @@ -98,12 +98,13 @@ check( "the JSON box was shown although the environment publishes a reset schema", ) // The form follows the schema's property order: fixture, then the startup -// keywords in startup's place, then the rest. +// keywords in startup's place, then setup_sql as a multi-line box, then the rest. const top = async (locator) => (await locator.boundingBox()).y const formOrder = [ await top(page.getByRole("button", { name: "agency", exact: true })), await top(page.getByLabel("startup · User Id")), await top(page.getByLabel("startup · Team")), + await top(page.getByLabel("Setup Sql")), await top(page.getByLabel("Now")), await top(page.getByLabel("Seed")), await top(page.getByLabel("Episode Id")), @@ -112,6 +113,10 @@ check( formOrder.every((y, index) => index === 0 || formOrder[index - 1] < y), `the reset form is not in schema order: ${formOrder}`, ) +check( + (await page.getByLabel("Setup Sql").evaluate((element) => element.tagName)) === "TEXTAREA", + "setup_sql is not a multi-line text box", +) await page.getByRole("button", { name: "agency", exact: true }).click() await page.getByLabel("Seed").fill("7") await page.getByLabel("startup · User Id").fill("u_dana") diff --git a/tools/console/mock/server.mjs b/tools/console/mock/server.mjs index c9e8821..71499d8 100644 --- a/tools/console/mock/server.mjs +++ b/tools/console/mock/server.mjs @@ -282,6 +282,13 @@ const RESET_SCHEMA = { default: null, description: "The world's own startup keywords, passed to its startup hooks.", }, + setup_sql: { + anyOf: [{ type: "string" }, { type: "null" }], + default: null, + title: "Setup Sql", + description: + "SQL statements, separated by ';', run against the new instance before its startup hooks. Rows only: INSERT, UPDATE, DELETE and SELECT.", + }, now: { anyOf: [{ type: "string" }, { type: "null" }], default: null, diff --git a/worlds/projecttracker/tests/test_issues.py b/worlds/projecttracker/tests/test_issues.py index 1970334..5c85381 100644 --- a/worlds/projecttracker/tests/test_issues.py +++ b/worlds/projecttracker/tests/test_issues.py @@ -5,7 +5,7 @@ import pytest import seahaven -from conftest import BLANK_NOW, SMALL_STARTUP, Scaffold, an_issue +from conftest import AGENCY, BLANK_NOW, SMALL_STARTUP, Scaffold, an_issue pytestmark = pytest.mark.seahaven(fixture=None, now=BLANK_NOW, clock_mode="fixed") @@ -410,6 +410,27 @@ def test_the_fixtures_viewer_is_the_actor_when_a_call_names_none( assert filed["creator_id"] == admin["id"] +NEW_HIRE = "0f6c2d4e-8a1b-4c3d-9e5f-7a8b9c0d1e2f" + + +@pytest.mark.seahaven( + fixture=AGENCY, + setup_sql=( + "INSERT INTO users (id, email, name, role, created_at) VALUES " + f"('{NEW_HIRE}', 'new.hire@example.com', 'New Hire', 'member', '2026-01-05T09:00:00.000Z')" + ), + startup={"user_id": NEW_HIRE}, +) +def test_a_user_setup_sql_inserts_can_be_the_viewer_who_files_an_issue( + instance: seahaven.Instance, +) -> None: + """The startup hook runs after the setup SQL, so it finds the user the SQL wrote.""" + project = instance.call("list_projects")["projects"][0] + filed = instance.call("create_issue", project_id=project["id"], title="My first issue") + assert filed["creator_id"] == NEW_HIRE + assert "users" not in {record.table for record in instance.change_log()} + + @pytest.mark.seahaven(fixture=SMALL_STARTUP) def test_the_forty_first_issue_of_the_small_startup_is_eng_41( instance: seahaven.Instance,