From 3e95313d8b673eb0abdb5f1a93d9196df65e616c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Apr 2026 23:41:23 +0000 Subject: [PATCH 1/3] Expose agent reasoning + validator-observed behaviour on experiments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three trust tiers now flow from each Phase A run to the experiments table and the public dashboard JSON: self_reported motivation / reasoning / tool_calls / trace (miner-authored; trivially fakeable) observed agent_behavior — pod wall-clock, exit status, per-category proxy call/error counters keyed by miner UID (validator-measured at the proxy boundary, hard to fake) verified results / score / manifest_sha256 (Phase C trust anchor) `Proposal` gains optional `reasoning` and `tool_calls` so agents can self-report intent. `DataElement` and the `experiments` Postgres schema gain `reasoning TEXT`, `tool_calls JSONB`, and `agent_behavior JSONB` (with ADD COLUMN IF NOT EXISTS migrations). `to_api_dict()` adds a `provenance` block grouping fields by tier; flat keys remain for backwards compatibility. Validator-side instrumentation lives in `validator/db_proxy.py`: `auth_middleware` bumps a per-UID counter on each authenticated agent request, `rotate_agent_token()` resets the counters at round start, and `get_agent_behavior(uid)` returns a snapshot for `collection.py` to attach. `validator/collection.py` measures pod wall-clock, snapshots the proxy counters, and propagates everything through the R2 proposal JSON so all validators see the same picture. Self-reported text is capped at 256 KB per field for storage hygiene (no redaction needed — agents only get round-scoped ephemeral credentials and presigned URLs, so there are no long-lived secrets to leak). --- miner_template/agent.py | 37 +++++++- runner/agent/harness.py | 12 +++ shared/database.py | 76 ++++++++++++++-- shared/pg_schema.py | 45 ++++++++-- shared/protocol.py | 16 +++- tests/test_collection.py | 91 +++++++++++++++++-- tests/test_database.py | 52 +++++++++++ tests/test_db_proxy.py | 78 ++++++++++++++++ tests/test_pg_store.py | 35 ++++++++ validator/collection.py | 190 +++++++++++++++++++++++++++++++++------ validator/db_proxy.py | 73 +++++++++++++++ validator/neuron.py | 8 +- 12 files changed, 659 insertions(+), 54 deletions(-) diff --git a/miner_template/agent.py b/miner_template/agent.py index bba065d..76f10bb 100644 --- a/miner_template/agent.py +++ b/miner_template/agent.py @@ -17,11 +17,17 @@ Your agent module MUST define: design_architecture(challenge: dict, client: GatedClient) -> dict - Returns {"code": str, "name": str, "motivation": str} + Returns {"code": str, "name": str, "motivation": str, + "reasoning": str (optional), "tool_calls": list (optional)} The ``code`` field should be a Python module that defines: build_model(context_len, prediction_len, num_variates, quantiles) -> nn.Module build_optimizer(model) -> torch.optim.Optimizer + +``reasoning`` and ``tool_calls`` are exposed verbatim on the public dashboard +and saved with the experiment. They're self-reported, so don't expect them to +count for trust — pair them with the validator-observed proxy access logs if +you want a story that holds up under scrutiny. """ import json @@ -150,8 +156,14 @@ def design_architecture(challenge: dict, client) -> dict: Use client.get_json(url), client.post_json(url, payload), etc. Returns: - dict with keys: code, name, motivation + dict with keys: code, name, motivation, reasoning, tool_calls """ + # Self-reported reasoning trace + tool call log. The validator copies + # these straight onto the experiment row, so anything you append here + # shows up on the public dashboard. Append entries as you go. + reasoning_steps: list[str] = [] + tool_calls: list[dict] = [] + # Load persistent state from previous rounds scratch_dir = load_scratchpad(challenge) @@ -159,6 +171,10 @@ def design_architecture(challenge: dict, client) -> dict: min_flops = challenge.get("min_flops_equivalent", 0) max_flops = challenge.get("max_flops_equivalent", 0) log(f"Designing for FLOPs range [{min_flops}, {max_flops}], frontier size: {len(frontier)}") + reasoning_steps.append( + f"Round target: FLOPs [{min_flops}, {max_flops}], " + f"frontier has {len(frontier)} entries." + ) # Example: search arxiv for relevant papers # papers = _search_arxiv(client, challenge, "time series forecasting transformer") @@ -176,6 +192,15 @@ def design_architecture(challenge: dict, client) -> dict: hp = _pick_hyperparams(min_flops, max_flops) log(f"Selected hyperparams: {hp}") + reasoning_steps.append( + f"Picked preset {hp} targeting ~60% of FLOPs cap to stay in range." + ) + tool_calls.append({ + "tool": "internal", + "name": "_pick_hyperparams", + "input": {"min_flops": min_flops, "max_flops": max_flops}, + "output": hp, + }) if frontier: best = min(frontier, key=lambda x: x.get("metric", float("inf"))) @@ -183,11 +208,17 @@ def design_architecture(challenge: dict, client) -> dict: f"Improving on frontier best (metric={best.get('metric', '?')}). " f"Target FLOPs range: [{min_flops}, {max_flops}]" ) + reasoning_steps.append( + f"Frontier present; aiming to beat best metric={best.get('metric', '?')}." + ) else: motivation = ( f"No frontier in size range [{min_flops}, {max_flops}]. " f"Exploring with baseline transformer." ) + reasoning_steps.append( + "No frontier in this bucket; submitting baseline transformer to bootstrap." + ) d_model = hp["d_model"] nhead = hp["nhead"] @@ -234,4 +265,6 @@ def build_optimizer(model): "code": code, "name": f"transformer_d{d_model}_l{num_layers}_ff{dim_feedforward}", "motivation": motivation, + "reasoning": "\n".join(reasoning_steps), + "tool_calls": tool_calls, } diff --git a/runner/agent/harness.py b/runner/agent/harness.py index 3e5fba0..7dc094c 100644 --- a/runner/agent/harness.py +++ b/runner/agent/harness.py @@ -225,10 +225,22 @@ def main(): print(json.dumps({"error": "design_architecture() must return dict with 'code' key"})) sys.exit(1) + # Pass through optional self-reported reasoning trace and tool-call log + # if the agent populated them. Both are open-schema; the harness does + # not validate or normalise contents — that's the validator's job. + reasoning = result.get("reasoning", "") + tool_calls = result.get("tool_calls", []) + if not isinstance(reasoning, str): + reasoning = str(reasoning) + if not isinstance(tool_calls, list): + tool_calls = [] + proposal = { "code": result.get("code", ""), "name": result.get("name", ""), "motivation": result.get("motivation", ""), + "reasoning": reasoning, + "tool_calls": tool_calls, } print(json.dumps(proposal)) diff --git a/shared/database.py b/shared/database.py index b2a5f79..6d7d964 100644 --- a/shared/database.py +++ b/shared/database.py @@ -17,7 +17,17 @@ @dataclass class DataElement: - """One experiment in the database.""" + """One experiment in the database. + + Fields fall into three trust tiers (see ``to_api_dict`` for the public + grouping): + + * Self-reported (miner-authored, trivially fakeable): + ``motivation``, ``reasoning``, ``tool_calls``, ``trace``. + * Observed (validator-authored, hard to fake): ``agent_behavior``. + * Verified (Phase C trust anchor): ``metric``, ``success``, + ``objectives``, ``loss_curve``, ``score``, ``manifest_sha256``. + """ index: int = -1 timestamp: float = 0.0 @@ -25,6 +35,9 @@ class DataElement: code: str = "" motivation: str = "" trace: str = "" + reasoning: str = "" + tool_calls: list = field(default_factory=list) + agent_behavior: dict = field(default_factory=dict) metric: Optional[float] = None success: bool = False analysis: str = "" @@ -54,6 +67,19 @@ def to_api_dict(self) -> dict: strings — this happens when a DataElement is built from a row whose JSONB columns weren't decoded by asyncpg (pools without a codec). The handler must never raise on well-formed DB rows. + + Adds a ``provenance`` block so consumers can decide which fields to + trust: + + * ``self_reported`` — written by the miner's agent code (motivation, + reasoning, tool_calls, trace). Fakeable; treat as explanation only. + * ``observed`` — measured by the validator at boundaries the agent + can't forge (proxy access counts, pod wall-clock, exit code). + * ``verified`` — Phase C trust anchor (results, score, + manifest_sha256). + + The legacy flat keys (``motivation``, ``results``, ``analysis``, + ``score``) remain at the top level for backward compatibility. """ objectives = self.objectives if isinstance(objectives, str): @@ -72,6 +98,24 @@ def to_api_dict(self) -> dict: if not isinstance(loss_curve, list): loss_curve = [] + tool_calls = self.tool_calls + if isinstance(tool_calls, str): + try: + tool_calls = json.loads(tool_calls) + except (ValueError, TypeError): + tool_calls = [] + if not isinstance(tool_calls, list): + tool_calls = [] + + agent_behavior = self.agent_behavior + if isinstance(agent_behavior, str): + try: + agent_behavior = json.loads(agent_behavior) + except (ValueError, TypeError): + agent_behavior = {} + if not isinstance(agent_behavior, dict): + agent_behavior = {} + def _safe(v): """Replace inf/nan floats with None for JSON compliance.""" if isinstance(v, float) and (math.isinf(v) or math.isnan(v)): @@ -83,6 +127,13 @@ def _safe(v): safe_obj = {k: _safe(v) for k, v in objectives.items()} safe_lc = [_safe(v) for v in loss_curve] + results = { + "success": self.success, + "metric": metric, + **safe_obj, + "loss_curve": safe_lc, + } + return { "index": self.index, "timestamp": self.timestamp, @@ -94,15 +145,26 @@ def _safe(v): "parent_index": self.parent, "code": self.code, "motivation": self.motivation, - "results": { - "success": self.success, - "metric": metric, - **safe_obj, - "loss_curve": safe_lc, - }, + "results": results, "analysis": self.analysis, "score": score, "round_id": self.round_id, + "provenance": { + "self_reported": { + "motivation": self.motivation, + "reasoning": self.reasoning, + "tool_calls": tool_calls, + "trace": self.trace, + }, + "observed": { + "agent_behavior": agent_behavior, + }, + "verified": { + "results": results, + "score": score, + "manifest_sha256": self.manifest_sha256, + }, + }, } def summary(self) -> str: diff --git a/shared/pg_schema.py b/shared/pg_schema.py index f59f0a7..10e9bde 100644 --- a/shared/pg_schema.py +++ b/shared/pg_schema.py @@ -14,6 +14,9 @@ code TEXT NOT NULL DEFAULT '', motivation TEXT NOT NULL DEFAULT '', trace TEXT NOT NULL DEFAULT '', + reasoning TEXT NOT NULL DEFAULT '', + tool_calls JSONB NOT NULL DEFAULT '[]', + agent_behavior JSONB NOT NULL DEFAULT '{}', metric DOUBLE PRECISION, success BOOLEAN NOT NULL DEFAULT FALSE, analysis TEXT NOT NULL DEFAULT '', @@ -35,6 +38,11 @@ -- ``shared.challenge.generate_challenge`` and can exceed INT32's 2.1B max. -- Widen existing deployments to BIGINT; no-op if already BIGINT. ALTER TABLE experiments ALTER COLUMN round_id TYPE BIGINT; +-- Migration: agent reasoning + tool_calls + validator-observed agent +-- behaviour. Older rows default to empty values via ADD COLUMN DEFAULT. +ALTER TABLE experiments ADD COLUMN IF NOT EXISTS reasoning TEXT NOT NULL DEFAULT ''; +ALTER TABLE experiments ADD COLUMN IF NOT EXISTS tool_calls JSONB NOT NULL DEFAULT '[]'; +ALTER TABLE experiments ADD COLUMN IF NOT EXISTS agent_behavior JSONB NOT NULL DEFAULT '{}'; """ SCHEMA_INDEX_DDL = """ @@ -266,12 +274,29 @@ def row_to_element(row) -> DataElement: ``_decode_jsonb`` so callers always get dict/list regardless of whether the asyncpg connection has a JSONB codec registered. """ + # New columns may be missing on rows produced by older schemas (e.g. a + # query against a yet-to-migrate replica). asyncpg Records raise on + # ``row[name]`` for missing keys, so probe via ``keys()``. + row_keys = set(row.keys()) if hasattr(row, "keys") else set() + reasoning = row["reasoning"] if "reasoning" in row_keys else "" + tool_calls = ( + _decode_jsonb(row["tool_calls"], []) + if "tool_calls" in row_keys else [] + ) + agent_behavior = ( + _decode_jsonb(row["agent_behavior"], {}) + if "agent_behavior" in row_keys else {} + ) + return DataElement( index=row["id"], name=row["name"], code=row["code"], motivation=row["motivation"], trace=row["trace"], + reasoning=reasoning, + tool_calls=tool_calls, + agent_behavior=agent_behavior, metric=_finite_or(row["metric"], None), success=bool(row["success"]), analysis=row["analysis"], @@ -313,12 +338,13 @@ def _jsonb(value) -> str: def element_to_params(element: DataElement, next_id: int) -> tuple: - """Convert a DataElement to a positional tuple for INSERT ($1..$20). + """Convert a DataElement to a positional tuple for INSERT ($1..$23). - JSONB columns (loss_curve, generated_samples, objectives) are explicitly - serialised with ``json.dumps`` so asyncpg sends a valid JSON string - regardless of statement-cache or connection-pooler settings. Float - ``inf``/``nan`` are replaced with ``null`` to keep PostgreSQL happy. + JSONB columns (loss_curve, generated_samples, objectives, tool_calls, + agent_behavior) are explicitly serialised with ``json.dumps`` so + asyncpg sends a valid JSON string regardless of statement-cache or + connection-pooler settings. Float ``inf``/``nan`` are replaced with + ``null`` to keep PostgreSQL happy. """ round_id = element.round_id if element.round_id >= 0 else None return ( @@ -342,6 +368,9 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: element.timestamp, round_id, element.task, + element.reasoning, + _jsonb(element.tool_calls), + _jsonb(element.agent_behavior), ) @@ -350,12 +379,14 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: id, name, code, motivation, trace, metric, success, analysis, parent_index, generation, score, miner_uid, miner_hotkey, loss_curve, manifest_sha256, generated_samples, objectives, - timestamp, round_id, task + timestamp, round_id, task, + reasoning, tool_calls, agent_behavior ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, - $18, $19, $20 + $18, $19, $20, + $21, $22, $23 ) """ diff --git a/shared/protocol.py b/shared/protocol.py index 5142a5b..1f346f9 100644 --- a/shared/protocol.py +++ b/shared/protocol.py @@ -78,17 +78,31 @@ def from_json(cls, s: str) -> Challenge: @dataclass class Proposal: - """Returned by a miner in response to a Challenge.""" + """Returned by a miner in response to a Challenge. + + ``reasoning`` and ``tool_calls`` are *self-reported* — the agent author + populates them. They are useful for explainability but trivially + fakeable; pair with validator-observed signals (proxy access logs, + pod wall-clock) for adversarial use. + """ code: str = "" name: str = "" motivation: str = "" + # Free-form prose the agent emits to explain its design choice. + reasoning: str = "" + # List of dicts the agent self-reports about external calls it made + # (e.g. {"tool": "llm", "prompt": "...", "summary": "..."}). Schema is + # intentionally open so different agents can use different shapes. + tool_calls: list = field(default_factory=list) def to_json(self) -> str: return json.dumps({ "code": self.code, "name": self.name, "motivation": self.motivation, + "reasoning": self.reasoning, + "tool_calls": self.tool_calls, }) @classmethod diff --git a/tests/test_collection.py b/tests/test_collection.py index 6d97503..8bb3daa 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -34,7 +34,7 @@ def _mock_db_client(): async def test_collect_empty_commitments(): """No commitments -> no proposals.""" mock_r2 = MagicMock() - proposals, agent_logs = await run_and_collect_agents( + proposals, agent_meta = await run_and_collect_agents( wallet=MagicMock(), metagraph=MagicMock(), challenge_json='{"round_id": 1}', @@ -48,7 +48,7 @@ async def test_collect_empty_commitments(): db_client=_mock_db_client(), ) assert proposals == {} - assert agent_logs == {} + assert agent_meta == {} @pytest.mark.asyncio @@ -66,7 +66,7 @@ async def test_dedup_removes_identical_code(): 1: ImageCommitment(miner_uid=1, hotkey="hk1"), } - proposals, agent_logs = await run_and_collect_agents( + proposals, agent_meta = await run_and_collect_agents( wallet=MagicMock(), metagraph=MagicMock(), challenge_json='{"round_id": 1}', @@ -94,12 +94,19 @@ def assign_none(all_uids, validator_uids, my_uid, seed): return [] mock_r2.download_json.side_effect = lambda key: { - "code": f"code_{key}", "name": "from_r2", "motivation": "other vali", + "code": f"code_{key}", + "name": "from_r2", + "motivation": "other vali", + "reasoning": "shared via R2", + "tool_calls": [{"tool": "llm", "summary": "asked for ideas"}], + "agent_log": "log line", + "agent_behavior": {"wall_clock_s": 12.5, + "proxy": {"calls": {"llm": 1}}}, } commitments = _mock_commitments() - proposals, agent_logs = await run_and_collect_agents( + proposals, agent_meta = await run_and_collect_agents( wallet=MagicMock(), metagraph=MagicMock(), challenge_json='{"round_id": 1}', @@ -117,6 +124,13 @@ def assign_none(all_uids, validator_uids, my_uid, seed): assert len(proposals) == 2 assert 0 in proposals assert 1 in proposals + # Self-reported metadata is propagated from R2 onto the proposal + meta + assert proposals[0].reasoning == "shared via R2" + assert proposals[0].tool_calls == [ + {"tool": "llm", "summary": "asked for ideas"}, + ] + assert agent_meta[0]["agent_log"] == "log line" + assert agent_meta[0]["agent_behavior"]["wall_clock_s"] == 12.5 @pytest.mark.asyncio @@ -135,7 +149,7 @@ async def test_run_single_agent_logs_error_result(caplog): mock_run.return_value = error_result with caplog.at_level(logging.WARNING, logger="validator.collection"): - proposal, agent_log = await _run_single_agent( + proposal, meta = await _run_single_agent( uid=0, commitment=commitment, challenge_json='{"round_id": 1}', @@ -146,6 +160,69 @@ async def test_run_single_agent_logs_error_result(caplog): ) assert proposal is None - assert agent_log == "" + # Empty metadata bag, all keys present so neuron.py can lookup safely + assert meta == { + "agent_log": "", + "reasoning": "", + "tool_calls": [], + "agent_behavior": {}, + } assert "UID 0 proposal rejected (no code returned)" in caplog.text assert "SyntaxError" in caplog.text + + +@pytest.mark.asyncio +async def test_run_single_agent_captures_reasoning_and_behavior(): + """Successful run propagates reasoning, tool_calls, and pod wall-clock.""" + commitment = ImageCommitment(miner_uid=7, hotkey="hk7") + bundle = {"files": {"agent.py": "..."}, "entry_point": "agent.py"} + mock_r2 = MagicMock() + success_result = { + "code": "def build_model(): ...\ndef build_optimizer(): ...", + "name": "tx", + "motivation": "frontier", + "reasoning": "Picked the smallest preset that fits the bucket.", + "tool_calls": [ + {"tool": "llm", "summary": "asked for ideas"}, + "stringified", + ], + "agent_log": "designing\n", + } + + with patch("validator.collection.launch_agent_pod") as mock_launch, \ + patch("validator.collection.run_agent_on_pod", new_callable=AsyncMock) as mock_run: + mock_env = AsyncMock() + mock_launch.return_value = mock_env + mock_run.return_value = success_result + + proposal, meta = await _run_single_agent( + uid=7, + commitment=commitment, + challenge_json='{"round_id": 1}', + r2=mock_r2, + round_id=1, + allowed_urls="", + bundle=bundle, + ) + + assert proposal is not None + assert proposal.reasoning == "Picked the smallest preset that fits the bucket." + # _normalise_tool_calls coerces non-dict entries to {"value": ...} + assert proposal.tool_calls == [ + {"tool": "llm", "summary": "asked for ideas"}, + {"value": "stringified"}, + ] + # _truncate_text only trims at the byte-cap, so trailing whitespace + # passes through unchanged (the env wrapper strips it before sending). + assert meta["agent_log"] == "designing\n" + # wall_clock_s is measured by the validator (>= 0) + assert "wall_clock_s" in meta["agent_behavior"] + assert meta["agent_behavior"]["wall_clock_s"] >= 0 + assert meta["agent_behavior"]["exit_status"] == "ok" + + # R2 upload payload includes the new fields so other validators get them + upload_args = mock_r2.upload_json.call_args + payload = upload_args[0][1] if upload_args else {} + assert payload.get("reasoning") == proposal.reasoning + assert payload.get("tool_calls") == proposal.tool_calls + assert "agent_behavior" in payload diff --git a/tests/test_database.py b/tests/test_database.py index c298f81..f9b9a4e 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -205,3 +205,55 @@ def test_to_api_dict_malformed_objectives(): assert set(d["results"].keys()) == {"success", "metric", "loss_curve"} +def test_to_api_dict_provenance_grouping(): + """to_api_dict groups fields by trust tier in a `provenance` block.""" + elem = DataElement( + name="x", code="y", + motivation="testing the provenance API", + reasoning="step 1\nstep 2", + tool_calls=[{"tool": "llm", "prompt": "hi"}], + trace="agent stderr", + agent_behavior={ + "wall_clock_s": 4.2, + "proxy": {"calls": {"db": 1, "llm": 2}}, + }, + manifest_sha256="abc123", + success=True, metric=0.42, score=0.8, + ) + d = elem.to_api_dict() + prov = d["provenance"] + + # Self-reported (miner-authored): motivation, reasoning, tool_calls, trace + assert prov["self_reported"]["motivation"] == "testing the provenance API" + assert prov["self_reported"]["reasoning"] == "step 1\nstep 2" + assert prov["self_reported"]["tool_calls"] == [ + {"tool": "llm", "prompt": "hi"}, + ] + assert prov["self_reported"]["trace"] == "agent stderr" + + # Observed (validator-measured): agent_behavior + assert prov["observed"]["agent_behavior"]["wall_clock_s"] == 4.2 + assert prov["observed"]["agent_behavior"]["proxy"]["calls"]["llm"] == 2 + + # Verified (Phase C trust anchor): results, score, manifest_sha256 + assert prov["verified"]["manifest_sha256"] == "abc123" + assert prov["verified"]["score"] == 0.8 + assert prov["verified"]["results"]["metric"] == 0.42 + + # Backwards-compat top-level keys still present + assert d["motivation"] == "testing the provenance API" + assert d["results"]["metric"] == 0.42 + + +def test_to_api_dict_string_tool_calls_and_behavior(): + """Defensive: tool_calls / agent_behavior arriving as JSON strings decode.""" + elem = DataElement( + name="x", code="y", + tool_calls='[{"tool":"llm"}]', + agent_behavior='{"wall_clock_s":1.0}', + ) + d = elem.to_api_dict() + assert d["provenance"]["self_reported"]["tool_calls"] == [{"tool": "llm"}] + assert d["provenance"]["observed"]["agent_behavior"] == {"wall_clock_s": 1.0} + + diff --git a/tests/test_db_proxy.py b/tests/test_db_proxy.py index 77e89ae..c477ced 100644 --- a/tests/test_db_proxy.py +++ b/tests/test_db_proxy.py @@ -9,6 +9,7 @@ app, set_config, set_metagraph, rotate_agent_token, get_agent_token, get_ready_trainers, clear_ready_trainers, _trainer_ready, _check_rate_limit, _rate_window, _rate_lock, + get_agent_behavior, reset_agent_behavior, _bump_agent_behavior, ) @@ -143,3 +144,80 @@ def mock_sign(wallet, body): assert resp.status_code == 503 assert sign_call_count == 1 assert mock_client.post.call_count == 1 + + +def test_agent_behavior_counters_track_per_category(): + """`_bump_agent_behavior` records calls + errors keyed by miner UID.""" + reset_agent_behavior() + _bump_agent_behavior("7", "db", 200) + _bump_agent_behavior("7", "db", 200) + _bump_agent_behavior("7", "llm", 200) + _bump_agent_behavior("7", "llm", 500) # error + _bump_agent_behavior("7", "desearch", 429) # rate-limited + + snap = get_agent_behavior(7) + assert snap["calls"]["db"] == 2 + assert snap["calls"]["llm"] == 2 + assert snap["calls"]["desearch"] == 1 + # 5xx + 429 both count as errors + assert snap["errors"]["llm"] == 1 + assert snap["errors"]["desearch"] == 1 + assert snap["errors"]["db"] == 0 + assert snap["first_call_ts"] is not None + assert snap["last_call_ts"] >= snap["first_call_ts"] + + +def test_agent_behavior_reset_on_token_rotate(): + """Rotating the agent token clears prior round counters.""" + reset_agent_behavior() + _bump_agent_behavior("3", "db", 200) + assert get_agent_behavior(3)["calls"]["db"] == 1 + rotate_agent_token() + # New round = fresh slate + assert get_agent_behavior(3) == {} + + +def test_agent_behavior_unknown_uid_returns_empty(): + reset_agent_behavior() + assert get_agent_behavior(99) == {} + # Bad UIDs are silently ignored, never raise + _bump_agent_behavior("not-an-int", "db", 200) + _bump_agent_behavior("-1", "db", 200) + assert get_agent_behavior(-1) == {} + + +def test_agent_behavior_unknown_category_falls_back_to_other(): + reset_agent_behavior() + _bump_agent_behavior("5", "unknown_route", 200) + snap = get_agent_behavior(5) + assert snap["calls"]["other"] == 1 + assert snap["calls"]["db"] == 0 + + +def test_agent_proxy_increments_counters_on_request(): + """A successful agent-token request increments the counter for its UID.""" + _setup_proxy() + reset_agent_behavior() + + from validator import db_proxy + + fake_resp = MagicMock() + fake_resp.status_code = 200 + fake_resp.headers = {"content-type": "application/json"} + fake_resp.content = b'{"ok": true}' + + mock_client = AsyncMock() + mock_client.is_closed = False + mock_client.get = AsyncMock(return_value=fake_resp) + + with patch.object(db_proxy, "_client", mock_client), \ + patch("validator.db_proxy.sign_request", return_value={}): + client = TestClient(app) + r = client.get( + "/experiments/recent", + headers={**_auth_headers(), "X-Miner-UID": "42"}, + ) + assert r.status_code == 200 + snap = get_agent_behavior(42) + assert snap["calls"]["db"] == 1 + assert snap["errors"]["db"] == 0 diff --git a/tests/test_pg_store.py b/tests/test_pg_store.py index 8a5cc4b..618d522 100644 --- a/tests/test_pg_store.py +++ b/tests/test_pg_store.py @@ -184,6 +184,41 @@ async def test_jsonb_columns_decoded(store): assert d["results"]["flops_equivalent_size"] == 1_000_000 +@skip_no_pg +@pytest.mark.asyncio +async def test_reasoning_tool_calls_agent_behavior_roundtrip(store): + """New self-reported + observed columns survive INSERT/SELECT.""" + await store.add(DataElement( + name="agent_reasoning_test", + code="def build_model(): ...", + success=True, + metric=0.5, + reasoning="step 1\nstep 2\nstep 3", + tool_calls=[ + {"tool": "llm", "prompt": "ideas?", "summary": "Mamba"}, + {"tool": "desearch", "query": "linear attention"}, + ], + agent_behavior={ + "wall_clock_s": 12.34, + "exit_status": "ok", + "proxy": {"calls": {"db": 3, "llm": 1, "desearch": 2}}, + }, + )) + got = await store.get(0) + assert got.reasoning == "step 1\nstep 2\nstep 3" + assert isinstance(got.tool_calls, list) + assert got.tool_calls[0]["tool"] == "llm" + assert got.tool_calls[1]["tool"] == "desearch" + assert isinstance(got.agent_behavior, dict) + assert got.agent_behavior["wall_clock_s"] == 12.34 + assert got.agent_behavior["proxy"]["calls"]["llm"] == 1 + + # Provenance grouping is exposed via to_api_dict + api = got.to_api_dict() + assert api["provenance"]["self_reported"]["reasoning"] == got.reasoning + assert api["provenance"]["observed"]["agent_behavior"]["wall_clock_s"] == 12.34 + + @skip_no_pg @pytest.mark.asyncio async def test_add_batch(store): diff --git a/validator/collection.py b/validator/collection.py index 1a7e441..535bb53 100644 --- a/validator/collection.py +++ b/validator/collection.py @@ -5,7 +5,16 @@ Work-split across validators — each runs a subset. Proposals uploaded to R2 so all validators see all submissions. -Agent stderr (reasoning trace) is captured and stored alongside proposals. +Each agent run produces three trust tiers of metadata, all collected here: + +* Self-reported (stderr ``trace``, ``reasoning``, ``tool_calls``): + written by the miner's agent code; trivially fakeable. +* Observed (``agent_behavior``): pod wall-clock + per-category proxy + call counts measured by the validator outside the pod. +* Verified: produced later in Phase C (results, score). + +The combined ``agent_meta`` per UID is uploaded to R2 alongside the +Proposal so other validators see the same picture. """ from __future__ import annotations @@ -14,6 +23,7 @@ import hashlib import json import logging +import time from typing import Optional from config import Config @@ -33,6 +43,71 @@ _PROPOSAL_SYNC_MAX_WAIT = 120 # max seconds to wait for other validators _PROPOSAL_SYNC_POLL_INTERVAL = 15 # seconds between polls +# Storage hygiene: agents that emit megabytes of stderr or reasoning would +# bloat the experiments table without producing extra signal. These caps +# are intentionally generous (a typical reasoning trace is < 64 KB). +_TRACE_MAX_BYTES = 256 * 1024 +_REASONING_MAX_BYTES = 256 * 1024 +_TOOL_CALLS_MAX = 256 + + +def _truncate_text(value: str, limit: int) -> str: + """Cap a string at ``limit`` UTF-8 bytes, preserving valid UTF-8.""" + if not value: + return "" + raw = value.encode("utf-8", errors="replace") + if len(raw) <= limit: + return value + truncated = raw[:limit].decode("utf-8", errors="ignore") + suffix = f"\n... [truncated {len(raw) - limit} bytes]" + return truncated + suffix + + +def _normalise_tool_calls(value) -> list: + """Coerce a self-reported tool_calls payload to a bounded list of dicts.""" + if not isinstance(value, list): + return [] + out: list = [] + for entry in value[:_TOOL_CALLS_MAX]: + if isinstance(entry, dict): + out.append(entry) + else: + # Stringify scalars so consumers always see a uniform shape. + out.append({"value": str(entry)}) + return out + + +def _empty_meta() -> dict: + """Per-UID metadata bag used when no real run produced anything.""" + return { + "agent_log": "", + "reasoning": "", + "tool_calls": [], + "agent_behavior": {}, + } + + +def _snapshot_agent_behavior( + uid: int, started_at: float, exit_status: str, +) -> dict: + """Build the validator-observed ``agent_behavior`` dict. + + Pulls per-agent proxy counters from ``validator.db_proxy`` (None if + the proxy module isn't initialised, e.g. in unit tests). Adds the + pod wall-clock and exit status the validator measured itself. + """ + proxy_metrics: dict = {} + try: + from validator import db_proxy as _db_proxy + proxy_metrics = _db_proxy.get_agent_behavior(uid) + except Exception: + proxy_metrics = {} + return { + "wall_clock_s": round(max(0.0, time.monotonic() - started_at), 3), + "exit_status": exit_status, + "proxy": proxy_metrics, + } + def _attach_scratchpad_urls(challenge_json: str, r2, hotkey: str) -> str: """Inject per-miner scratchpad presigned URLs into a challenge JSON string.""" @@ -130,8 +205,14 @@ async def _run_single_agent( round_id: int, allowed_urls: str, bundle: dict, -) -> tuple[Optional[Proposal], str]: - """Run a single miner agent and return (proposal, agent_log) or (None, "").""" +) -> tuple[Optional[Proposal], dict]: + """Run a single miner agent. + + Returns ``(proposal, meta)``. ``meta`` is a dict with keys + ``agent_log``, ``reasoning``, ``tool_calls``, ``agent_behavior``; + populated even when the proposal is None so the caller can record + why the run failed. + """ miner_challenge_json = _attach_scratchpad_urls( challenge_json, r2, commitment.hotkey, ) @@ -160,6 +241,7 @@ async def _run_single_agent( allowed_urls=allowed_urls, ) + started_at = time.monotonic() try: result = await run_agent_on_pod( agent_env, miner_challenge_json, @@ -172,18 +254,34 @@ async def _run_single_agent( code=result.get("code", ""), name=result.get("name", ""), motivation=result.get("motivation", ""), + reasoning=_truncate_text( + result.get("reasoning", "") or "", _REASONING_MAX_BYTES, + ), + tool_calls=_normalise_tool_calls(result.get("tool_calls", [])), + ) + agent_log = _truncate_text( + result.get("agent_log", "") or "", _TRACE_MAX_BYTES, ) - agent_log = result.get("agent_log", "") + agent_behavior = _snapshot_agent_behavior(uid, started_at, "ok") + meta = { + "agent_log": agent_log, + "reasoning": proposal.reasoning, + "tool_calls": proposal.tool_calls, + "agent_behavior": agent_behavior, + } r2.upload_json( f"round_{round_id}/proposals/{uid}.json", { "code": proposal.code, "name": proposal.name, "motivation": proposal.motivation, + "reasoning": proposal.reasoning, + "tool_calls": proposal.tool_calls, "agent_log": agent_log, + "agent_behavior": agent_behavior, }, ) - return proposal, agent_log + return proposal, meta else: error_msg = "" result_keys = "" @@ -204,7 +302,7 @@ async def _run_single_agent( await agent_env.cleanup() except Exception: pass - return None, "" + return None, _empty_meta() async def _run_agents_concurrently( @@ -217,33 +315,33 @@ async def _run_agents_concurrently( round_id: int, allowed_urls: str, concurrency: int, -) -> dict[int, tuple[Optional[Proposal], str]]: +) -> dict[int, tuple[Optional[Proposal], dict]]: """Run multiple agents concurrently with a bounded semaphore. - Returns {uid: (proposal_or_None, agent_log)}. Per-agent exceptions - are logged and produce (None, "") so one bad agent never aborts the - batch. + Returns {uid: (proposal_or_None, meta)}. Per-agent exceptions are + logged and produce ``(None, _empty_meta())`` so one bad agent never + aborts the batch. """ sem = asyncio.Semaphore(max(1, int(concurrency))) - async def _bounded(uid: int) -> tuple[int, Optional[Proposal], str]: + async def _bounded(uid: int) -> tuple[int, Optional[Proposal], dict]: async with sem: try: - proposal, agent_log = await _run_single_agent( + proposal, meta = await _run_single_agent( uid, commitments[uid], challenge_json, r2, round_id, allowed_urls, bundles[uid], ) - return uid, proposal, agent_log + return uid, proposal, meta except Exception as e: logger.error("UID %d agent failed: %s", uid, e) - return uid, None, "" + return uid, None, _empty_meta() logger.info( "Running %d agents concurrently (cap=%d)", len(uids), max(1, int(concurrency)), ) raw = await asyncio.gather(*[_bounded(uid) for uid in uids]) - return {uid: (proposal, log) for uid, proposal, log in raw} + return {uid: (proposal, meta) for uid, proposal, meta in raw} async def run_and_collect_agents( @@ -258,7 +356,7 @@ async def run_and_collect_agents( commitments: dict[int, ImageCommitment], get_my_assignments_fn, db_client: Optional[DatabaseClient] = None, -) -> tuple[dict[int, Proposal], dict[int, str]]: +) -> tuple[dict[int, Proposal], dict[int, dict]]: """Phase A: run assigned miner agents, upload proposals to R2, read all. 1. Work-split: get_my_assignments() for which agents this validator runs @@ -267,11 +365,13 @@ async def run_and_collect_agents( b. Verify code_hash against on-chain commitment c. launch_agent_pod() with official image + code injection d. run_agent_on_pod() with challenge JSON + URL allowlist - e. Upload proposal + agent_log to R2: round_{id}/proposals/{uid}.json + e. Upload proposal + per-agent metadata to R2: + round_{id}/proposals/{uid}.json 3. Wait for other validators to upload 4. Read all proposals from R2 5. Dedup by code hash - 6. Return ({uid: Proposal}, {uid: agent_log}) + 6. Return ``({uid: Proposal}, {uid: meta})`` where ``meta`` is + ``{"agent_log", "reasoning", "tool_calls", "agent_behavior"}``. """ all_uids = list(commitments.keys()) my_agent_uids = get_my_assignments_fn( @@ -303,7 +403,7 @@ async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: # so no local resource contention; the cap throttles orchestration / # R2 fan-out and prevents one slow agent from serialising the round. proposals: dict[int, Proposal] = {} - agent_logs: dict[int, str] = {} + agent_meta: dict[int, dict] = {} runnable_uids = [uid for uid in my_agent_uids if uid in bundles] if runnable_uids: @@ -317,10 +417,10 @@ async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: allowed_urls=allowed_urls, concurrency=Config.AGENT_CONCURRENCY, ) - for uid, (proposal, agent_log) in results.items(): + for uid, (proposal, meta) in results.items(): if proposal: proposals[uid] = proposal - agent_logs[uid] = agent_log + agent_meta[uid] = meta logger.info( "Ran %d agents, got %d proposals", len(my_agent_uids), len(proposals), @@ -348,8 +448,25 @@ async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: code=data["code"], name=data.get("name", ""), motivation=data.get("motivation", ""), + reasoning=_truncate_text( + data.get("reasoning", "") or "", + _REASONING_MAX_BYTES, + ), + tool_calls=_normalise_tool_calls( + data.get("tool_calls", []), + ), ) - agent_logs[uid] = data.get("agent_log", "") + agent_meta[uid] = { + "agent_log": _truncate_text( + data.get("agent_log", "") or "", + _TRACE_MAX_BYTES, + ), + "reasoning": proposals[uid].reasoning, + "tool_calls": proposals[uid].tool_calls, + "agent_behavior": data.get("agent_behavior", {}) + if isinstance(data.get("agent_behavior"), dict) + else {}, + } else: still_missing.add(uid) except Exception: @@ -375,8 +492,25 @@ async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: code=data["code"], name=data.get("name", ""), motivation=data.get("motivation", ""), + reasoning=_truncate_text( + data.get("reasoning", "") or "", + _REASONING_MAX_BYTES, + ), + tool_calls=_normalise_tool_calls( + data.get("tool_calls", []), + ), ) - agent_logs[uid] = data.get("agent_log", "") + agent_meta[uid] = { + "agent_log": _truncate_text( + data.get("agent_log", "") or "", + _TRACE_MAX_BYTES, + ), + "reasoning": proposals[uid].reasoning, + "tool_calls": proposals[uid].tool_calls, + "agent_behavior": data.get("agent_behavior", {}) + if isinstance(data.get("agent_behavior"), dict) + else {}, + } except Exception: pass @@ -414,21 +548,21 @@ async def _fetch(uid: int) -> tuple[int, Optional[dict]]: allowed_urls=allowed_urls, concurrency=Config.AGENT_CONCURRENCY, ) - for uid, (proposal, agent_log) in results.items(): + for uid, (proposal, meta) in results.items(): if proposal: proposals[uid] = proposal - agent_logs[uid] = agent_log + agent_meta[uid] = meta # Dedup by code hash seen_hashes: dict[str, int] = {} # hash -> first uid that claimed it deduped: dict[int, Proposal] = {} - deduped_logs: dict[int, str] = {} + deduped_meta: dict[int, dict] = {} for uid, p in sorted(proposals.items()): h = hashlib.sha256(p.code.strip().encode()).hexdigest() if h not in seen_hashes: seen_hashes[h] = uid deduped[uid] = p - deduped_logs[uid] = agent_logs.get(uid, "") + deduped_meta[uid] = agent_meta.get(uid, _empty_meta()) else: logger.warning( "UID %d proposal rejected (duplicate): code hash %s " @@ -441,4 +575,4 @@ async def _fetch(uid: int) -> tuple[int, Optional[dict]]: "Phase A complete: %d total proposals, %d after dedup (%d duplicates removed)", len(proposals), len(deduped), dup_count, ) - return deduped, deduped_logs + return deduped, deduped_meta diff --git a/validator/db_proxy.py b/validator/db_proxy.py index e3fac3c..359f3f5 100644 --- a/validator/db_proxy.py +++ b/validator/db_proxy.py @@ -42,10 +42,12 @@ def rotate_agent_token() -> str: """Generate a new agent token for the current round. + Resets the per-agent behaviour metrics so each round starts from zero. Returns the new token (caller injects it into the challenge JSON). """ global _agent_token _agent_token = secrets.token_urlsafe(32) + reset_agent_behavior() logger.info("Agent token rotated (token=%s...)", _agent_token[:8]) return _agent_token @@ -55,6 +57,73 @@ def get_agent_token() -> str: return _agent_token +# ── Per-agent behaviour counters ─────────────────────────────────── +# Bumped inside ``auth_middleware`` when a request authenticates with the +# round's agent token. Keyed by miner UID (from the X-Miner-UID header). +# These are *validator-observed* — the miner's agent code can't forge them +# because the increments happen on the validator side of the network. The +# round loop snapshots these via ``get_agent_behavior(uid)`` after running +# each agent and persists them as ``DataElement.agent_behavior``. +_agent_behavior: dict[int, dict] = {} +_agent_behavior_lock = threading.Lock() + + +def _new_agent_behavior_entry() -> dict: + return { + "calls": {"db": 0, "desearch": 0, "llm": 0, "other": 0}, + "errors": {"db": 0, "desearch": 0, "llm": 0, "other": 0}, + "first_call_ts": None, + "last_call_ts": None, + } + + +def _bump_agent_behavior(uid_str: str, category: str, status_code: int) -> None: + """Record one authenticated agent request for a miner UID.""" + try: + uid = int(uid_str) + except (TypeError, ValueError): + return + if uid < 0: + return + now = time.time() + cat = category if category in ("db", "desearch", "llm") else "other" + with _agent_behavior_lock: + entry = _agent_behavior.get(uid) + if entry is None: + entry = _new_agent_behavior_entry() + _agent_behavior[uid] = entry + entry["calls"][cat] += 1 + if status_code >= 400: + entry["errors"][cat] += 1 + if entry["first_call_ts"] is None: + entry["first_call_ts"] = now + entry["last_call_ts"] = now + + +def get_agent_behavior(uid: int) -> dict: + """Snapshot the proxy-observed behaviour for one miner UID. + + Returns a fresh dict (callers may mutate freely). Empty dict if the + agent never authenticated with the proxy this round. + """ + with _agent_behavior_lock: + entry = _agent_behavior.get(uid) + if entry is None: + return {} + return { + "calls": dict(entry["calls"]), + "errors": dict(entry["errors"]), + "first_call_ts": entry["first_call_ts"], + "last_call_ts": entry["last_call_ts"], + } + + +def reset_agent_behavior() -> None: + """Clear all per-agent counters. Called on round start via rotate.""" + with _agent_behavior_lock: + _agent_behavior.clear() + + def get_ready_trainers(round_id: int) -> dict: """Called by coordinator to poll for TrainerReady responses.""" return dict(_trainer_ready.get(round_id, {})) @@ -196,10 +265,14 @@ async def auth_middleware(request: Request, call_next): if _verify_agent_token(request): rate_key = request.headers.get("X-Miner-UID", "agent-shared") if not _check_rate_limit(f"agent:{rate_key}", category): + # Rate-limit rejection still counts as a request the agent made + # for behaviour tracking — bump before returning. + _bump_agent_behavior(rate_key, category, 429) return JSONResponse(status_code=429, content={ "error": f"Rate limit exceeded for {category}", }) response = await call_next(request) + _bump_agent_behavior(rate_key, category, response.status_code) logger.info( "Agent proxy: miner=%s %s %s -> %d", rate_key, request.method, path, response.status_code, diff --git a/validator/neuron.py b/validator/neuron.py index a598d45..98b70dd 100644 --- a/validator/neuron.py +++ b/validator/neuron.py @@ -510,7 +510,7 @@ async def _no_endpoints(): get_my_assignments_fn=get_my_assignments, ) - dynamic_endpoints, (submissions, agent_logs) = await asyncio.gather( + dynamic_endpoints, (submissions, agent_meta) = await asyncio.gather( prepare_coro, collect_coro, ) @@ -818,6 +818,7 @@ async def _no_endpoints(): proposal = filtered.get(uid, Proposal()) miner_hk = self.metagraph.hotkeys[uid] if uid < len(self.metagraph.hotkeys) else "" + meta = agent_meta.get(uid, {}) element = DataElement( name=f"round_{challenge.round_id}_miner_{uid}", code=proposal.code, @@ -827,7 +828,10 @@ async def _no_endpoints(): miner_uid=uid, miner_hotkey=miner_hk, motivation=proposal.motivation, - trace=agent_logs.get(uid, ""), + trace=meta.get("agent_log", ""), + reasoning=meta.get("reasoning", "") or proposal.reasoning, + tool_calls=meta.get("tool_calls", []) or proposal.tool_calls, + agent_behavior=meta.get("agent_behavior", {}), task=task_name, round_id=challenge.round_id, ) From 70f63c25460103aee1d9dffaa5b46abb36eda8f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Apr 2026 23:56:49 +0000 Subject: [PATCH 2/3] Add 001 migration for reasoning/tool_calls/agent_behavior columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI's schema-migrations workflow blocks any change to shared/pg_schema.py without a companion database/migrations/NNN_*.sql file. Add 001 with the three ADD COLUMN IF NOT EXISTS statements (reasoning TEXT, tool_calls JSONB, agent_behavior JSONB) so existing testnet/mainnet deployments pick up the new columns at startup. Fresh databases get them from CREATE TABLE in SCHEMA_TABLE_DDL — drop the duplicate inline ALTERs there since the migration runner is now the single source of truth for forward migrations. --- .../001_agent_reasoning_and_behavior.sql | 21 +++++++++++++++++++ shared/pg_schema.py | 5 ----- 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 database/migrations/001_agent_reasoning_and_behavior.sql diff --git a/database/migrations/001_agent_reasoning_and_behavior.sql b/database/migrations/001_agent_reasoning_and_behavior.sql new file mode 100644 index 0000000..a939801 --- /dev/null +++ b/database/migrations/001_agent_reasoning_and_behavior.sql @@ -0,0 +1,21 @@ +-- Adds the columns that back the new "self-reported" + "validator-observed" +-- provenance tiers on experiments rows: +-- +-- reasoning — agent-authored prose explaining the design (capped +-- at 256 KB before INSERT in validator/collection.py) +-- tool_calls — agent-authored list of self-reported tool invocations +-- ({"tool": "...", ...}) +-- agent_behavior — validator-observed: pod wall-clock, exit status, and +-- per-category proxy call/error counters keyed by miner +-- UID +-- +-- All three default to empty values so legacy rows remain valid. + +ALTER TABLE CURRENT_SCHEMA.experiments + ADD COLUMN IF NOT EXISTS reasoning TEXT NOT NULL DEFAULT ''; + +ALTER TABLE CURRENT_SCHEMA.experiments + ADD COLUMN IF NOT EXISTS tool_calls JSONB NOT NULL DEFAULT '[]'; + +ALTER TABLE CURRENT_SCHEMA.experiments + ADD COLUMN IF NOT EXISTS agent_behavior JSONB NOT NULL DEFAULT '{}'; diff --git a/shared/pg_schema.py b/shared/pg_schema.py index 10e9bde..19a903e 100644 --- a/shared/pg_schema.py +++ b/shared/pg_schema.py @@ -38,11 +38,6 @@ -- ``shared.challenge.generate_challenge`` and can exceed INT32's 2.1B max. -- Widen existing deployments to BIGINT; no-op if already BIGINT. ALTER TABLE experiments ALTER COLUMN round_id TYPE BIGINT; --- Migration: agent reasoning + tool_calls + validator-observed agent --- behaviour. Older rows default to empty values via ADD COLUMN DEFAULT. -ALTER TABLE experiments ADD COLUMN IF NOT EXISTS reasoning TEXT NOT NULL DEFAULT ''; -ALTER TABLE experiments ADD COLUMN IF NOT EXISTS tool_calls JSONB NOT NULL DEFAULT '[]'; -ALTER TABLE experiments ADD COLUMN IF NOT EXISTS agent_behavior JSONB NOT NULL DEFAULT '{}'; """ SCHEMA_INDEX_DDL = """ From ff8300fac6a29b8479635b3843376d76eee60aaa Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 27 Apr 2026 23:58:44 +0000 Subject: [PATCH 3/3] Guard 001 migration on experiments table existing The schema-migrations CI workflow runs `apply_migrations` against an empty schema (only `CREATE SCHEMA`, no `init_schema()`), so the bare `ALTER TABLE experiments` failed with "relation does not exist". Wrap the ALTERs in a DO block that no-ops when the table is absent. On a fresh deploy the base `CREATE TABLE` in `shared/pg_schema.py` already declares the new columns; on existing testnet/mainnet deployments the ALTERs apply normally because `init_schema()` runs before migrations. --- .../001_agent_reasoning_and_behavior.sql | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/database/migrations/001_agent_reasoning_and_behavior.sql b/database/migrations/001_agent_reasoning_and_behavior.sql index a939801..474f9bd 100644 --- a/database/migrations/001_agent_reasoning_and_behavior.sql +++ b/database/migrations/001_agent_reasoning_and_behavior.sql @@ -10,12 +10,29 @@ -- UID -- -- All three default to empty values so legacy rows remain valid. +-- +-- The whole block is guarded on `experiments` existing because the +-- migration runner is exercised against an empty schema in CI before +-- ``init_schema()`` runs (see workflows/schema-migrations.yml :: +-- "Migrations apply to empty DB"). On a fresh deploy the base +-- ``CREATE TABLE`` in ``shared/pg_schema.py`` already declares the new +-- columns, so the conditional makes this migration a no-op there. On +-- existing testnet/mainnet deployments ``init_schema()`` runs first and +-- the table exists, so the ALTERs apply normally. -ALTER TABLE CURRENT_SCHEMA.experiments - ADD COLUMN IF NOT EXISTS reasoning TEXT NOT NULL DEFAULT ''; - -ALTER TABLE CURRENT_SCHEMA.experiments - ADD COLUMN IF NOT EXISTS tool_calls JSONB NOT NULL DEFAULT '[]'; - -ALTER TABLE CURRENT_SCHEMA.experiments - ADD COLUMN IF NOT EXISTS agent_behavior JSONB NOT NULL DEFAULT '{}'; +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'CURRENT_SCHEMA' + AND table_name = 'experiments' + ) THEN + ALTER TABLE CURRENT_SCHEMA.experiments + ADD COLUMN IF NOT EXISTS reasoning TEXT NOT NULL DEFAULT ''; + ALTER TABLE CURRENT_SCHEMA.experiments + ADD COLUMN IF NOT EXISTS tool_calls JSONB NOT NULL DEFAULT '[]'; + ALTER TABLE CURRENT_SCHEMA.experiments + ADD COLUMN IF NOT EXISTS agent_behavior JSONB NOT NULL DEFAULT '{}'; + END IF; +END $$;