Add agent reasoning, tool calls, and validator-observed behavior tracking

database/migrations/001_agent_reasoning_and_behavior.sql

@@ -0,0 +1,38 @@
+-- Adds the columns that back the new "self-reported" + "validator-observed"
+-- provenance tiers on experiments rows:
+--
+--   reasoning       — agent-authored prose explaining the design (capped
+--                     at 256 KB before INSERT in validator/collection.py)
+--   tool_calls      — agent-authored list of self-reported tool invocations
+--                     ({"tool": "...", ...})
+--   agent_behavior  — validator-observed: pod wall-clock, exit status, and
+--                     per-category proxy call/error counters keyed by miner
+--                     UID
+--
+-- All three default to empty values so legacy rows remain valid.
+--
+-- The whole block is guarded on `experiments` existing because the
+-- migration runner is exercised against an empty schema in CI before
+-- ``init_schema()`` runs (see workflows/schema-migrations.yml ::
+-- "Migrations apply to empty DB"). On a fresh deploy the base
+-- ``CREATE TABLE`` in ``shared/pg_schema.py`` already declares the new
+-- columns, so the conditional makes this migration a no-op there. On
+-- existing testnet/mainnet deployments ``init_schema()`` runs first and
+-- the table exists, so the ALTERs apply normally.
+
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM information_schema.tables
+        WHERE table_schema = 'CURRENT_SCHEMA'
+          AND table_name = 'experiments'
+    ) THEN
+        ALTER TABLE CURRENT_SCHEMA.experiments
+            ADD COLUMN IF NOT EXISTS reasoning TEXT NOT NULL DEFAULT '';
+        ALTER TABLE CURRENT_SCHEMA.experiments
+            ADD COLUMN IF NOT EXISTS tool_calls JSONB NOT NULL DEFAULT '[]';
+        ALTER TABLE CURRENT_SCHEMA.experiments
+            ADD COLUMN IF NOT EXISTS agent_behavior JSONB NOT NULL DEFAULT '{}';
+    END IF;
+END $$;

miner_template/agent.py

@@ -17,11 +17,17 @@
 
 Your agent module MUST define:
     design_architecture(challenge: dict, client: GatedClient) -> dict
-        Returns {"code": str, "name": str, "motivation": str}
+        Returns {"code": str, "name": str, "motivation": str,
+                 "reasoning": str (optional), "tool_calls": list (optional)}
 
 The ``code`` field should be a Python module that defines:
     build_model(context_len, prediction_len, num_variates, quantiles) -> nn.Module
     build_optimizer(model) -> torch.optim.Optimizer
+
+``reasoning`` and ``tool_calls`` are exposed verbatim on the public dashboard
+and saved with the experiment. They're self-reported, so don't expect them to
+count for trust — pair them with the validator-observed proxy access logs if
+you want a story that holds up under scrutiny.
 """
 
 import json
@@ -150,15 +156,25 @@ def design_architecture(challenge: dict, client) -> dict:
             Use client.get_json(url), client.post_json(url, payload), etc.
 
     Returns:
-        dict with keys: code, name, motivation
+        dict with keys: code, name, motivation, reasoning, tool_calls
     """
+    # Self-reported reasoning trace + tool call log. The validator copies
+    # these straight onto the experiment row, so anything you append here
+    # shows up on the public dashboard. Append entries as you go.
+    reasoning_steps: list[str] = []
+    tool_calls: list[dict] = []
+
     # Load persistent state from previous rounds
     scratch_dir = load_scratchpad(challenge)
 
     frontier = challenge.get("feasible_frontier", [])
     min_flops = challenge.get("min_flops_equivalent", 0)
     max_flops = challenge.get("max_flops_equivalent", 0)
     log(f"Designing for FLOPs range [{min_flops}, {max_flops}], frontier size: {len(frontier)}")
+    reasoning_steps.append(
+        f"Round target: FLOPs [{min_flops}, {max_flops}], "
+        f"frontier has {len(frontier)} entries."
+    )
 
     # Example: search arxiv for relevant papers
     # papers = _search_arxiv(client, challenge, "time series forecasting transformer")
@@ -176,18 +192,33 @@ def design_architecture(challenge: dict, client) -> dict:
 
     hp = _pick_hyperparams(min_flops, max_flops)
     log(f"Selected hyperparams: {hp}")
+    reasoning_steps.append(
+        f"Picked preset {hp} targeting ~60% of FLOPs cap to stay in range."
+    )
+    tool_calls.append({
+        "tool": "internal",
+        "name": "_pick_hyperparams",
+        "input": {"min_flops": min_flops, "max_flops": max_flops},
+        "output": hp,
+    })
 
     if frontier:
         best = min(frontier, key=lambda x: x.get("metric", float("inf")))
         motivation = (
             f"Improving on frontier best (metric={best.get('metric', '?')}). "
             f"Target FLOPs range: [{min_flops}, {max_flops}]"
         )
+        reasoning_steps.append(
+            f"Frontier present; aiming to beat best metric={best.get('metric', '?')}."
+        )
     else:
         motivation = (
             f"No frontier in size range [{min_flops}, {max_flops}]. "
             f"Exploring with baseline transformer."
         )
+        reasoning_steps.append(
+            "No frontier in this bucket; submitting baseline transformer to bootstrap."
+        )
 
     d_model = hp["d_model"]
     nhead = hp["nhead"]
@@ -234,4 +265,6 @@ def build_optimizer(model):
         "code": code,
         "name": f"transformer_d{d_model}_l{num_layers}_ff{dim_feedforward}",
         "motivation": motivation,
+        "reasoning": "\n".join(reasoning_steps),
+        "tool_calls": tool_calls,
     }

runner/agent/harness.py

@@ -225,10 +225,22 @@ def main():
         print(json.dumps({"error": "design_architecture() must return dict with 'code' key"}))
         sys.exit(1)
 
+    # Pass through optional self-reported reasoning trace and tool-call log
+    # if the agent populated them. Both are open-schema; the harness does
+    # not validate or normalise contents — that's the validator's job.
+    reasoning = result.get("reasoning", "")
+    tool_calls = result.get("tool_calls", [])
+    if not isinstance(reasoning, str):
+        reasoning = str(reasoning)
+    if not isinstance(tool_calls, list):
+        tool_calls = []
+
     proposal = {
         "code": result.get("code", ""),
         "name": result.get("name", ""),
         "motivation": result.get("motivation", ""),
+        "reasoning": reasoning,
+        "tool_calls": tool_calls,
     }
     print(json.dumps(proposal))
 

shared/database.py

@@ -17,14 +17,27 @@
 
 @dataclass
 class DataElement:
-    """One experiment in the database."""
+    """One experiment in the database.
+
+    Fields fall into three trust tiers (see ``to_api_dict`` for the public
+    grouping):
+
+    * Self-reported (miner-authored, trivially fakeable):
+      ``motivation``, ``reasoning``, ``tool_calls``, ``trace``.
+    * Observed (validator-authored, hard to fake): ``agent_behavior``.
+    * Verified (Phase C trust anchor): ``metric``, ``success``,
+      ``objectives``, ``loss_curve``, ``score``, ``manifest_sha256``.
+    """
 
     index: int = -1
     timestamp: float = 0.0
     name: str = ""
     code: str = ""
     motivation: str = ""
     trace: str = ""
+    reasoning: str = ""
+    tool_calls: list = field(default_factory=list)
+    agent_behavior: dict = field(default_factory=dict)
     metric: Optional[float] = None
     success: bool = False
     analysis: str = ""
@@ -54,6 +67,19 @@ def to_api_dict(self) -> dict:
         strings — this happens when a DataElement is built from a row whose
         JSONB columns weren't decoded by asyncpg (pools without a codec). The
         handler must never raise on well-formed DB rows.
+
+        Adds a ``provenance`` block so consumers can decide which fields to
+        trust:
+
+        * ``self_reported`` — written by the miner's agent code (motivation,
+          reasoning, tool_calls, trace). Fakeable; treat as explanation only.
+        * ``observed`` — measured by the validator at boundaries the agent
+          can't forge (proxy access counts, pod wall-clock, exit code).
+        * ``verified`` — Phase C trust anchor (results, score,
+          manifest_sha256).
+
+        The legacy flat keys (``motivation``, ``results``, ``analysis``,
+        ``score``) remain at the top level for backward compatibility.
         """
         objectives = self.objectives
         if isinstance(objectives, str):
@@ -72,6 +98,24 @@ def to_api_dict(self) -> dict:
         if not isinstance(loss_curve, list):
             loss_curve = []
 
+        tool_calls = self.tool_calls
+        if isinstance(tool_calls, str):
+            try:
+                tool_calls = json.loads(tool_calls)
+            except (ValueError, TypeError):
+                tool_calls = []
+        if not isinstance(tool_calls, list):
+            tool_calls = []
+
+        agent_behavior = self.agent_behavior
+        if isinstance(agent_behavior, str):
+            try:
+                agent_behavior = json.loads(agent_behavior)
+            except (ValueError, TypeError):
+                agent_behavior = {}
+        if not isinstance(agent_behavior, dict):
+            agent_behavior = {}
+
         def _safe(v):
             """Replace inf/nan floats with None for JSON compliance."""
             if isinstance(v, float) and (math.isinf(v) or math.isnan(v)):
@@ -83,6 +127,13 @@ def _safe(v):
         safe_obj = {k: _safe(v) for k, v in objectives.items()}
         safe_lc = [_safe(v) for v in loss_curve]
 
+        results = {
+            "success": self.success,
+            "metric": metric,
+            **safe_obj,
+            "loss_curve": safe_lc,
+        }
+
         return {
             "index": self.index,
             "timestamp": self.timestamp,
@@ -94,15 +145,26 @@ def _safe(v):
             "parent_index": self.parent,
             "code": self.code,
             "motivation": self.motivation,
-            "results": {
-                "success": self.success,
-                "metric": metric,
-                **safe_obj,
-                "loss_curve": safe_lc,
-            },
+            "results": results,
             "analysis": self.analysis,
             "score": score,
             "round_id": self.round_id,
+            "provenance": {
+                "self_reported": {
+                    "motivation": self.motivation,
+                    "reasoning": self.reasoning,
+                    "tool_calls": tool_calls,
+                    "trace": self.trace,
+                },
+                "observed": {
+                    "agent_behavior": agent_behavior,
+                },
+                "verified": {
+                    "results": results,
+                    "score": score,
+                    "manifest_sha256": self.manifest_sha256,
+                },
+            },
         }
 
     def summary(self) -> str:

shared/pg_schema.py

@@ -14,6 +14,9 @@
     code TEXT NOT NULL DEFAULT '',
     motivation TEXT NOT NULL DEFAULT '',
     trace TEXT NOT NULL DEFAULT '',
+    reasoning TEXT NOT NULL DEFAULT '',
+    tool_calls JSONB NOT NULL DEFAULT '[]',
+    agent_behavior JSONB NOT NULL DEFAULT '{}',
     metric DOUBLE PRECISION,
     success BOOLEAN NOT NULL DEFAULT FALSE,
     analysis TEXT NOT NULL DEFAULT '',
@@ -266,12 +269,29 @@ def row_to_element(row) -> DataElement:
     ``_decode_jsonb`` so callers always get dict/list regardless of whether
     the asyncpg connection has a JSONB codec registered.
     """
+    # New columns may be missing on rows produced by older schemas (e.g. a
+    # query against a yet-to-migrate replica). asyncpg Records raise on
+    # ``row[name]`` for missing keys, so probe via ``keys()``.
+    row_keys = set(row.keys()) if hasattr(row, "keys") else set()
+    reasoning = row["reasoning"] if "reasoning" in row_keys else ""
+    tool_calls = (
+        _decode_jsonb(row["tool_calls"], [])
+        if "tool_calls" in row_keys else []
+    )
+    agent_behavior = (
+        _decode_jsonb(row["agent_behavior"], {})
+        if "agent_behavior" in row_keys else {}
+    )
+
     return DataElement(
         index=row["id"],
         name=row["name"],
         code=row["code"],
         motivation=row["motivation"],
         trace=row["trace"],
+        reasoning=reasoning,
+        tool_calls=tool_calls,
+        agent_behavior=agent_behavior,
         metric=_finite_or(row["metric"], None),
         success=bool(row["success"]),
         analysis=row["analysis"],
@@ -313,12 +333,13 @@ def _jsonb(value) -> str:
 
 
 def element_to_params(element: DataElement, next_id: int) -> tuple:
-    """Convert a DataElement to a positional tuple for INSERT ($1..$20).
+    """Convert a DataElement to a positional tuple for INSERT ($1..$23).
 
-    JSONB columns (loss_curve, generated_samples, objectives) are explicitly
-    serialised with ``json.dumps`` so asyncpg sends a valid JSON string
-    regardless of statement-cache or connection-pooler settings.  Float
-    ``inf``/``nan`` are replaced with ``null`` to keep PostgreSQL happy.
+    JSONB columns (loss_curve, generated_samples, objectives, tool_calls,
+    agent_behavior) are explicitly serialised with ``json.dumps`` so
+    asyncpg sends a valid JSON string regardless of statement-cache or
+    connection-pooler settings.  Float ``inf``/``nan`` are replaced with
+    ``null`` to keep PostgreSQL happy.
     """
     round_id = element.round_id if element.round_id >= 0 else None
     return (
@@ -342,6 +363,9 @@ def element_to_params(element: DataElement, next_id: int) -> tuple:
         element.timestamp,
         round_id,
         element.task,
+        element.reasoning,
+        _jsonb(element.tool_calls),
+        _jsonb(element.agent_behavior),
     )
 
 
@@ -350,12 +374,14 @@ def element_to_params(element: DataElement, next_id: int) -> tuple:
     id, name, code, motivation, trace, metric, success, analysis,
     parent_index, generation, score, miner_uid, miner_hotkey,
     loss_curve, manifest_sha256, generated_samples, objectives,
-    timestamp, round_id, task
+    timestamp, round_id, task,
+    reasoning, tool_calls, agent_behavior
 ) VALUES (
     $1, $2, $3, $4, $5, $6, $7, $8,
     $9, $10, $11, $12, $13,
     $14, $15, $16, $17,
-    $18, $19, $20
+    $18, $19, $20,
+    $21, $22, $23
 )
 """
 

shared/protocol.py

@@ -78,17 +78,31 @@ def from_json(cls, s: str) -> Challenge:
 
 @dataclass
 class Proposal:
-    """Returned by a miner in response to a Challenge."""
+    """Returned by a miner in response to a Challenge.
+
+    ``reasoning`` and ``tool_calls`` are *self-reported* — the agent author
+    populates them. They are useful for explainability but trivially
+    fakeable; pair with validator-observed signals (proxy access logs,
+    pod wall-clock) for adversarial use.
+    """
 
     code: str = ""
     name: str = ""
     motivation: str = ""
+    # Free-form prose the agent emits to explain its design choice.
+    reasoning: str = ""
+    # List of dicts the agent self-reports about external calls it made
+    # (e.g. {"tool": "llm", "prompt": "...", "summary": "..."}). Schema is
+    # intentionally open so different agents can use different shapes.
+    tool_calls: list = field(default_factory=list)
 
     def to_json(self) -> str:
         return json.dumps({
             "code": self.code,
             "name": self.name,
             "motivation": self.motivation,
+            "reasoning": self.reasoning,
+            "tool_calls": self.tool_calls,
         })
 
     @classmethod