diff --git a/.gitignore b/.gitignore index 05dc5e4..af20327 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,3 @@ __pycache__/ experiments/ *.backup data/ -logs/ -prompts/ diff --git a/CLAUDE.md b/CLAUDE.md index 0223d86..75e3ff3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -259,33 +259,6 @@ add `HIPPIUS_*` once your Hippius keys are issued — the client will pick up the new backend automatically. To cut over fully, also set `HIPPIUS_BUCKET` (or unset `R2_BUCKET`) so dispatched URLs point at Hippius. -## Non-competitive Mode - -Set `RADAR_NONCOMPETITIVE=true` for a non-subnet deployment: - -- Validator skips wallet, subtensor, metagraph, weight-setting, on-chain - commitments. Existing competitive deployments leave it false. -- Auth uses HMAC service key (`RADAR_SERVICE_KEY`) for validator ↔ trainer - ↔ DB. Headers: `X-Radar-{Signature,Timestamp,Key-Id}`. -- Miner identity is bearer tokens issued via the operator CLI: - ``` - python -m database.operator_cli register --name alice [--hotkey ss58] - python -m database.operator_cli issue-key --miner-id --label prod - python -m database.operator_cli rotate-service-key - python -m database.operator_cli list-miners | list-keys --miner-id - python -m database.operator_cli revoke-key --key-id - ``` -- Miners pull scored history + evolve prompts locally via the miner CLI: - ``` - RADAR_DB_URL=http://db:8090 RADAR_MINER_API_KEY=rdrk_... \ - python miner/neuron.py optimize --optimizer gepa --seed --watch - python miner/neuron.py results --json - python miner/neuron.py prompts list | history | rollback - ``` -- `prompts/active.json` (atomic-write) drives which prompt variant the - agent picks each round; `prompt_id` round-trips back via - `experiments.prompt_id` so Phase C scores attribute to the variant. - ## Neuron Mode Environment Variables | Var | Default | Purpose | diff --git a/config.py b/config.py index 4d44c3c..e6d55d4 100644 --- a/config.py +++ b/config.py @@ -410,23 +410,6 @@ class Config: # all — Everything on one process (legacy / dev default). NEURON_MODE: str = os.getenv("RADAR_NEURON_MODE", "all").lower() - # ── Non-competitive mode ──────────────────────────────────── - # When true, validators + miners SKIP all chain interactions: - # * no metagraph sync, no Subtensor connection, no wallet - # * no weight setting, no on-chain commitments - # * outbound auth uses HMAC service key (RADAR_SERVICE_KEY) - # * miner identity comes from bearer API keys, not hotkeys - # Existing competitive deployments leave this false and run unchanged. - NONCOMPETITIVE: bool = os.getenv( - "RADAR_NONCOMPETITIVE", "false", - ).lower() in ("1", "true", "yes") - - # Shared HMAC secret for service-to-service auth in non-competitive - # mode (validator -> trainer, validator -> DB). Required when - # NONCOMPETITIVE is true. - SERVICE_KEY: str = os.getenv("RADAR_SERVICE_KEY", "") - SERVICE_KEY_ID: str = os.getenv("RADAR_SERVICE_KEY_ID", "operator") - # CORS origins for the public JSON API. Comma-separated, empty = no CORS # (safe default for validator mode, which doesn't speak to browsers). # In production dashboard mode set this to "https://radarnet.io". diff --git a/database/migrations/005_miner_registry_and_prompt_id.sql b/database/migrations/005_miner_registry_and_prompt_id.sql deleted file mode 100644 index 55d17a4..0000000 --- a/database/migrations/005_miner_registry_and_prompt_id.sql +++ /dev/null @@ -1,77 +0,0 @@ --- Miner registry + bearer API keys for the non-competitive cutover, --- plus the ``experiments.prompt_id`` column that closes the GEPA --- feedback loop on the miner CLI. --- --- Non-competitive deployments use operator-issued bearer tokens --- (rdrk_*) in place of Epistula/metagraph identity. A ``miner_id`` --- is an opaque operator-issued identifier (UUID by default) that --- decouples the feedback API from any on-chain hotkey. During the --- dual-stack period a miner row carries both: ``hotkey`` is the --- bittensor SS58 (nullable, only set for chain-registered miners) --- and ``miner_id`` is the bearer-auth identifier scoped to --- /miners/me/*. --- --- ``prompt_id`` on experiments lets miners correlate Phase C scores --- with the prompt variant that produced the architecture. Opaque --- to the operator; only the issuing miner interprets it. Two --- partial indexes keep the table footprint sparse for rows without --- a prompt_id (every existing row + every competitive-mode write). --- --- All DDL is idempotent — safe under partial re-runs. The --- experiments ALTER + its indexes are wrapped in an information_schema --- guard (same pattern as 002_substrate_cids.sql) because CI exercises --- the migration runner against an empty schema before init_schema() --- has bootstrapped the experiments table. On a fresh deploy the base --- CREATE TABLE in shared/pg_schema.py already declares prompt_id, so --- the guarded block is a no-op there. - -CREATE TABLE IF NOT EXISTS CURRENT_SCHEMA.miners ( - miner_id TEXT PRIMARY KEY, - name TEXT NOT NULL DEFAULT '', - hotkey TEXT NOT NULL DEFAULT '', - contact TEXT NOT NULL DEFAULT '', - created_at DOUBLE PRECISION NOT NULL DEFAULT extract(epoch from now()), - revoked_at DOUBLE PRECISION -); - -CREATE INDEX IF NOT EXISTS idx_miners_hotkey - ON CURRENT_SCHEMA.miners(hotkey) - WHERE hotkey <> ''; - -CREATE TABLE IF NOT EXISTS CURRENT_SCHEMA.miner_api_keys ( - key_id TEXT PRIMARY KEY, - key_hash TEXT NOT NULL, - miner_id TEXT NOT NULL REFERENCES CURRENT_SCHEMA.miners(miner_id) - ON DELETE CASCADE, - scope TEXT NOT NULL DEFAULT 'miner', - label TEXT NOT NULL DEFAULT '', - created_at DOUBLE PRECISION NOT NULL DEFAULT extract(epoch from now()), - last_used_at DOUBLE PRECISION, - revoked_at DOUBLE PRECISION -); - -CREATE INDEX IF NOT EXISTS idx_miner_api_keys_hash - ON CURRENT_SCHEMA.miner_api_keys(key_hash) - WHERE revoked_at IS NULL; - -CREATE INDEX IF NOT EXISTS idx_miner_api_keys_miner - ON CURRENT_SCHEMA.miner_api_keys(miner_id); - -DO $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = 'CURRENT_SCHEMA' - AND table_name = 'experiments' - ) THEN - ALTER TABLE CURRENT_SCHEMA.experiments - ADD COLUMN IF NOT EXISTS prompt_id TEXT; - CREATE INDEX IF NOT EXISTS idx_experiments_prompt_id - ON CURRENT_SCHEMA.experiments(prompt_id) - WHERE prompt_id IS NOT NULL; - CREATE INDEX IF NOT EXISTS idx_experiments_miner_prompt - ON CURRENT_SCHEMA.experiments(miner_hotkey, prompt_id) - WHERE prompt_id IS NOT NULL; - END IF; -END $$; diff --git a/database/miner_feedback.py b/database/miner_feedback.py deleted file mode 100644 index 0bfdca4..0000000 --- a/database/miner_feedback.py +++ /dev/null @@ -1,339 +0,0 @@ -"""Miner-feedback endpoints — bearer-auth ``/miners/me/*`` + public -per-task ``/tasks/{task}/frontier``. - -Mounted onto the DB FastAPI app via ``include_miner_feedback_routes``. -Auth runs through ``database.server.auth_middleware`` which delegates -bearer lookups to ``shared.miner_auth`` for any path prefixed -``/miners/me/``. - -These endpoints are the data plane behind the miner CLI -(``miner/cli.py``): ``MinerResultsClient`` calls them to pull -submissions + Phase C scores for the local optimizer (GEPA, etc.). -""" - -from __future__ import annotations - -import logging -from typing import Optional - -from fastapi import APIRouter, FastAPI, HTTPException, Request - -from shared.miner_auth import touch_key_usage - -logger = logging.getLogger(__name__) - -miner_feedback_router = APIRouter() -public_router = APIRouter() - - -def _identity(request: Request): - """Pull the resolved ``MinerIdentity`` off ``request.state`` (set by - the auth middleware). 401 if absent — middleware should always set - it for ``/miners/me/*`` paths.""" - ident = getattr(request.state, "miner_identity", None) - if ident is None: - raise HTTPException(status_code=401, detail="Bearer auth required") - return ident - - -def _pool_or_503(): - """Lazy import to avoid circular dependency with database.server.""" - from database import server - if server._pool is None: - raise HTTPException(status_code=503, detail="DB pool unavailable") - return server._pool - - -def _miner_filter_clause(ident) -> tuple[str, list]: - """SQL fragment + params to scope a query to this miner's rows. - - During the dual-stack period a miner row may have a ``hotkey`` (the - on-chain identity used by older write paths) and a ``miner_id`` (the - new operator-issued identifier). We accept BOTH so submissions - written with either identity surface in the feedback API. - """ - # We match on miner_hotkey OR a computed miner_id fallback so old - # rows that were written before the registry existed still surface. - if ident.hotkey: - return ( - "(experiments.miner_hotkey = $1)", - [ident.hotkey], - ) - # No hotkey on file — use the miner_id directly against the prompt_id - # tail (best-effort; miners with no hotkey are post-cutover only). - return ("FALSE", []) - - -# ── /miners/me/submissions ────────────────────────────────────────── - - -@miner_feedback_router.get("/miners/me/submissions") -async def miner_submissions( - request: Request, - since: Optional[str] = None, - limit: int = 200, - task: str = "", -): - """Return Phase A submissions this miner produced. - - JSON shape:: - - {"submissions": [{round_id, submission_id, task_name, prompt_id, - architecture_code, motivation, reasoning, - tool_calls, created_at}, ...]} - """ - ident = _identity(request) - pool = _pool_or_503() - limit = max(1, min(int(limit or 200), 1000)) - where, params = _miner_filter_clause(ident) - if where == "FALSE": - return {"submissions": []} - - sql_parts = [ - "SELECT id AS submission_id, round_id, task AS task_name, " - "prompt_id, code AS architecture_code, motivation, reasoning, " - "tool_calls, timestamp AS created_at " - "FROM experiments WHERE " + where, - ] - if task: - sql_parts.append(f"AND task = ${len(params) + 1}") - params.append(task) - if since: - sql_parts.append(f"AND timestamp >= ${len(params) + 1}") - params.append(_parse_iso(since)) - sql_parts.append("ORDER BY timestamp DESC") - sql_parts.append(f"LIMIT ${len(params) + 1}") - params.append(limit) - - rows = await pool.fetch(" ".join(sql_parts), *params) - await touch_key_usage(pool, ident.key_id) - return {"submissions": [_row_to_submission(r) for r in rows]} - - -# ── /miners/me/results ────────────────────────────────────────────── - - -@miner_feedback_router.get("/miners/me/results") -async def miner_results( - request: Request, - since: Optional[str] = None, - limit: int = 200, - task: str = "", -): - """Submissions joined with Phase C scoring metadata. - - The experiments table already holds the scoring columns (``metric``, - ``score``, ``success``, ``objectives``) — Phase C writes them in - place — so we just project them alongside the submission shape and - fold them into a ``scores`` blob the optimizer can consume. - """ - ident = _identity(request) - pool = _pool_or_503() - limit = max(1, min(int(limit or 200), 1000)) - where, params = _miner_filter_clause(ident) - if where == "FALSE": - return {"results": []} - - sql_parts = [ - "SELECT id AS submission_id, round_id, task AS task_name, " - "prompt_id, code AS architecture_code, motivation, " - "metric, score, success, objectives, " - "timestamp AS created_at " - "FROM experiments WHERE " + where, - ] - if task: - sql_parts.append(f"AND task = ${len(params) + 1}") - params.append(task) - if since: - sql_parts.append(f"AND timestamp >= ${len(params) + 1}") - params.append(_parse_iso(since)) - sql_parts.append("ORDER BY timestamp DESC") - sql_parts.append(f"LIMIT ${len(params) + 1}") - params.append(limit) - - rows = await pool.fetch(" ".join(sql_parts), *params) - await touch_key_usage(pool, ident.key_id) - return {"results": [_row_to_result(r) for r in rows]} - - -# ── /miners/me/summary ────────────────────────────────────────────── - - -@miner_feedback_router.get("/miners/me/summary") -async def miner_summary(request: Request): - """Quick stats — total submissions, last seen round, mean score.""" - ident = _identity(request) - pool = _pool_or_503() - where, params = _miner_filter_clause(ident) - if where == "FALSE": - return { - "miner_id": ident.miner_id, - "total_submissions": 0, - "last_round_id": None, - "mean_score_recent": None, - "successes_recent": 0, - } - - summary_sql = ( - "SELECT COUNT(*) AS total, MAX(round_id) AS last_round " - "FROM experiments WHERE " + where - ) - s = await pool.fetchrow(summary_sql, *params) - - # Recent-window stats (last 50 rows). - recent_sql = ( - "SELECT AVG(score) AS mean_score, " - "SUM(CASE WHEN success THEN 1 ELSE 0 END) AS successes, " - "COUNT(*) AS n " - "FROM (SELECT score, success FROM experiments WHERE " + where + - " ORDER BY timestamp DESC LIMIT 50) sub" - ) - r = await pool.fetchrow(recent_sql, *params) - - await touch_key_usage(pool, ident.key_id) - return { - "miner_id": ident.miner_id, - "total_submissions": int(s["total"] or 0), - "last_round_id": int(s["last_round"]) if s["last_round"] is not None else None, - "mean_score_recent": float(r["mean_score"]) if r["mean_score"] is not None else None, - "successes_recent": int(r["successes"] or 0), - "recent_window": int(r["n"] or 0), - } - - -# ── /tasks/{task}/frontier (public) ───────────────────────────────── - - -@public_router.get("/tasks/{task}/frontier") -async def task_frontier(task: str, limit: int = 50): - """Per-task Pareto frontier as ``{points: [{flops, crps, mase}, …]}``. - - Public — no bearer required. Same data the optimizer's - ``MinerResultsClient.frontier()`` hits. Computed live from the - experiments table (top successful rows per FLOPs bucket). - """ - pool = _pool_or_503() - limit = max(1, min(int(limit or 50), 500)) - sql = ( - "SELECT metric, objectives, score " - "FROM experiments " - "WHERE task = $1 AND success = TRUE AND metric IS NOT NULL " - "ORDER BY metric ASC " - "LIMIT $2" - ) - rows = await pool.fetch(sql, task, limit) - points = [] - for r in rows: - # objectives is JSONB; asyncpg may return as str without codec. - obj = r["objectives"] - if isinstance(obj, (bytes, bytearray)): - obj = obj.decode() - if isinstance(obj, str): - import json as _json - try: - obj = _json.loads(obj) - except (ValueError, TypeError): - obj = {} - if not isinstance(obj, dict): - obj = {} - points.append({ - "flops": obj.get("flops_equivalent_size"), - "metric": r["metric"], - "score": r["score"], - }) - return {"task": task, "points": points} - - -# ── Helpers ──────────────────────────────────────────────────────── - - -def _parse_iso(s: str) -> float: - """Accept an ISO-8601 timestamp and return a POSIX seconds float so - we can compare against ``experiments.timestamp``.""" - if not s: - return 0.0 - from datetime import datetime - try: - return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp() - except ValueError: - # Fall back to "treat as already-numeric epoch". - try: - return float(s) - except (TypeError, ValueError): - return 0.0 - - -def _row_to_submission(row) -> dict: - return { - "submission_id": str(row["submission_id"]), - "round_id": int(row["round_id"]) if row["round_id"] is not None else None, - "task_name": row["task_name"] or "", - "prompt_id": row["prompt_id"], - "architecture_code": row["architecture_code"] or "", - "motivation": row["motivation"] or "", - "reasoning": row["reasoning"] or "", - "tool_calls": _decode_jsonb(row["tool_calls"], []), - "created_at": _to_iso(row["created_at"]), - } - - -def _row_to_result(row) -> dict: - obj = _decode_jsonb(row["objectives"], {}) - return { - "submission_id": str(row["submission_id"]), - "round_id": int(row["round_id"]) if row["round_id"] is not None else None, - "task_name": row["task_name"] or "", - "prompt_id": row["prompt_id"], - "architecture_code": row["architecture_code"] or "", - "motivation": row["motivation"] or "", - "scores": { - "metric": row["metric"], - "raw_score": row["score"], - "success": bool(row["success"]), - "flops_equivalent_size": obj.get("flops_equivalent_size"), - **{k: v for k, v in obj.items() if k != "flops_equivalent_size"}, - }, - "created_at": _to_iso(row["created_at"]), - } - - -def _decode_jsonb(value, default): - if value is None: - return default - if isinstance(value, (dict, list)): - return value - if isinstance(value, (bytes, bytearray)): - value = value.decode("utf-8") - if isinstance(value, str): - import json as _json - try: - return _json.loads(value) - except (ValueError, TypeError): - return default - return default - - -def _to_iso(value) -> str: - if value is None: - return "" - if isinstance(value, str): - return value - try: - from datetime import datetime, timezone - return datetime.fromtimestamp(float(value), tz=timezone.utc).isoformat(timespec="seconds") - except (TypeError, ValueError): - return "" - - -# ── Mount ────────────────────────────────────────────────────────── - - -def include_miner_feedback_routes(target_app: FastAPI) -> None: - """Idempotent mount. Safe to call multiple times.""" - already = any( - getattr(r, "path", None) == "/miners/me/summary" for r in target_app.routes - ) - if already: - return - target_app.include_router(miner_feedback_router) - target_app.include_router(public_router) diff --git a/database/operator_cli.py b/database/operator_cli.py deleted file mode 100644 index 6dda76a..0000000 --- a/database/operator_cli.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Operator CLI — register miners and issue / revoke their API keys. - -Run against the DB Postgres directly: - - python -m database.operator_cli register --name alice [--hotkey 5xxx] [--contact ...] - python -m database.operator_cli issue-key --miner-id [--label prod] - python -m database.operator_cli revoke-key --key-id - python -m database.operator_cli list-keys --miner-id - python -m database.operator_cli list-miners - python -m database.operator_cli rotate-service-key - -DSN is read from ``RADAR_PG_DSN`` (same as the rest of the stack). - -Plaintext API keys are printed exactly once at issuance — copy them -out of the terminal immediately; the DB only stores the SHA-256. -""" - -from __future__ import annotations - -import argparse -import asyncio -import os -import secrets -import sys -from typing import Optional - -# Importing ``config`` triggers ``load_dotenv`` so RADAR_PG_DSN and -# friends resolve from ``radar/.env`` the same way they do for every -# other CLI in the repo. Without this, operator_cli read os.environ -# directly and silently ignored .env. -import config # noqa: F401 - -from shared import miner_auth - - -# ── DB connection ────────────────────────────────────────────────── - - -async def _connect(): - import asyncpg - dsn = os.getenv("RADAR_PG_DSN", "").strip() - if not dsn: - print( - "operator: RADAR_PG_DSN not set. Example: " - "postgresql://radar:radar@localhost:5432/radar", - file=sys.stderr, - ) - sys.exit(2) - return await asyncpg.connect(dsn) - - -async def _ensure_schema(conn) -> None: - """Create the miners + miner_api_keys tables if they don't exist - yet. Mirrors PgExperimentStore.init_schema so the operator CLI - works against a stock Postgres on day one.""" - from shared.pg_schema import MINER_REGISTRY_SCHEMA - - # The DDL references experiments.prompt_id; if the table doesn't - # exist yet we still want miners/miner_api_keys created. Run each - # statement separately and tolerate the ALTER failing. - for stmt in [s.strip() for s in MINER_REGISTRY_SCHEMA.split(";") if s.strip()]: - try: - await conn.execute(stmt + ";") - except Exception as e: - # ALTER on missing experiments table is fine to ignore; - # other failures surface so the operator notices. - msg = str(e).lower() - if "experiments" in msg and "does not exist" in msg: - continue - print(f"warn: DDL stmt failed: {e}", file=sys.stderr) - - -# ── Command handlers ─────────────────────────────────────────────── - - -async def cmd_register(args) -> int: - conn = await _connect() - try: - await _ensure_schema(conn) - mid = await miner_auth.register_miner( - conn, - miner_id=args.miner_id, - name=args.name, - hotkey=args.hotkey, - contact=args.contact, - ) - print(f"miner_id: {mid}") - return 0 - finally: - await conn.close() - - -async def cmd_issue_key(args) -> int: - conn = await _connect() - try: - await _ensure_schema(conn) - plaintext, key_id = await miner_auth.issue_api_key( - conn, args.miner_id, scope=args.scope, label=args.label, - ) - print("=" * 60) - print("API KEY (shown ONCE — copy now, will not be stored):") - print(plaintext) - print() - print(f"key_id {key_id}") - print(f"miner_id {args.miner_id}") - print(f"scope {args.scope}") - print(f"label {args.label or '(none)'}") - print("=" * 60) - return 0 - finally: - await conn.close() - - -async def cmd_revoke_key(args) -> int: - conn = await _connect() - try: - ok = await miner_auth.revoke_api_key(conn, args.key_id) - if ok: - print(f"revoked {args.key_id}") - return 0 - print(f"no active key with id {args.key_id}", file=sys.stderr) - return 1 - finally: - await conn.close() - - -async def cmd_list_keys(args) -> int: - conn = await _connect() - try: - rows = await miner_auth.list_api_keys(conn, args.miner_id) - if not rows: - print("(no keys)") - return 0 - print(f"{'key_id':<34} {'scope':<8} {'label':<18} " - f"{'created':<22} {'revoked':<22} last_used") - for r in rows: - print( - f"{r['key_id']:<34} {r['scope']:<8} " - f"{(r.get('label') or '')[:18]:<18} " - f"{_fmt_ts(r.get('created_at')):<22} " - f"{_fmt_ts(r.get('revoked_at')):<22} " - f"{_fmt_ts(r.get('last_used_at'))}" - ) - return 0 - finally: - await conn.close() - - -async def cmd_list_miners(args) -> int: - conn = await _connect() - try: - rows = await conn.fetch( - "SELECT miner_id, name, hotkey, contact, created_at, revoked_at " - "FROM miners ORDER BY created_at DESC LIMIT $1", - int(args.limit), - ) - if not rows: - print("(no miners)") - return 0 - print(f"{'miner_id':<34} {'name':<20} {'hotkey':<20} " - f"{'created':<22} revoked") - for r in rows: - print( - f"{r['miner_id']:<34} " - f"{(r['name'] or '')[:20]:<20} " - f"{(r['hotkey'] or '')[:20]:<20} " - f"{_fmt_ts(r.get('created_at')):<22} " - f"{_fmt_ts(r.get('revoked_at'))}" - ) - return 0 - finally: - await conn.close() - - -def cmd_rotate_service_key(args) -> int: - """Print a fresh service key — operator pastes into env.""" - key = secrets.token_hex(32) - print("New RADAR_SERVICE_KEY value (set on operator + trainers + validators):") - print(key) - print() - print("Rotation procedure:") - print(" 1. Roll out to every trainer first (they're the verifiers).") - print(" 2. Then update validators / operator dispatchers.") - print(" 3. Confirm /train accepts the new key before retiring the old one.") - return 0 - - -# ── argparse plumbing ────────────────────────────────────────────── - - -def _build_parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser( - prog="radar-operator", - description="Manage miners + API keys for non-competitive Radar.", - ) - sub = p.add_subparsers(dest="command", required=True) - - r = sub.add_parser("register", help="Register a new miner.") - r.add_argument("--name", default="") - r.add_argument("--hotkey", default="", - help="Bittensor SS58 (optional; for dual-stack period).") - r.add_argument("--contact", default="") - r.add_argument("--miner-id", default=None, - help="Explicit id (default: random UUID).") - - i = sub.add_parser("issue-key", help="Mint an API key for a miner.") - i.add_argument("--miner-id", required=True) - i.add_argument("--scope", default="miner", - help="Scope tag stored on the key (default: 'miner').") - i.add_argument("--label", default="", - help="Operator note shown in list-keys output.") - - rk = sub.add_parser("revoke-key", help="Revoke an API key by key_id.") - rk.add_argument("--key-id", required=True) - - lk = sub.add_parser("list-keys", help="List keys for a miner.") - lk.add_argument("--miner-id", required=True) - - lm = sub.add_parser("list-miners", help="List registered miners.") - lm.add_argument("--limit", type=int, default=100) - - sub.add_parser("rotate-service-key", - help="Print a fresh RADAR_SERVICE_KEY.") - return p - - -def _fmt_ts(value) -> str: - if value in (None, 0, 0.0): - return "-" - try: - from datetime import datetime, timezone - return datetime.fromtimestamp(float(value), tz=timezone.utc).isoformat( - timespec="seconds", - ) - except (TypeError, ValueError): - return str(value) - - -def main(argv: Optional[list[str]] = None) -> int: - args = _build_parser().parse_args(argv) - if args.command == "rotate-service-key": - return cmd_rotate_service_key(args) - handler = { - "register": cmd_register, - "issue-key": cmd_issue_key, - "revoke-key": cmd_revoke_key, - "list-keys": cmd_list_keys, - "list-miners": cmd_list_miners, - }[args.command] - return asyncio.run(handler(args)) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/database/server.py b/database/server.py index 3568dda..e24ee46 100644 --- a/database/server.py +++ b/database/server.py @@ -304,12 +304,6 @@ async def auth_middleware(request: Request, call_next): path = request.url.path - # Bearer-auth surface for the non-competitive feedback API. Handled - # before the Epistula path because tokens use Authorization: Bearer - # headers, not the X-Epistula-* set. - if path.startswith("/miners/me/"): - return await _bearer_auth_then_call(request, call_next) - needs_auth = ( path.startswith("/experiments") or path.startswith("/challenge") @@ -334,42 +328,18 @@ async def auth_middleware(request: Request, call_next): status_code=413, content={"error": "Request body too large"}, ) - # ── Validator API key: trusted proxy, fast path ── + # ── Validator API key: trusted proxy, skip Epistula ── proxy_authed = False if Config.DB_API_KEY: import secrets as _secrets provided = request.headers.get("X-Radar-API-Key", "") if provided and _secrets.compare_digest(provided, Config.DB_API_KEY): + # Trusted validator proxy — rate-limit by IP instead of hotkey if not _check_rate_limit(client_ip): return JSONResponse(status_code=429, content={"error": "Rate limit exceeded"}) proxy_authed = True - # ── HMAC service-key auth (non-competitive default) ── - if not proxy_authed: - import os as _os - service_key = _os.getenv("RADAR_SERVICE_KEY", "").strip() - if service_key: - from shared.auth import ( - hmac_verify_request, static_key_lookup, - ) - expected_kid = _os.getenv("RADAR_SERVICE_KEY_ID", "operator") - ok, err, kid = hmac_verify_request( - dict(request.headers), body, - static_key_lookup(expected_kid, service_key.encode()), - ) - if ok: - request.state.caller_hotkey = kid - if not _check_rate_limit(kid): - return JSONResponse( - status_code=429, - content={"error": "Rate limit exceeded"}, - ) - proxy_authed = True - elif _metagraph is None: - # No Epistula fallback configured — reject. - return JSONResponse(status_code=403, content={"error": err}) - - # ── Legacy Epistula path (dual-stack period) ── + # ── Epistula auth: miners and validators without API key ── if not proxy_authed and _metagraph: from shared.auth import verify_request ok, err, hotkey = verify_request(dict(request.headers), body, _metagraph) @@ -1089,57 +1059,9 @@ def include_validator_routes(target_app: FastAPI) -> None: target_app.include_router(validator_router) -async def _bearer_auth_then_call(request: Request, call_next): - """Resolve a bearer token against ``miner_api_keys`` and attach the - identity to ``request.state`` for the route handler. - - 401 if the token is missing / unknown / revoked, 503 if the asyncpg - pool isn't wired up yet (dashboard-only mode that accidentally - received bearer traffic, or pre-startup race). - """ - from shared.auth import extract_bearer - from shared.miner_auth import lookup_bearer - - token = extract_bearer(dict(request.headers)) - if not token: - return JSONResponse( - status_code=401, content={"error": "Bearer token required"}, - ) - if _pool is None: - return JSONResponse( - status_code=503, content={"error": "DB pool not ready"}, - ) - try: - ident = await lookup_bearer(_pool, token) - except Exception as e: - logger.warning("bearer lookup failed: %s", e) - return JSONResponse( - status_code=503, content={"error": "auth lookup failed"}, - ) - if ident is None: - return JSONResponse( - status_code=403, content={"error": "Unknown or revoked token"}, - ) - request.state.miner_identity = ident - return await call_next(request) - - -def include_miner_feedback_routes(target_app: FastAPI) -> None: - """Mount the bearer-auth ``/miners/me/*`` + public per-task frontier - routes onto ``target_app``. Idempotent.""" - from database.miner_feedback import ( - include_miner_feedback_routes as _mount, - ) - _mount(target_app) - - # Legacy auto-mount: keep the validator surface on ``app`` at import time # unless this process is running in ``dashboard`` mode. Preserves backward # compatibility for tests that import ``app`` directly and expect every # route to be reachable. if Config.NEURON_MODE != "dashboard": include_validator_routes(app) -# Miner feedback routes are mounted unconditionally — they need bearer -# auth (no chain dependency) and are useful in both validator and -# dashboard deployments. -include_miner_feedback_routes(app) diff --git a/ecosystem.config.js b/ecosystem.config.js deleted file mode 100644 index 659e62b..0000000 --- a/ecosystem.config.js +++ /dev/null @@ -1,85 +0,0 @@ -// pm2 ecosystem for running a non-competitive miner alongside the GEPA -// prompt-optimizer loop on the same host. -// -// Usage: -// mkdir -p logs prompts -// python miner/neuron.py optimize --seed --dry-run # one-time: seed active.json -// pm2 start ecosystem.config.js -// pm2 logs -// pm2 save && pm2 startup # survive reboot -// -// Both processes share env via `sharedEnv` so prompts/active.json is the -// single source of truth: `radar-gepa` rewrites it after each round's -// scores land, `radar-miner` reads it on the next round — no restart -// needed. -// -// Secrets: do NOT commit RADAR_MINER_API_KEY. Either keep this file out -// of git on the deploy host or load secrets from a sibling .env and -// reference process.env.RADAR_MINER_API_KEY here. - -const path = require("path"); - -const sharedEnv = { - RADAR_NONCOMPETITIVE: "true", - RADAR_DB_URL: process.env.RADAR_DB_URL || "http://localhost:8090", - RADAR_MINER_API_KEY: process.env.RADAR_MINER_API_KEY || "", - MINER_PROMPTS_DIR: path.join(__dirname, "prompts"), - - // GEPA reflector LM — operator's shared LLM proxy. Swap to - // MINER_REFLECTOR_API_{BASE,KEY,MODEL} for an explicit OpenAI / - // Anthropic-style endpoint instead. - MINER_LLM_URL: process.env.MINER_LLM_URL || "", - MINER_LLM_API_KEY: process.env.MINER_LLM_API_KEY || "", - MINER_LLM_MODEL: process.env.MINER_LLM_MODEL || - "deepseek-ai/DeepSeek-V3-0324", -}; - -// Default points at the autonomous example agent in a sibling -// radar-miner-examples checkout. Override with RADAR_AGENT_DIR to use a -// different example (openai_sdk_v2, claude_style_v2, patch_decoder) or -// your own agent directory. -const agentDir = process.env.RADAR_AGENT_DIR || - "../radar-miner-examples/agents/autonomous/"; - -module.exports = { - apps: [ - { - name: "radar-miner", - script: "miner/neuron.py", - interpreter: "python3", - args: `--agent_dir ${agentDir}`, - cwd: __dirname, - env: sharedEnv, - autorestart: true, - max_restarts: 20, - restart_delay: 5000, - out_file: "./logs/miner.out.log", - error_file: "./logs/miner.err.log", - merge_logs: true, - time: true, - }, - { - name: "radar-gepa", - script: "miner/neuron.py", - interpreter: "python3", - args: [ - "optimize", - "--optimizer", "gepa", - "--population", "8", - "--budget", "30", - "--elite-k", "2", - "--watch", - "--every-seconds", "600", - ].join(" "), - cwd: __dirname, - env: sharedEnv, - autorestart: true, - max_restarts: 20, - restart_delay: 10000, - out_file: "./logs/gepa.out.log", - error_file: "./logs/gepa.err.log", - merge_logs: true, - time: true, - }, - ], -}; diff --git a/miner/cli.py b/miner/cli.py deleted file mode 100644 index d02767a..0000000 --- a/miner/cli.py +++ /dev/null @@ -1,384 +0,0 @@ -"""Miner CLI subcommands — ``results``, ``optimize``, ``prompts``. - -Routed from ``miner/neuron.py``'s ``main()``. When the first argv is -one of the known subcommands the neuron delegates here; otherwise the -neuron falls through to the legacy ``run`` behavior so existing -deployments keep working without flag churn. - - miner/neuron.py run [existing flags] - legacy entrypoint — serve agent + trainer-launcher listener - - miner/neuron.py results [--since 1d] [--limit 50] [--json] [--task X] - one-shot: print recent scored results from the DB - - miner/neuron.py optimize \\ - [--optimizer gepa|random_mutate|pkg.mod:func] \\ - [--rounds-back 20] [--population 8] [--budget 30] \\ - [--score-key raw_score] [--dry-run] [--task ts_forecasting] - one-shot: pull results, evolve the prompt population, write new - active.json (archives the old as history/gen_NNN.json) - - miner/neuron.py optimize --watch [--every-seconds 600] - daemon loop: re-run optimize whenever ``my_summary().last_round_id`` - advances - - miner/neuron.py prompts list - show the current population - - miner/neuron.py prompts history - list archived generation numbers - - miner/neuron.py prompts rollback - restore an archived generation (with safety-archive of current) - -DB URL + API key are read from env (``RADAR_DB_URL``, -``RADAR_MINER_API_KEY``) or flags. -""" - -from __future__ import annotations - -import argparse -import json -import logging -import os -import sys -import time -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Optional - -from miner_template import optimizers, prompts as prompts_mod -from miner_template.optimizers import ResultRow -from miner_template.results_client import MinerResultsClient - -logger = logging.getLogger(__name__) - -# argv[1] values that route here instead of the legacy `run` entrypoint. -SUBCOMMANDS = frozenset({"results", "optimize", "prompts"}) - - -def is_subcommand(argv: list[str]) -> bool: - """Return True if argv[1] looks like one of our subcommands.""" - return len(argv) >= 2 and argv[1] in SUBCOMMANDS - - -def dispatch(argv: list[str]) -> int: - """Top-level entrypoint. argv[0] is the program, argv[1] is the - subcommand. Returns a process exit code.""" - if len(argv) < 2 or argv[1] not in SUBCOMMANDS: - print( - f"Unknown subcommand. Expected one of: {sorted(SUBCOMMANDS)}", - file=sys.stderr, - ) - return 2 - - parser = _root_parser() - args = parser.parse_args(argv[1:]) - - if args.subcommand == "results": - return cmd_results(args) - if args.subcommand == "optimize": - return cmd_optimize(args) - if args.subcommand == "prompts": - return cmd_prompts(args) - parser.error(f"unhandled subcommand {args.subcommand}") - return 2 - - -# ── argparse plumbing ──────────────────────────────────────────────── - - -def _root_parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser( - prog="miner", - description="Radar miner — agent host, results polling, prompt optimizer.", - ) - sub = p.add_subparsers(dest="subcommand", required=True) - - # results - r = sub.add_parser("results", help="Print recent scored results.") - _add_db_flags(r) - _add_window_flags(r) - r.add_argument("--json", action="store_true", - help="Emit one JSON line per result instead of a table.") - - # optimize - o = sub.add_parser("optimize", help="Run a prompt-optimization pass.") - _add_db_flags(o) - _add_window_flags(o) - o.add_argument("--optimizer", default="gepa", - help="Optimizer to use: 'gepa', 'random_mutate', or " - "'package.module:func'. Default: gepa.") - o.add_argument("--population", type=int, default=8, - help="Target size of the new prompt population.") - o.add_argument("--budget", type=int, default=30, - help="Optimizer rollout/compile budget (GEPA-only).") - o.add_argument("--score-key", default="raw_score", - help="scores[] field to optimize against.") - o.add_argument("--elite-k", type=int, default=2, - help="Top-K survivors carried forward (random_mutate).") - o.add_argument("--prompts-dir", default=None, - help=f"Where prompts live " - f"(default ${prompts_mod.DEFAULT_PROMPTS_DIR}).") - o.add_argument("--seed", action="store_true", - help="Seed a default prompt if active.json is missing.") - o.add_argument("--dry-run", action="store_true", - help="Print the diff but don't write files.") - o.add_argument("--watch", action="store_true", - help="Loop forever, re-running when new rounds finish.") - o.add_argument("--every-seconds", type=int, default=600, - help="--watch poll interval.") - - # prompts - pr = sub.add_parser("prompts", help="Inspect/manage the prompt population.") - psub = pr.add_subparsers(dest="prompts_command", required=True) - pl = psub.add_parser("list", help="Show the current active population.") - pl.add_argument("--prompts-dir", default=None) - pl.add_argument("--json", action="store_true") - psub.add_parser("history", help="List archived generations.").add_argument( - "--prompts-dir", default=None, - ) - rb = psub.add_parser("rollback", help="Restore an archived generation.") - rb.add_argument("generation", type=int) - rb.add_argument("--prompts-dir", default=None) - - return p - - -def _add_db_flags(p: argparse.ArgumentParser) -> None: - p.add_argument("--db-url", - default=os.getenv("RADAR_DB_URL", ""), - help="DB server URL (env: RADAR_DB_URL).") - p.add_argument("--api-key", - default=os.getenv("RADAR_MINER_API_KEY", ""), - help="Miner bearer API key (env: RADAR_MINER_API_KEY).") - - -def _add_window_flags(p: argparse.ArgumentParser) -> None: - p.add_argument("--rounds-back", type=int, default=20, - help="How many rounds of history to consider.") - p.add_argument("--since", default="", - help="ISO timestamp override; takes precedence over --rounds-back.") - p.add_argument("--limit", type=int, default=500, - help="Max rows pulled from the DB.") - p.add_argument("--task", default="", - help="Filter to one task name (e.g. ts_forecasting).") - - -# ── results ───────────────────────────────────────────────────────── - - -def cmd_results(args) -> int: - if not args.db_url or not args.api_key: - print( - "results: missing --db-url or --api-key " - "(or RADAR_DB_URL / RADAR_MINER_API_KEY).", - file=sys.stderr, - ) - return 2 - since = args.since or _approx_since(args.rounds_back) - with MinerResultsClient(args.db_url, args.api_key) as c: - rows = c.results(since=since, limit=args.limit, task=args.task) - if args.json: - for r in rows: - print(json.dumps({ - "round_id": r.round_id, - "submission_id": r.submission_id, - "task_name": r.task_name, - "prompt_id": r.prompt_id, - "scores": r.scores, - "created_at": r.created_at, - })) - else: - _print_results_table(rows) - return 0 - - -def _print_results_table(rows: list[ResultRow]) -> None: - if not rows: - print("(no results)") - return - header = f"{'round':>6} {'prompt_id':<14} {'score':>8} {'task':<16} submission" - print(header) - print("-" * len(header)) - for r in rows: - pid = (r.prompt_id or "-")[:12] - score = float(r.scores.get("raw_score", 0.0) or 0.0) - task = (r.task_name or "-")[:14] - print(f"{r.round_id:>6} {pid:<14} {score:>8.4f} {task:<16} {r.submission_id}") - - -# ── optimize ──────────────────────────────────────────────────────── - - -def cmd_optimize(args) -> int: - if not args.db_url or not args.api_key: - print( - "optimize: missing --db-url or --api-key " - "(or RADAR_DB_URL / RADAR_MINER_API_KEY).", - file=sys.stderr, - ) - return 2 - prompts_dir = Path(args.prompts_dir) if args.prompts_dir else None - - if args.watch: - return _watch_loop(args, prompts_dir) - return _optimize_once(args, prompts_dir) - - -def _optimize_once(args, prompts_dir: Optional[Path]) -> int: - population = prompts_mod.load_active(prompts_dir) - if not population: - if args.seed: - population = prompts_mod.seed_default(prompts_dir) - print(f"Seeded default prompt ({population[0].id[:8]}…)") - else: - print( - "optimize: no active prompt population yet. " - "Re-run with --seed to write a default starter prompt.", - file=sys.stderr, - ) - return 2 - - since = args.since or _approx_since(args.rounds_back) - with MinerResultsClient(args.db_url, args.api_key) as c: - results = c.results(since=since, limit=args.limit, task=args.task) - print( - f"Loaded {len(results)} scored results since {since or 'beginning'} " - f"(population size {len(population)}).", - ) - - try: - fn = optimizers.resolve(args.optimizer) - except (ImportError, ValueError, AttributeError, TypeError) as e: - print(f"optimize: cannot resolve optimizer: {e}", file=sys.stderr) - return 2 - - config = { - "population": args.population, - "budget": args.budget, - "score_key": args.score_key, - "elite_k": args.elite_k, - } - try: - new_pop = fn(results, population, config) - except Exception as e: - print(f"optimize: optimizer raised: {e}", file=sys.stderr) - return 1 - - if not new_pop: - print("optimize: optimizer returned an empty population — no change.", - file=sys.stderr) - return 1 - - if args.dry_run: - _print_pop_diff(population, new_pop) - return 0 - - next_gen = (prompts_mod.list_history(prompts_dir) or [0])[-1] + 1 - prompts_mod.archive_current(next_gen, prompts_dir) - prompts_mod.save_active(new_pop, prompts_dir) - print( - f"Wrote generation {next_gen + 1} " - f"({len(new_pop)} prompts); previous archived as gen_{next_gen:03d}.json", - ) - return 0 - - -def _watch_loop(args, prompts_dir: Optional[Path]) -> int: - last_round = -1 - print(f"watch: polling every {args.every_seconds}s …") - while True: - try: - with MinerResultsClient(args.db_url, args.api_key) as c: - summary = c.summary() - cur = int(summary.get("last_round_id", -1) or -1) - except Exception as e: - logger.warning("watch: summary fetch failed: %s — retrying", e) - time.sleep(args.every_seconds) - continue - if cur > last_round: - print(f"watch: new round {cur} — running optimizer …") - code = _optimize_once(args, prompts_dir) - if code != 0: - logger.warning("watch: optimizer exited %d", code) - last_round = cur - time.sleep(args.every_seconds) - - -def _print_pop_diff(old: list, new: list) -> None: - old_ids = {p.id for p in old} - new_ids = {p.id for p in new} - survivors = old_ids & new_ids - fresh = new_ids - old_ids - dropped = old_ids - new_ids - print(f"would write population: size={len(new)}") - print(f" survivors: {len(survivors)} new: {len(fresh)} dropped: {len(dropped)}") - for p in new: - marker = "·" if p.id in old_ids else "+" - snippet = p.template.replace("\n", " ⏎ ")[:80] - print(f" {marker} [gen {p.generation}] {p.id[:8]}… {snippet}") - - -# ── prompts subcommands ──────────────────────────────────────────── - - -def cmd_prompts(args) -> int: - if args.prompts_command == "list": - return _prompts_list(args) - if args.prompts_command == "history": - return _prompts_history(args) - if args.prompts_command == "rollback": - return _prompts_rollback(args) - return 2 - - -def _prompts_list(args) -> int: - prompts_dir = Path(args.prompts_dir) if args.prompts_dir else None - pop = prompts_mod.load_active(prompts_dir) - if args.json: - print(json.dumps([p.to_dict() for p in pop], indent=2)) - return 0 - if not pop: - print("(no active prompt population)") - return 0 - for p in pop: - snippet = p.template.replace("\n", " ⏎ ")[:80] - parent = (p.parent_id or "-")[:8] - print(f" [gen {p.generation}] {p.id[:8]}… parent={parent} {snippet}") - return 0 - - -def _prompts_history(args) -> int: - prompts_dir = Path(args.prompts_dir) if args.prompts_dir else None - gens = prompts_mod.list_history(prompts_dir) - if not gens: - print("(no archived generations)") - return 0 - for g in gens: - print(f" gen_{g:03d}") - return 0 - - -def _prompts_rollback(args) -> int: - prompts_dir = Path(args.prompts_dir) if args.prompts_dir else None - try: - prompts_mod.rollback_to(args.generation, prompts_dir) - except FileNotFoundError as e: - print(f"rollback: {e}", file=sys.stderr) - return 2 - print(f"Rolled back to generation {args.generation}.") - return 0 - - -# ── helpers ──────────────────────────────────────────────────────── - - -def _approx_since(rounds_back: int) -> str: - """Map ``--rounds-back`` to an ISO timestamp. Heuristic: each round - is roughly 55 min (see CLAUDE.md). Good enough for an upper-bound - filter — the DB will still cap with --limit.""" - if rounds_back <= 0: - return "" - delta = timedelta(minutes=55 * rounds_back) - return (datetime.now(timezone.utc) - delta).isoformat(timespec="seconds") diff --git a/miner/neuron.py b/miner/neuron.py index e6b2cbd..5809c1c 100644 --- a/miner/neuron.py +++ b/miner/neuron.py @@ -16,10 +16,7 @@ import os import time -try: - import bittensor as bt -except ImportError: # pragma: no cover — only fires in noncompetitive deploys - bt = None # type: ignore[assignment] +import bittensor as bt import httpx import uvicorn from fastapi import FastAPI, Request @@ -55,9 +52,9 @@ class Miner: """Radar subnet miner — Docker agent + warm-standby trainer listener.""" - def __init__(self, config): + def __init__(self, config: bt.Config): self.config = config - self.netuid = getattr(config, "netuid", 1) + self.netuid = config.netuid # Fail-fast on Targon misconfig — operator gets a clear error at # startup instead of a cryptic deploy failure on the first round. @@ -88,21 +85,8 @@ def __init__(self, config): "image and set RADAR_RUNPOD_ENDPOINT_ID to its id." ) - # Bittensor chain — skipped entirely in non-competitive mode - if Config.NONCOMPETITIVE: - self.wallet = None - self.subtensor = None - self.metagraph = None - else: - if bt is None: - raise RuntimeError( - "competitive mode requires the bittensor package; " - "install with `pip install bittensor` or run with " - "RADAR_NONCOMPETITIVE=true." - ) - self.wallet = bt.Wallet(config=config) - self.subtensor = bt.Subtensor(config=config) - self.metagraph = self.subtensor.metagraph(self.netuid) + self.wallet = bt.Wallet(config=config) + self.subtensor = bt.Subtensor(config=config) self._targon_client = None self._runpod_client = None # Per-round RunPod job ids the miner has submitted on behalf of @@ -110,6 +94,9 @@ def __init__(self, config): # round_id → list[job_id] self._runpod_jobs: dict[int, list[str]] = {} + # Cache metagraph — synced periodically in keep-alive loop + self.metagraph = self.subtensor.metagraph(self.netuid) + # Agent code directory — validators fetch the code and run it # in the official sandboxed image self.agent_dir = getattr(config, "agent_dir", "agent/") @@ -767,7 +754,7 @@ async def run(self): logger.warning("Metagraph sync failed: %s", e) -def get_config(): +def get_config() -> bt.Config: parser = argparse.ArgumentParser(description="Radar Miner") parser.add_argument("--netuid", type=int, default=1) parser.add_argument("--agent_dir", type=str, default="agent/", @@ -779,31 +766,15 @@ def get_config(): help="Trainer Docker image deployed on Basilica on-demand") parser.add_argument("--external_ip", type=str, default="", help="External IP for listener URL committed to chain") - if Config.NONCOMPETITIVE or bt is None: - return parser.parse_args() bt.Wallet.add_args(parser) bt.Subtensor.add_args(parser) return bt.Config(parser) def main(): - import sys - logging.basicConfig(level=logging.INFO, format="%(asctime)s | %(levelname)s | %(message)s") logging.getLogger("httpx").setLevel(logging.WARNING) logging.getLogger("httpcore").setLevel(logging.WARNING) - - # Subcommand router: ``results`` / ``optimize`` / ``prompts`` go to - # ``miner/cli.py``. Anything else (or no args) falls through to the - # legacy ``run`` behavior so existing deployments keep working. - from miner import cli - if cli.is_subcommand(sys.argv): - sys.exit(cli.dispatch(sys.argv)) - # Optional explicit ``run`` token for clarity; strip it before the - # bittensor argparse kicks in. - if len(sys.argv) >= 2 and sys.argv[1] == "run": - del sys.argv[1] - config = get_config() miner = Miner(config) asyncio.run(miner.run()) diff --git a/miner_template/__init__.py b/miner_template/__init__.py deleted file mode 100644 index fc56542..0000000 --- a/miner_template/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Miner template — starter agent + prompt-evolution scaffolding. - -The miner CLI (``miner/neuron.py``) drives this package's two surfaces: - - * ``agent.py`` — ``design_architecture(challenge, client)`` entrypoint - called by the validator each round. - * ``prompts`` + ``optimizers`` + ``results_client`` — feedback loop - that lets miners pull their own scored history and evolve their - agent's prompts (e.g. via GEPA). - -See ``docs/optimizer.md`` for the loop in full. -""" diff --git a/miner_template/agent.py b/miner_template/agent.py index 9da4521..d1a547c 100644 --- a/miner_template/agent.py +++ b/miner_template/agent.py @@ -56,34 +56,6 @@ def log(msg: str): print(msg, file=sys.stderr) -def _load_active_prompt(round_id: int) -> dict: - """Return ``{id, template}`` for the prompt variant this round will - use, or ``{"id": "", "template": ""}`` if the miner hasn't run the - optimizer yet. Reads from ``prompts/active.json`` next to the - miner's working dir; override via ``MINER_PROMPTS_DIR``. - - Kept import-light so the sandbox doesn't need the full - ``miner_template`` package — the agent runs in a stripped image. - """ - prompts_dir = os.getenv("MINER_PROMPTS_DIR", "prompts") - path = os.path.join(prompts_dir, "active.json") - try: - with open(path) as f: - payload = json.load(f) - except (OSError, json.JSONDecodeError): - return {"id": "", "template": ""} - rows = payload.get("prompts") if isinstance(payload, dict) else payload - if not isinstance(rows, list) or not rows: - return {"id": "", "template": ""} - pick = rows[round_id % len(rows)] - if not isinstance(pick, dict): - return {"id": "", "template": ""} - return { - "id": str(pick.get("id", "")), - "template": str(pick.get("template", "")), - } - - def _pick_hyperparams(min_flops: int, max_flops: int) -> dict: """Pick d_model, nhead, num_layers, dim_feedforward to fit the FLOPs range. @@ -211,18 +183,6 @@ def design_architecture(challenge: dict, client) -> dict: # Load persistent state from previous rounds scratch_dir = load_scratchpad(challenge) - # Pull the active prompt variant for this round. If the miner is - # running the optimizer (e.g. ``miner/neuron.py optimize --watch``) - # this rotates through the current population; otherwise it's empty - # and the agent falls back to its hardcoded heuristic. - round_id = int(challenge.get("round_id", 0) or 0) - prompt = _load_active_prompt(round_id) - if prompt["id"]: - reasoning_steps.append( - f"Using prompt variant {prompt['id'][:8]}… " - f"(rotating round_id={round_id} into population)." - ) - frontier = challenge.get("feasible_frontier", []) min_flops = challenge.get("min_flops_equivalent", 0) max_flops = challenge.get("max_flops_equivalent", 0) @@ -323,5 +283,4 @@ def build_optimizer(model): "motivation": motivation, "reasoning": "\n".join(reasoning_steps), "tool_calls": tool_calls, - "prompt_id": prompt["id"], } diff --git a/miner_template/optimizers/__init__.py b/miner_template/optimizers/__init__.py deleted file mode 100644 index 73f92cb..0000000 --- a/miner_template/optimizers/__init__.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Prompt-optimizer plugin registry for miners. - -A plugin is any callable with the signature:: - - optimize(results: list[ResultRow], - population: list[Prompt], - config: dict) -> list[Prompt] - -Built-ins: - ``random_mutate`` — deterministic baseline, no LLM required (CI-safe) - ``gepa`` — DSPy GEPA adapter (Genetic-Pareto reflective evolution) - -Custom optimizers can be referenced as ``"package.module:func"`` — -e.g. ``my_research.optimizers:my_optimizer`` — and ``resolve()`` will -import them on demand. -""" - -from __future__ import annotations - -import importlib -from dataclasses import dataclass, field -from typing import Callable, Optional - -from miner_template.prompts import Prompt - -# ── Public types ───────────────────────────────────────────────────── - - -@dataclass(slots=True) -class ResultRow: - """One scored submission, as returned by ``/miners/me/results``.""" - - round_id: int - submission_id: str - task_name: str - prompt_id: Optional[str] - architecture_code: str - scores: dict = field(default_factory=dict) - created_at: str = "" - - @classmethod - def from_dict(cls, d: dict) -> "ResultRow": - return cls( - round_id=int(d.get("round_id", 0) or 0), - submission_id=str(d.get("submission_id", "") or ""), - task_name=str(d.get("task_name", "") or ""), - prompt_id=d.get("prompt_id"), - architecture_code=str(d.get("architecture_code", "") or ""), - scores=dict(d.get("scores") or {}), - created_at=str(d.get("created_at", "") or ""), - ) - - -Optimizer = Callable[[list[ResultRow], list[Prompt], dict], list[Prompt]] - - -# ── Resolver ───────────────────────────────────────────────────────── - - -_BUILTINS = { - "random_mutate": "miner_template.optimizers.random_mutate:optimize", - "gepa": "miner_template.optimizers.gepa:optimize", -} - - -def resolve(name: str) -> Optimizer: - """Return the optimizer callable for ``name``. - - Accepts a built-in alias (``"gepa"``) or a fully qualified - ``"package.module:func"`` path. - """ - target = _BUILTINS.get(name, name) - if ":" not in target: - raise ValueError( - f"unknown optimizer {name!r}; expected one of {sorted(_BUILTINS)} " - f"or 'package.module:func'" - ) - module_path, func_name = target.split(":", 1) - try: - mod = importlib.import_module(module_path) - except ImportError as e: - raise ImportError( - f"optimizer {name!r}: cannot import {module_path!r}: {e}" - ) from e - try: - func = getattr(mod, func_name) - except AttributeError: - raise AttributeError( - f"optimizer {name!r}: {module_path}:{func_name} not found" - ) - if not callable(func): - raise TypeError(f"optimizer {name!r}: {target} is not callable") - return func - - -def list_builtins() -> list[str]: - return sorted(_BUILTINS) diff --git a/miner_template/optimizers/gepa.py b/miner_template/optimizers/gepa.py deleted file mode 100644 index 0918b23..0000000 --- a/miner_template/optimizers/gepa.py +++ /dev/null @@ -1,219 +0,0 @@ -"""GEPA — Genetic-Pareto reflective prompt evolution. - -Adapter around ``dspy.GEPA``. Imports DSPy lazily so the rest of the -miner CLI runs without it installed. - -The adapter: - - 1. Group ``ResultRow`` by ``prompt_id`` to build per-prompt traces - ``(architecture_code, scores)`` for GEPA to learn from. - 2. Configure a reflector LM — either an explicit OpenAI/Anthropic- - style API (``MINER_REFLECTOR_API_*`` env vars) or the operator's - shared LLM proxy (``MINER_LLM_URL`` / ``MINER_LLM_API_KEY``). - 3. Wrap each prompt as a ``dspy.Signature`` and run ``dspy.GEPA``'s - reflective mutation loop with a Pareto-front sampler over the - score vector. - 4. Map the resulting optimized programs back to ``Prompt`` rows with - fresh UUIDs, the new generation number, and ``parent_id`` set - from GEPA's lineage. - -Config keys (passed via the ``optimize`` subcommand): - - ``population`` (int, default 8): target population size. - ``budget`` (int, default 30): GEPA compile budget (rollouts). - ``score_key`` (str, default "raw_score"): which scalar GEPA reads. - ``reflector_lm`` (dict, optional): explicit DSPy LM kwargs. When - omitted, the adapter reads ``MINER_REFLECTOR_*`` / ``MINER_LLM_*`` - env vars (see ``_default_lm()`` below). - -The score the metric returns is the float at ``scores[score_key]`` for -the matched row; missing rows score 0. Override ``score_key`` to -optimize against a different objective (e.g. ``"frontier_dominance"``). -""" - -from __future__ import annotations - -import logging -import os -from typing import Any - -from miner_template.optimizers import ResultRow -from miner_template.prompts import Prompt - -logger = logging.getLogger(__name__) - - -def _import_dspy(): - """Lazy import so the CLI works without DSPy installed.""" - try: - import dspy # type: ignore - except ImportError as e: - raise ImportError( - "GEPA optimizer requires the 'dspy' package. " - "Install with: pip install dspy" - ) from e - if not hasattr(dspy, "GEPA"): - raise ImportError( - "Installed dspy build does not expose dspy.GEPA. " - "Upgrade: pip install -U dspy" - ) - return dspy - - -def _default_lm(dspy_mod) -> Any: - """Build a DSPy LM from environment, preferring an explicit - reflector LM and falling back to the operator's LLM proxy.""" - base = os.getenv("MINER_REFLECTOR_API_BASE", "").strip() - key = os.getenv("MINER_REFLECTOR_API_KEY", "").strip() - model = os.getenv("MINER_REFLECTOR_MODEL", "").strip() - if base and key and model: - return dspy_mod.LM(model=model, api_base=base, api_key=key) - - base = os.getenv("MINER_LLM_URL", "").strip() - key = os.getenv("MINER_LLM_API_KEY", "").strip() - model = os.getenv("MINER_LLM_MODEL", "").strip() or "deepseek-ai/DeepSeek-V3-0324" - if base and key: - return dspy_mod.LM(model=model, api_base=base, api_key=key) - - raise RuntimeError( - "GEPA reflector LM not configured. Set either " - "MINER_REFLECTOR_{API_BASE,API_KEY,MODEL} or " - "MINER_LLM_{URL,API_KEY,MODEL}." - ) - - -def _group_by_prompt(results: list[ResultRow]) -> dict[str, list[ResultRow]]: - groups: dict[str, list[ResultRow]] = {} - for r in results: - if r.prompt_id: - groups.setdefault(r.prompt_id, []).append(r) - return groups - - -def _build_metric(score_key: str): - """Return a metric callable GEPA invokes per rollout. - - The default DSPy metric signature is ``metric(example, pred, trace) - -> float``; we don't need the example/trace here because we already - know which prompt produced which scored row. GEPA passes through - user-supplied kwargs, so we attach ``score_key`` as a closure. - """ - - def metric(example, pred, trace=None) -> float: - # GEPA stores result metadata on ``pred`` when wrapped through - # this adapter; the caller's program puts ``scores`` there. - scores = getattr(pred, "scores", None) or {} - try: - return float(scores.get(score_key, 0.0) or 0.0) - except (TypeError, ValueError): - return 0.0 - - return metric - - -def _to_example(dspy_mod, row: ResultRow, score_key: str): - """Wrap a ResultRow as a DSPy example GEPA can iterate over.""" - return dspy_mod.Example( - challenge=row.scores.get("challenge", ""), - architecture=row.architecture_code, - score=float(row.scores.get(score_key, 0.0) or 0.0), - ).with_inputs("challenge") - - -def optimize( - results: list[ResultRow], - population: list[Prompt], - config: dict, -) -> list[Prompt]: - """Run GEPA on the population. - - See module docstring for ``config`` keys. Returns a new population - of size ``config['population']`` (or the input size if unset). - """ - dspy_mod = _import_dspy() - - if not population: - return [] - - target = int(config.get("population", len(population) or 8)) - budget = int(config.get("budget", 30)) - score_key = str(config.get("score_key", "raw_score")) - - # Configure reflector LM. - lm_kwargs = config.get("reflector_lm") - if isinstance(lm_kwargs, dict): - lm = dspy_mod.LM(**lm_kwargs) - else: - lm = _default_lm(dspy_mod) - dspy_mod.configure(lm=lm) - - # Build the training set from scored rows. - examples = [_to_example(dspy_mod, r, score_key) - for r in results if r.prompt_id] - if not examples: - logger.warning( - "GEPA: no scored examples — returning population unchanged." - ) - return population[:target] - - next_gen = max(p.generation for p in population) + 1 - - # GEPA optimizes a DSPy program; each program is a ``Predict`` over - # the same ``Signature`` but with the prompt template as its - # instruction. We seed the search from every active variant so the - # current population is preserved unless GEPA finds something - # strictly better. - class _ArchSignature(dspy_mod.Signature): - """Design a model architecture for the given challenge.""" - - challenge: str = dspy_mod.InputField() - architecture: str = dspy_mod.OutputField() - - programs = [] - for p in population: - prog = dspy_mod.Predict(_ArchSignature) - prog.signature = prog.signature.with_instructions(p.template) - programs.append((p, prog)) - - metric = _build_metric(score_key) - teleprompter = dspy_mod.GEPA(metric=metric, num_candidates=target, budget=budget) - - # GEPA compiles each seed program; we keep the best ``target`` - # programs by their reported score and map them back to Prompt rows. - optimized: list[tuple[Prompt, Any]] = [] - for parent, prog in programs: - try: - compiled = teleprompter.compile(prog, trainset=examples) - except Exception as e: - logger.warning( - "GEPA: compile failed for prompt %s: %s — keeping parent", - parent.id, e, - ) - optimized.append((parent, prog)) - continue - new_instruction = _extract_instruction(compiled) - optimized.append(( - Prompt.new( - template=new_instruction or parent.template, - generation=next_gen, - parent_id=parent.id, - metadata={"mutation": "gepa", "budget": budget}, - ), - compiled, - )) - - new_pop = [p for p, _ in optimized][:target] - while len(new_pop) < target: - new_pop.append(population[len(new_pop) % len(population)]) - return new_pop - - -def _extract_instruction(program) -> str: - """Pull the optimized instruction string off a DSPy program.""" - sig = getattr(program, "signature", None) - if sig is None: - return "" - instructions = getattr(sig, "instructions", "") - if isinstance(instructions, str): - return instructions - return str(instructions or "") diff --git a/miner_template/optimizers/random_mutate.py b/miner_template/optimizers/random_mutate.py deleted file mode 100644 index 3ca71c8..0000000 --- a/miner_template/optimizers/random_mutate.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Baseline optimizer: rank prompts by mean score and clone+perturb winners. - -No LLM, no DSPy dependency — used as a CI-safe default and as a -sanity-check sibling to GEPA. Useful when a miner wants to bootstrap -a population without an LLM cost on day one. - -Strategy:: - - 1. group ResultRow by prompt_id; compute mean score per prompt - 2. promote the top-K to the new population - 3. for each survivor, generate (population_size - K) // K children - by appending a deterministic perturbation suffix - 4. if no scores yet, simply clone the active population -""" - -from __future__ import annotations - -import hashlib -from typing import Iterable - -from miner_template.optimizers import ResultRow -from miner_template.prompts import Prompt - -# Cheap deterministic prompt perturbations. Picked by hashing the -# parent's id + child index so two runs on the same inputs produce the -# same children — important for reproducibility. -_PERTURBATIONS: list[str] = [ - "\n\nThink step by step before answering.", - "\n\nFavor architectures with explicit attention sparsity.", - "\n\nPrefer designs that reuse parameters across layers.", - "\n\nConsider hybrid CNN+attention encoders.", - "\n\nWhen in doubt, fall back to a small transformer baseline.", - "\n\nReport hyperparameter choices in the motivation field.", - "\n\nOptimize for sample efficiency over raw capacity.", - "\n\nAvoid recurrent layers unless justified by data length.", -] - - -def _mean_score(rows: Iterable[ResultRow], key: str = "raw_score") -> float: - vals = [float(r.scores.get(key, 0.0) or 0.0) for r in rows] - if not vals: - return 0.0 - return sum(vals) / len(vals) - - -def _rank_population( - results: list[ResultRow], - population: list[Prompt], - score_key: str, -) -> list[tuple[Prompt, float]]: - """Return ``(prompt, mean_score)`` sorted best-first. Prompts with - no observed rows fall to the bottom but stay in the ranking.""" - by_pid: dict[str, list[ResultRow]] = {} - for r in results: - if r.prompt_id: - by_pid.setdefault(r.prompt_id, []).append(r) - scored: list[tuple[Prompt, float]] = [] - for p in population: - scored.append((p, _mean_score(by_pid.get(p.id, []), score_key))) - scored.sort(key=lambda t: t[1], reverse=True) - return scored - - -def _perturb(parent: Prompt, child_idx: int, generation: int) -> Prompt: - seed = hashlib.sha256( - f"{parent.id}:{child_idx}".encode() - ).digest() - idx = seed[0] % len(_PERTURBATIONS) - return Prompt.new( - template=parent.template + _PERTURBATIONS[idx], - generation=generation, - parent_id=parent.id, - metadata={ - "mutation": "random_mutate", - "perturbation_idx": idx, - }, - ) - - -def optimize( - results: list[ResultRow], - population: list[Prompt], - config: dict, -) -> list[Prompt]: - """Return the next population. - - Config keys: - ``population`` (int, default 8): target size of the new population. - ``elite_k`` (int, default 2): how many top performers to carry. - ``score_key`` (str, default "raw_score"): scores field to rank on. - """ - target = int(config.get("population", 8)) - elite_k = int(config.get("elite_k", 2)) - score_key = str(config.get("score_key", "raw_score")) - if target < 1: - raise ValueError("population must be >= 1") - if elite_k < 1: - elite_k = 1 - - if not population: - return [] # caller should seed_default() before optimizing - - next_gen = max(p.generation for p in population) + 1 - - ranked = _rank_population(results, population, score_key) - elites = [p for p, _ in ranked[:elite_k]] - - new_pop: list[Prompt] = list(elites) - child_idx = 0 - while len(new_pop) < target: - parent = elites[child_idx % len(elites)] - new_pop.append(_perturb(parent, child_idx, next_gen)) - child_idx += 1 - - return new_pop[:target] diff --git a/miner_template/prompts.py b/miner_template/prompts.py deleted file mode 100644 index 95e55c3..0000000 --- a/miner_template/prompts.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Local prompt-population store for miners. - -A *population* is a small list of ``Prompt`` rows kept on disk under -``prompts/`` next to the miner's working directory: - - prompts/ - active.json # current population the agent reads - history/ - gen_001.json # archived populations - gen_002.json - ... - -Atomic writes use ``tmp + os.replace`` so an interrupted optimizer -never leaves the agent with a half-written file. - -The agent reads the active population each round via ``load_active()`` -and picks one variant with ``pick_for_round(pop, round_id)``. The -chosen prompt's ``id`` round-trips back via the submission so the -optimizer can correlate Phase C scores with prompt variants. -""" - -from __future__ import annotations - -import dataclasses -import json -import logging -import os -import uuid -from dataclasses import dataclass, field -from pathlib import Path -from typing import Iterable, Optional - -logger = logging.getLogger(__name__) - -DEFAULT_PROMPTS_DIR = Path(os.getenv("MINER_PROMPTS_DIR", "prompts")) -ACTIVE_FILE = "active.json" -HISTORY_DIR = "history" - - -@dataclass -class Prompt: - """A single prompt variant in the population.""" - - id: str - template: str - generation: int = 0 - parent_id: Optional[str] = None - metadata: dict = field(default_factory=dict) - - @classmethod - def new(cls, template: str, *, generation: int = 0, - parent_id: Optional[str] = None, - metadata: Optional[dict] = None) -> "Prompt": - return cls( - id=uuid.uuid4().hex, - template=template, - generation=generation, - parent_id=parent_id, - metadata=dict(metadata or {}), - ) - - def to_dict(self) -> dict: - return dataclasses.asdict(self) - - @classmethod - def from_dict(cls, d: dict) -> "Prompt": - return cls( - id=str(d.get("id") or uuid.uuid4().hex), - template=str(d.get("template", "")), - generation=int(d.get("generation", 0)), - parent_id=d.get("parent_id"), - metadata=dict(d.get("metadata") or {}), - ) - - -# ── Filesystem layout helpers ──────────────────────────────────────── - - -def _root(prompts_dir: Optional[Path] = None) -> Path: - return Path(prompts_dir or DEFAULT_PROMPTS_DIR) - - -def _active_path(prompts_dir: Optional[Path] = None) -> Path: - return _root(prompts_dir) / ACTIVE_FILE - - -def _history_dir(prompts_dir: Optional[Path] = None) -> Path: - return _root(prompts_dir) / HISTORY_DIR - - -def _atomic_write_json(path: Path, payload) -> None: - """Write ``payload`` to ``path`` via ``.tmp`` + ``os.replace`` so the - reader never sees a partial file.""" - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_suffix(path.suffix + ".tmp") - with tmp.open("w") as f: - json.dump(payload, f, indent=2, sort_keys=True) - f.flush() - os.fsync(f.fileno()) - os.replace(tmp, path) - - -# ── Active-population I/O ──────────────────────────────────────────── - - -def load_active(prompts_dir: Optional[Path] = None) -> list[Prompt]: - """Read the active population. Returns ``[]`` if the file doesn't - exist yet (fresh miner) — callers should treat this as "seed me".""" - path = _active_path(prompts_dir) - if not path.exists(): - return [] - try: - payload = json.loads(path.read_text()) - except (OSError, json.JSONDecodeError) as e: - logger.warning("prompts: active.json unreadable (%s) — empty pop", e) - return [] - rows = payload.get("prompts") if isinstance(payload, dict) else payload - if not isinstance(rows, list): - return [] - return [Prompt.from_dict(r) for r in rows if isinstance(r, dict)] - - -def save_active(prompts: Iterable[Prompt], - prompts_dir: Optional[Path] = None) -> None: - """Atomically write the active population.""" - items = [p.to_dict() for p in prompts] - payload = {"prompts": items} - _atomic_write_json(_active_path(prompts_dir), payload) - - -def archive_current(generation: int, - prompts_dir: Optional[Path] = None) -> Optional[Path]: - """Copy the current ``active.json`` into ``history/gen_NNN.json``. - - Returns the destination path, or ``None`` if there's no active file - yet (first generation — nothing to archive). - """ - src = _active_path(prompts_dir) - if not src.exists(): - return None - dst = _history_dir(prompts_dir) / f"gen_{generation:03d}.json" - dst.parent.mkdir(parents=True, exist_ok=True) - dst.write_bytes(src.read_bytes()) - return dst - - -def list_history(prompts_dir: Optional[Path] = None) -> list[int]: - """Return archived generation numbers, sorted ascending.""" - d = _history_dir(prompts_dir) - if not d.exists(): - return [] - gens: list[int] = [] - for f in d.iterdir(): - if not f.name.startswith("gen_") or not f.name.endswith(".json"): - continue - try: - gens.append(int(f.stem.split("_", 1)[1])) - except (IndexError, ValueError): - continue - return sorted(gens) - - -def load_generation(generation: int, - prompts_dir: Optional[Path] = None) -> list[Prompt]: - """Read an archived generation back into memory.""" - path = _history_dir(prompts_dir) / f"gen_{generation:03d}.json" - if not path.exists(): - raise FileNotFoundError(f"no archived generation {generation}") - payload = json.loads(path.read_text()) - rows = payload.get("prompts") if isinstance(payload, dict) else payload - return [Prompt.from_dict(r) for r in rows if isinstance(r, dict)] - - -def rollback_to(generation: int, - prompts_dir: Optional[Path] = None) -> None: - """Restore an archived generation as the new active population. - - The currently active population is first archived under the next - free generation number, so rollback is reversible. - """ - pop = load_generation(generation, prompts_dir) - next_gen = (list_history(prompts_dir) or [0])[-1] + 1 - archive_current(next_gen, prompts_dir) - save_active(pop, prompts_dir) - - -# ── Selection ──────────────────────────────────────────────────────── - - -def pick_for_round(population: list[Prompt], round_id: int) -> Prompt: - """Deterministic round-robin over the population. - - Same ``round_id`` always picks the same prompt — so explore/exploit - looks the same to anyone replaying the round. Empty population - raises ``ValueError``; callers should ``seed_default()`` first. - """ - if not population: - raise ValueError("empty population — call seed_default() first") - return population[round_id % len(population)] - - -# ── Seeding ────────────────────────────────────────────────────────── - - -DEFAULT_SEED_TEMPLATE = ( - "You are a model-architecture researcher designing a PyTorch model " - "for time-series forecasting.\n" - "Round budget: FLOPs in [{min_flops}, {max_flops}].\n" - "Frontier so far (best metric per architecture):\n" - "{frontier}\n\n" - "Design a single architecture targeting ~60% of the FLOPs ceiling. " - "Return the model code, a short name, and a one-line motivation." -) - - -def seed_default(prompts_dir: Optional[Path] = None, - template: str = DEFAULT_SEED_TEMPLATE) -> list[Prompt]: - """Write a single starter prompt as the active population. No-op - if ``active.json`` already exists. Returns the resulting active - population either way. - """ - existing = load_active(prompts_dir) - if existing: - return existing - seed = Prompt.new(template=template, generation=0) - save_active([seed], prompts_dir) - return [seed] diff --git a/miner_template/results_client.py b/miner_template/results_client.py deleted file mode 100644 index 298bb08..0000000 --- a/miner_template/results_client.py +++ /dev/null @@ -1,127 +0,0 @@ -"""HTTP client for the miner-feedback endpoints on the database server. - -The DB exposes a small set of bearer-auth'd routes under ``/miners/me/*`` -that let a miner pull its own scored history without going through -the validator. This is the feedback channel the optimizer reads. - - GET /miners/me/submissions — Phase A submissions made by this miner - GET /miners/me/results — submissions joined with Phase C scores - GET /miners/me/summary — counts + rolling-window stats - GET /tasks/{task}/frontier — public per-task Pareto frontier -""" - -from __future__ import annotations - -import logging -from typing import Optional - -import httpx - -from miner_template.optimizers import ResultRow - -logger = logging.getLogger(__name__) - -DEFAULT_TIMEOUT = 30.0 - - -class MinerResultsClient: - """Pull this miner's scored history from the central DB.""" - - def __init__( - self, - db_url: str, - api_key: str, - *, - timeout: float = DEFAULT_TIMEOUT, - client: Optional[httpx.Client] = None, - ): - if not db_url: - raise ValueError("db_url required") - if not api_key: - raise ValueError("api_key required") - self.db_url = db_url.rstrip("/") - self.api_key = api_key - self.timeout = timeout - self._owned_client = client is None - self._client = client or httpx.Client(timeout=timeout) - - # ── Context manager ───────────────────────────────────────────── - - def __enter__(self): - return self - - def __exit__(self, *exc): - self.close() - - def close(self) -> None: - if self._owned_client and not self._client.is_closed: - self._client.close() - - # ── Internal ──────────────────────────────────────────────────── - - def _headers(self) -> dict[str, str]: - return {"Authorization": f"Bearer {self.api_key}"} - - def _get(self, path: str, params: Optional[dict] = None): - resp = self._client.get( - f"{self.db_url}{path}", params=params, headers=self._headers(), - ) - resp.raise_for_status() - return resp.json() - - # ── Public ────────────────────────────────────────────────────── - - def submissions( - self, - *, - since: str = "", - limit: int = 200, - task: str = "", - ) -> list[dict]: - """Phase A submissions made by this miner. Returns raw dicts — - the optimizer doesn't need these; this is for the CLI's - ``results`` view + custom miner tooling.""" - params: dict = {"limit": limit} - if since: - params["since"] = since - if task: - params["task"] = task - payload = self._get("/miners/me/submissions", params=params) - if isinstance(payload, dict): - return list(payload.get("submissions", []) or []) - return list(payload or []) - - def results( - self, - *, - since: str = "", - limit: int = 200, - task: str = "", - ) -> list[ResultRow]: - """Scored history — submissions joined with Phase C results. - Returned as ``ResultRow`` so the optimizer can consume directly.""" - params: dict = {"limit": limit} - if since: - params["since"] = since - if task: - params["task"] = task - payload = self._get("/miners/me/results", params=params) - rows = payload.get("results") if isinstance(payload, dict) else payload - return [ResultRow.from_dict(r) for r in (rows or []) if isinstance(r, dict)] - - def summary(self) -> dict: - """Rolling stats — total submissions, last round id, mean score - over the recent window, etc. Useful for ``--watch`` to detect - when a new round of results is available.""" - payload = self._get("/miners/me/summary") - return payload if isinstance(payload, dict) else {} - - def frontier(self, task: str) -> list[dict]: - """Public per-task frontier (no auth required, but we send the - bearer anyway — harmless).""" - if not task: - raise ValueError("task required") - payload = self._get(f"/tasks/{task}/frontier") - if isinstance(payload, dict): - return list(payload.get("points", []) or []) - return list(payload or []) diff --git a/runner/server.py b/runner/server.py index 7db912d..ced57ef 100644 --- a/runner/server.py +++ b/runner/server.py @@ -19,6 +19,8 @@ import os import threading import time +from contextlib import asynccontextmanager + from fastapi import FastAPI, Request from fastapi.responses import JSONResponse @@ -36,24 +38,32 @@ logger = logging.getLogger(__name__) -app = FastAPI(title="Radar Trainer") - -# ── Auth (shared HMAC service key) ─────────────────────────────────── -# In non-competitive mode there's exactly one issuer (the operator), so -# the trainer just verifies HMAC against ``RADAR_SERVICE_KEY``. No -# metagraph fetch, no bittensor import. +@asynccontextmanager +async def _lifespan(app: FastAPI): + """Preload metagraph at startup so /train doesn't block on first request.""" + localnet = os.getenv("RADAR_LOCALNET", "").lower() in ("1", "true") + if not localnet: + logger.info("Preloading metagraph at startup...") + mg = await asyncio.to_thread(_load_metagraph) + if mg is not None: + logger.info("Metagraph preloaded (%d neurons)", mg.n) + else: + logger.warning("Metagraph preload failed — will retry on first request") + yield -def _service_secret() -> bytes | None: - raw = os.getenv("RADAR_SERVICE_KEY", "").strip() - return raw.encode() if raw else None +app = FastAPI(title="Radar Trainer", lifespan=_lifespan) -_EXPECTED_KEY_ID = os.getenv("RADAR_SERVICE_KEY_ID", "operator") +# ── Metagraph cache ────────────────────────────────────────────────── +_metagraph_cache = None +_metagraph_lock = threading.Lock() +_metagraph_last_refresh: float = 0.0 +METAGRAPH_REFRESH_INTERVAL = float(os.getenv("METAGRAPH_REFRESH_INTERVAL", "300")) -# ── Rate limiting (per key_id) ─────────────────────────────────────── +# ── Rate limiting ──────────────────────────────────────────────────── _train_semaphore = asyncio.Semaphore(1) # max 1 concurrent /train -_key_last_request: dict[str, float] = {} -_key_lock = threading.Lock() +_hotkey_last_request: dict[str, float] = {} +_hotkey_lock = threading.Lock() HOTKEY_COOLDOWN_SECONDS = float(os.getenv("TRAINER_HOTKEY_COOLDOWN", "60")) # ── Task registry ─────────────────────────────────────────────────── @@ -79,38 +89,32 @@ async def train(request: Request): body = await request.body() - # 1. Auth — FAIL CLOSED (HMAC service key) + # 1. Auth — FAIL CLOSED localnet = os.getenv("RADAR_LOCALNET", "").lower() in ("1", "true") if localnet: logger.info("Localnet mode: skipping auth") sender = "localnet" else: - secret = _service_secret() - if not secret: - logger.error("RADAR_SERVICE_KEY not set — rejecting (fail closed)") - return JSONResponse( - status_code=503, - content={"error": "Auth not configured on trainer"}, - ) + metagraph = _load_metagraph() + if metagraph is None: + logger.error("Metagraph unavailable — rejecting request (fail closed)") + return JSONResponse(status_code=503, content={"error": "Auth unavailable, try again later"}) try: - from shared.auth import hmac_verify_request, static_key_lookup - ok, err, sender = hmac_verify_request( - dict(request.headers), body, - static_key_lookup(_EXPECTED_KEY_ID, secret), - ) + from shared.auth import verify_request + ok, err, sender = verify_request(dict(request.headers), body, metagraph, require_stake=True) if not ok: - kid = request.headers.get("x-radar-key-id", "?") - logger.warning("Auth rejected (key_id=%s): %s", kid, err) + signed_by = request.headers.get("x-epistula-signed-by", "?") + logger.warning( + "Auth rejected from %s: %s", signed_by, err, + ) return JSONResponse(status_code=403, content={"error": err}) except Exception as e: logger.error("Auth verification failed: %s", e) - return JSONResponse( - status_code=403, content={"error": "Auth verification error"}, - ) + return JSONResponse(status_code=403, content={"error": "Auth verification error"}) - # 2. Per-key rate limit (check only — timestamp recorded after semaphore) - with _key_lock: - last = _key_last_request.get(sender, 0.0) + # 2. Per-hotkey rate limit (check only — timestamp recorded after semaphore) + with _hotkey_lock: + last = _hotkey_last_request.get(sender, 0.0) if time.time() - last < HOTKEY_COOLDOWN_SECONDS: remaining = HOTKEY_COOLDOWN_SECONDS - (time.time() - last) return JSONResponse(status_code=429, content={ @@ -151,8 +155,8 @@ async def train(request: Request): # Record rate-limit timestamp AFTER semaphore acquired — if the # request fails before this point, retries won't be rate-limited. - with _key_lock: - _key_last_request[sender] = time.time() + with _hotkey_lock: + _hotkey_last_request[sender] = time.time() training_config = { "seed": data.get("seed", 42), @@ -311,6 +315,33 @@ async def boot_proof(): return body +def _load_metagraph(): + """Load metagraph with caching. Returns None if never reachable.""" + global _metagraph_cache, _metagraph_last_refresh + now = time.time() + if _metagraph_cache is not None and (now - _metagraph_last_refresh) < METAGRAPH_REFRESH_INTERVAL: + return _metagraph_cache + with _metagraph_lock: + if _metagraph_cache is not None and (now - _metagraph_last_refresh) < METAGRAPH_REFRESH_INTERVAL: + return _metagraph_cache + try: + import bittensor as bt + netuid = int(os.getenv("NETUID", "1")) + network = os.getenv("SUBTENSOR_NETWORK", "finney") + subtensor = bt.Subtensor(network=network) + mg = subtensor.metagraph(netuid) + _metagraph_cache = mg + _metagraph_last_refresh = time.time() + logger.info("Metagraph refreshed (%d neurons)", mg.n) + return mg + except Exception as e: + logger.warning("Metagraph refresh failed: %s", e) + if _metagraph_cache is not None: + logger.info("Using stale metagraph cache") + return _metagraph_cache + return None + + if __name__ == "__main__": import uvicorn # Configure the application root logger so INFO messages from diff --git a/shared/auth.py b/shared/auth.py index 4a6ce3f..258cf86 100644 --- a/shared/auth.py +++ b/shared/auth.py @@ -1,50 +1,35 @@ -"""Auth primitives — both legacy Epistula and new HMAC/bearer surfaces. +"""Epistula authentication — sign and verify HTTP requests. -Two coexisting auth schemes during the non-competitive cutover: - - Legacy (Epistula / SR25519, metagraph-backed) - sign_request(wallet, body) -> headers - verify_request(headers, body, mg, …) -> (ok, error, hotkey) - - New (HMAC service-key + bearer API tokens) - hmac_sign_request(secret, body, kid) -> headers - hmac_verify_request(headers, body, lk) -> (ok, error, key_id) - extract_bearer(headers) -> token | None - hash_api_key(key) -> sha256 hex (for DB storage) - -The HMAC scheme signs ``f"{timestamp}.{sha256(body).hex()}"`` with -HMAC-SHA256. ``key_id`` is sent in cleartext so the verifier can look -up the right secret; secrets themselves are never on the wire. - -Once every caller is on the new surface we'll drop the Epistula -functions and the ``bittensor`` import below. +Extracted from validator/gossip.py. Used by validators, trainers, and miners +for authenticating HTTP requests with SR25519 hotkey signatures. """ from __future__ import annotations -import hashlib -import hmac import logging import os import time import uuid -from typing import Callable, Optional +from typing import Optional -logger = logging.getLogger(__name__) +import bittensor as bt -# ── Legacy Epistula ────────────────────────────────────────────────── +logger = logging.getLogger(__name__) # Epistula timestamp tolerance (seconds). Cross-machine clock skew between # validator hosts and miner trainer pods is the most common cause of 403s -# here, so the default is intentionally generous. +# here, so the default is intentionally generous. Replay protection still +# relies on the SR25519 signature + uuid4 nonce; the timestamp window only +# bounds how long a captured signed message can be replayed. EPISTULA_TIMESTAMP_TOLERANCE = int(os.getenv("RADAR_EPISTULA_TOLERANCE", "120")) -def sign_request(wallet, body: bytes) -> dict[str, str]: - """Sign an outgoing HTTP request with Epistula headers (legacy).""" - import bittensor as bt # local import keeps non-chain processes light +def sign_request(wallet: bt.Wallet, body: bytes) -> dict[str, str]: + """Sign an outgoing HTTP request with Epistula headers. - _ = bt # silence linter + Headers: X-Epistula-Signed-By, X-Epistula-Timestamp, + X-Epistula-Nonce, X-Epistula-Signature + """ timestamp = str(int(time.time())) nonce = uuid.uuid4().hex message = body + timestamp.encode() + nonce.encode() @@ -63,9 +48,10 @@ def verify_request( metagraph, require_stake: bool = False, ) -> tuple[bool, str, str]: - """Verify Epistula headers on an inbound request (legacy).""" - import bittensor as bt + """Verify an incoming request's Epistula headers. + Returns (ok, error_message, sender_hotkey). + """ hotkey = headers.get("x-epistula-signed-by", "") timestamp_str = headers.get("x-epistula-timestamp", "") nonce = headers.get("x-epistula-nonce", "") @@ -74,18 +60,24 @@ def verify_request( if not all([hotkey, timestamp_str, nonce, signature]): return False, "Missing Epistula headers", "" + # Check timestamp freshness try: ts = int(timestamp_str) except ValueError: return False, "Invalid timestamp", "" skew = int(time.time() - ts) if abs(skew) > EPISTULA_TIMESTAMP_TOLERANCE: + # Surface the actual skew + tolerance so miner operators can see at + # a glance that this is a clock-sync problem (NTP, container drift) + # rather than a signing bug, and tune RADAR_EPISTULA_TOLERANCE if + # needed. err = ( f"Timestamp too old or too far in future " f"(skew={skew}s, tolerance=±{EPISTULA_TIMESTAMP_TOLERANCE}s)" ) return False, err, "" + # Verify sender is registered uid = get_uid_for_hotkey(metagraph, hotkey) if uid is None: return False, "Unknown hotkey", "" @@ -93,6 +85,7 @@ def verify_request( if require_stake and float(metagraph.S[uid]) <= 0: return False, "Not a validator (no stake)", "" + # Verify signature message = body + timestamp_str.encode() + nonce.encode() try: keypair = bt.Keypair(ss58_address=hotkey) @@ -105,6 +98,7 @@ def verify_request( def get_uid_for_hotkey(metagraph, hotkey: str) -> Optional[int]: + """Look up UID for a hotkey in the metagraph.""" hotkeys = metagraph.hotkeys if hotkeys is None: return None @@ -112,131 +106,3 @@ def get_uid_for_hotkey(metagraph, hotkey: str) -> Optional[int]: if i < len(hotkeys) and hotkeys[i] == hotkey: return i return None - - -# ── New HMAC primitives ────────────────────────────────────────────── - -# How far the wire timestamp may drift from the verifier's clock. -HMAC_TIMESTAMP_TOLERANCE = int(os.getenv("RADAR_HMAC_TOLERANCE", "300")) - -DEFAULT_KEY_ID = "operator" - -H_SIG = "x-radar-signature" -H_TS = "x-radar-timestamp" -H_KID = "x-radar-key-id" -H_AUTH = "authorization" - -KeyLookup = Callable[[str], Optional[bytes]] - - -def _canonical(body: bytes, timestamp: str) -> bytes: - body_digest = hashlib.sha256(body).hexdigest() - return f"{timestamp}.{body_digest}".encode() - - -def hmac_sign_request( - secret: bytes, - body: bytes, - key_id: str = DEFAULT_KEY_ID, -) -> dict[str, str]: - """HMAC-SHA256 sign a request. Caller attaches the returned headers - to its outbound HTTP request.""" - if not isinstance(secret, (bytes, bytearray)): - raise TypeError("secret must be bytes") - timestamp = str(int(time.time())) - sig = hmac.new(secret, _canonical(body, timestamp), hashlib.sha256).hexdigest() - return { - "X-Radar-Signature": sig, - "X-Radar-Timestamp": timestamp, - "X-Radar-Key-Id": key_id, - } - - -def hmac_verify_request( - headers: dict[str, str], - body: bytes, - key_lookup: KeyLookup, - tolerance_s: int = HMAC_TIMESTAMP_TOLERANCE, -) -> tuple[bool, str, str]: - """Verify HMAC headers on an inbound request. - - ``key_lookup`` resolves ``key_id -> secret``. Returning ``None`` - fails verification closed. Returns ``(ok, error, key_id)`` with - ``key_id`` empty on failure. - """ - h = {k.lower(): v for k, v in headers.items()} - sig = h.get(H_SIG, "") - ts = h.get(H_TS, "") - kid = h.get(H_KID, "") - - if not sig or not ts or not kid: - return False, "missing HMAC headers", "" - - try: - ts_int = int(ts) - except ValueError: - return False, "invalid timestamp", "" - skew = int(time.time()) - ts_int - if abs(skew) > tolerance_s: - return False, ( - f"timestamp out of tolerance (skew={skew}s, " - f"tolerance=±{tolerance_s}s)" - ), "" - - secret = key_lookup(kid) - if secret is None: - return False, "unknown key_id", "" - - expected = hmac.new(secret, _canonical(body, ts), hashlib.sha256).hexdigest() - if not hmac.compare_digest(expected, sig): - return False, "bad signature", "" - - return True, "", kid - - -def static_key_lookup(key_id: str, secret: bytes) -> KeyLookup: - """Build a ``key_lookup`` that accepts exactly one ``(key_id, secret)`` - — the common case for trainer-side service auth.""" - expected = key_id - - def lookup(kid: str) -> Optional[bytes]: - if kid == expected: - return secret - return None - - return lookup - - -# ── Bearer tokens (per-agent / per-miner API keys) ─────────────────── - - -def extract_bearer(headers: dict[str, str]) -> Optional[str]: - """Return the bearer token from an ``Authorization`` header, or - ``None`` if missing/malformed.""" - h = {k.lower(): v for k, v in headers.items()} - raw = h.get(H_AUTH, "") - if not raw: - return None - parts = raw.split(None, 1) - if len(parts) != 2 or parts[0].lower() != "bearer": - return None - token = parts[1].strip() - return token or None - - -def hash_api_key(key: str) -> str: - """SHA-256 hex digest. The DB stores this — never the raw key.""" - return hashlib.sha256(key.encode()).hexdigest() - - -def load_service_secret(env: str = "RADAR_SERVICE_KEY") -> bytes: - """Read the shared HMAC service key from the environment. Raises - ``RuntimeError`` if unset — callers that need fail-closed behavior - should let this propagate at startup.""" - raw = os.getenv(env, "").strip() - if not raw: - raise RuntimeError( - f"{env} is unset — set it to a long random string shared " - f"between operator, validator, and trainer processes." - ) - return raw.encode() diff --git a/shared/database.py b/shared/database.py index a08d262..e8ff3f6 100644 --- a/shared/database.py +++ b/shared/database.py @@ -56,11 +56,6 @@ class DataElement: # entries written by validators after Phase C. One per validator that # published a Hippius bundle covering this experiment. substrate_cids: list = field(default_factory=list) - # Miner-local prompt-variant id (from Proposal.prompt_id). Opaque to - # the operator; round-tripped so the miner's GEPA/random_mutate loop - # can correlate Phase C scores with the variant that produced this - # architecture. Empty string when the miner submitted without one. - prompt_id: str = "" def to_dict(self) -> dict: return asdict(self) diff --git a/shared/db_client.py b/shared/db_client.py index b3d6739..dd6f4c3 100644 --- a/shared/db_client.py +++ b/shared/db_client.py @@ -1,55 +1,28 @@ -"""HTTP client for validators / miners to talk to the centralized DB. - -Two auth modes selected at construction time: - - * Service HMAC — passes ``service_secret`` (bytes). Outbound requests - carry X-Radar-{Signature, Timestamp, Key-Id} headers. Used by the - validator dispatcher and any other operator-side process holding - the shared service key. - * Bearer token — passes ``api_key`` (str). Outbound requests carry - ``Authorization: Bearer ``. Used by miner-side tooling - (``MinerResultsClient`` lives in ``miner_template`` for the same - surface). - -Legacy Epistula path (``wallet`` keyword) is retained as a back-compat -shim during the cutover. When ``wallet`` is supplied and no HMAC/key -is set, requests are signed with the on-chain hotkey via -``shared.auth.sign_request``. Pass exactly ONE of the three. +"""HTTP client for validators to talk to the centralized database server. + +Used for writes (POST experiments, provenance) and frontier fetches. +All methods sign requests with Epistula via the validator's wallet. """ from __future__ import annotations -import json import logging from typing import Optional import httpx +from shared.auth import sign_request + logger = logging.getLogger(__name__) class DatabaseClient: """Async HTTP client for the centralized database API.""" - def __init__( - self, - db_url: str, - wallet=None, - *, - service_secret: Optional[bytes] = None, - key_id: str = "operator", - api_key: str = "", - ): + def __init__(self, db_url: str, wallet, api_key: str = ""): self.db_url = db_url.rstrip("/") self.wallet = wallet - self.service_secret = service_secret - self.key_id = key_id self.api_key = api_key or "" - if not any([self.service_secret, self.api_key, self.wallet]): - raise ValueError( - "DatabaseClient needs one of service_secret (HMAC), " - "api_key (bearer), or wallet (legacy Epistula).", - ) self._client: Optional[httpx.AsyncClient] = None async def _get_client(self) -> httpx.AsyncClient: @@ -58,20 +31,16 @@ async def _get_client(self) -> httpx.AsyncClient: return self._client def _sign(self, body: bytes) -> dict[str, str]: - if self.service_secret: - from shared.auth import hmac_sign_request - return hmac_sign_request( - self.service_secret, body, key_id=self.key_id, - ) + headers = sign_request(self.wallet, body) if self.api_key: - return {"Authorization": f"Bearer {self.api_key}"} - # Legacy: wallet (Epistula). - from shared.auth import sign_request - return sign_request(self.wallet, body) + headers["X-Radar-API-Key"] = self.api_key + return headers async def _post(self, path: str, json_data: dict) -> Optional[dict]: + """POST with Epistula signing. Returns JSON response or None.""" try: client = await self._get_client() + import json body = json.dumps(json_data).encode() headers = self._sign(body) headers["Content-Type"] = "application/json" @@ -92,6 +61,7 @@ async def _post(self, path: str, json_data: dict) -> Optional[dict]: return None async def _get(self, path: str, params: Optional[dict] = None) -> Optional[dict]: + """GET with Epistula signing. Returns JSON response or None.""" try: client = await self._get_client() headers = self._sign(b"") @@ -114,6 +84,7 @@ async def _get(self, path: str, params: Optional[dict] = None) -> Optional[dict] # ── Public API ─────────────────────────────────────── async def health(self) -> bool: + """Check database server health.""" try: client = await self._get_client() resp = await client.get(f"{self.db_url}/health") @@ -128,9 +99,16 @@ async def add_experiment( validator_hotkey: str = "", artifact_cids: Optional[list[dict]] = None, ) -> Optional[int]: - """POST a DataElement dict. ``validator_hotkey`` is the legacy - name for the writer's identifier — operator dispatchers in - non-competitive mode pass an opaque operator_id here. + """POST a DataElement dict to the database. Returns new index or None. + + When ``substrate_cid`` is non-empty the server appends a + ``{kind: "phase_c_record", validator_hotkey, cid, round_id}`` entry + to the row's ``substrate_cids`` audit list. + + ``artifact_cids`` (TEN-240 Phase 7) supplies extra audit entries + for dual-written non-DB artifacts. Each element is appended to the + list verbatim — callers set the ``kind`` (``checkpoint``, + ``architecture``, ``training_meta``, …). """ payload: dict = {"data": element_data} if substrate_cid: @@ -144,29 +122,35 @@ async def add_experiment( return None async def get_frontier(self, task: str = "") -> list[dict]: + """GET the current Pareto frontier.""" params = {"task": task} if task else None result = await self._get("/frontier", params=params) return result if isinstance(result, list) else [] async def update_frontier(self, frontier_data: list[dict], task: str = "") -> bool: + """POST updated frontier data.""" result = await self._post("/frontier/update", { "frontier": frontier_data, "task": task, }) return result is not None async def get_pareto_elements(self, task: str = "") -> list[dict]: + """GET all pareto-eligible elements.""" params = {"task": task} if task else None result = await self._get("/experiments/pareto", params=params) return result if isinstance(result, list) else [] async def get_challenge(self) -> Optional[dict]: + """GET the current challenge.""" return await self._get("/challenge") async def set_challenge(self, challenge_data) -> bool: + """POST challenge data (used by validators to update challenge).""" result = await self._post("/challenge/update", {"challenge": challenge_data}) return result is not None async def record_components(self, experiment_id: int, components: list[str]) -> bool: + """POST detected components for an experiment.""" result = await self._post("/provenance/record_components", { "experiment_id": experiment_id, "components": components, }) @@ -175,6 +159,7 @@ async def record_components(self, experiment_id: int, components: list[str]) -> async def record_round_context( self, round_id: int, experiment_id: int, context_type: str = "frontier", ) -> bool: + """POST round context (which experiments were shown).""" result = await self._post("/provenance/record_context", { "round_id": round_id, "experiment_id": experiment_id, "context_type": context_type, @@ -182,19 +167,23 @@ async def record_round_context( return result is not None async def get_experiment(self, index: int) -> Optional[dict]: + """GET a single experiment by index.""" return await self._get(f"/experiments/{index}") async def get_diff(self, index: int) -> Optional[dict]: + """GET diff for an experiment vs its parent.""" return await self._get(f"/experiments/{index}/diff") - async def get_agent_code(self, miner_id: str) -> Optional[dict]: - return await self._get(f"/agent_code/{miner_id}") + async def get_agent_code(self, hotkey: str) -> Optional[dict]: + """GET a miner's agent code bundle from the DB server.""" + return await self._get(f"/agent_code/{hotkey}") async def get_agent_code_history( - self, miner_id: str, limit: int = 100, + self, hotkey: str, limit: int = 100, ) -> list[dict]: + """GET the full submission timeline for a hotkey (most recent first).""" result = await self._get( - f"/agent_code/{miner_id}/history", params={"limit": limit}, + f"/agent_code/{hotkey}/history", params={"limit": limit}, ) if isinstance(result, dict): subs = result.get("submissions") @@ -203,11 +192,13 @@ async def get_agent_code_history( return [] async def get_agent_code_by_hash(self, code_hash: str) -> Optional[dict]: + """GET an immutable agent bundle by its content hash.""" return await self._get(f"/agent_code/by_hash/{code_hash}") async def submit_agent_code( self, files: dict[str, str], entry_point: str = "agent.py", ) -> Optional[dict]: + """POST agent code bundle. Returns {"code_hash", "r2_key"} or None.""" return await self._post("/agent_code", { "files": files, "entry_point": entry_point, }) @@ -215,6 +206,8 @@ async def submit_agent_code( async def submit_training_meta( self, round_id: int, hotkey: str, meta: dict, ) -> bool: + """POST a training_meta.json blob so the public dashboard can render + loss curves without R2 access. Idempotent on (round_id, hotkey).""" result = await self._post("/training_metas", { "round_id": int(round_id), "hotkey": hotkey, @@ -225,45 +218,19 @@ async def submit_training_meta( async def submit_submission_reveal( self, round_id: int, entries: list[dict], ) -> bool: - """POST per-round submission_id -> miner_id map. Idempotent.""" + """POST the per-round submission_id -> miner_hotkey reveal map. + + Called after Phase C closes. Each entry is a dict with keys + ``submission_id``, ``miner_hotkey``, ``miner_uid``. Idempotent on + (round_id, submission_id). + """ result = await self._post("/round_submissions/reveal", { "round_id": int(round_id), "entries": entries, }) return result is not None - # ── Miner feedback surface (bearer auth) ────────────────────── - - async def my_submissions( - self, since: str = "", limit: int = 200, task: str = "", - ) -> list[dict]: - params: dict = {"limit": limit} - if since: - params["since"] = since - if task: - params["task"] = task - result = await self._get("/miners/me/submissions", params=params) - if isinstance(result, dict): - return result.get("submissions", []) or [] - return result if isinstance(result, list) else [] - - async def my_results( - self, since: str = "", limit: int = 200, task: str = "", - ) -> list[dict]: - params: dict = {"limit": limit} - if since: - params["since"] = since - if task: - params["task"] = task - result = await self._get("/miners/me/results", params=params) - if isinstance(result, dict): - return result.get("results", []) or [] - return result if isinstance(result, list) else [] - - async def my_summary(self) -> dict: - result = await self._get("/miners/me/summary") - return result if isinstance(result, dict) else {} - async def close(self): + """Close the underlying HTTP client.""" if self._client and not self._client.is_closed: await self._client.aclose() diff --git a/shared/miner_auth.py b/shared/miner_auth.py deleted file mode 100644 index 1b4837a..0000000 --- a/shared/miner_auth.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Bearer-token miner auth — registry helpers backed by Postgres. - -The non-competitive Radar feedback API (``/miners/me/*`` and a couple -of CLI helpers) authenticates miners with operator-issued bearer -tokens stored hashed in ``miner_api_keys``. This module is the choke -point for issuing, looking up, and revoking those keys; the FastAPI -middleware delegates here so the SQL stays in one place. - -The hash is SHA-256 of the raw key (see ``shared.auth.hash_api_key``) -so a database leak doesn't recover the plaintext bearer. - -Key format on the wire: ``rdrk_<32-hex-chars>`` — the prefix lets -operators grep keys out of logs, but isn't required by the verifier. -""" - -from __future__ import annotations - -import logging -import secrets -import time -import uuid -from dataclasses import dataclass -from typing import Optional - -from shared.auth import hash_api_key - -logger = logging.getLogger(__name__) - -KEY_PREFIX = "rdrk_" - - -@dataclass(slots=True) -class MinerIdentity: - """Resolved bearer-auth identity returned to the request handler.""" - - miner_id: str - key_id: str - scope: str - hotkey: str = "" - name: str = "" - - -def _new_key_pair() -> tuple[str, str]: - """Return ``(plaintext_key, key_id)``. ``key_id`` is the - user-facing identifier (not a secret) so operators can list + - revoke keys without ever holding the plaintext again.""" - token = secrets.token_hex(16) - return f"{KEY_PREFIX}{token}", uuid.uuid4().hex - - -async def register_miner( - pool, - *, - miner_id: Optional[str] = None, - name: str = "", - hotkey: str = "", - contact: str = "", -) -> str: - """Insert a row in ``miners`` and return the (possibly generated) id.""" - mid = miner_id or uuid.uuid4().hex - await pool.execute( - "INSERT INTO miners (miner_id, name, hotkey, contact) " - "VALUES ($1, $2, $3, $4) " - "ON CONFLICT (miner_id) DO UPDATE SET " - " name = EXCLUDED.name, hotkey = EXCLUDED.hotkey, " - " contact = EXCLUDED.contact", - mid, name, hotkey, contact, - ) - return mid - - -async def issue_api_key( - pool, - miner_id: str, - *, - scope: str = "miner", - label: str = "", -) -> tuple[str, str]: - """Mint a new API key. Returns ``(plaintext, key_id)``. - - The plaintext is shown to the operator ONCE; the DB only stores - the hash. ``key_id`` is the stable handle for revocation. - """ - plaintext, key_id = _new_key_pair() - await pool.execute( - "INSERT INTO miner_api_keys (key_id, key_hash, miner_id, scope, label) " - "VALUES ($1, $2, $3, $4, $5)", - key_id, hash_api_key(plaintext), miner_id, scope, label, - ) - return plaintext, key_id - - -async def revoke_api_key(pool, key_id: str) -> bool: - """Mark a key revoked. Returns True iff a row was updated.""" - result = await pool.execute( - "UPDATE miner_api_keys SET revoked_at = $1 " - "WHERE key_id = $2 AND revoked_at IS NULL", - time.time(), key_id, - ) - # asyncpg returns 'UPDATE n' — split off the count. - try: - return int(result.split()[-1]) > 0 - except (ValueError, IndexError): - return False - - -async def lookup_bearer(pool, token: str) -> Optional[MinerIdentity]: - """Resolve a bearer token to its miner identity, or ``None`` if - the token is unknown / revoked.""" - if not token: - return None - digest = hash_api_key(token) - row = await pool.fetchrow( - "SELECT k.key_id, k.miner_id, k.scope, m.hotkey, m.name " - "FROM miner_api_keys k " - "JOIN miners m ON m.miner_id = k.miner_id " - "WHERE k.key_hash = $1 AND k.revoked_at IS NULL " - " AND m.revoked_at IS NULL " - "LIMIT 1", - digest, - ) - if row is None: - return None - return MinerIdentity( - miner_id=row["miner_id"], - key_id=row["key_id"], - scope=row["scope"], - hotkey=row["hotkey"] or "", - name=row["name"] or "", - ) - - -async def touch_key_usage(pool, key_id: str) -> None: - """Update ``last_used_at`` (best-effort — swallow errors).""" - try: - await pool.execute( - "UPDATE miner_api_keys SET last_used_at = $1 WHERE key_id = $2", - time.time(), key_id, - ) - except Exception as e: - logger.warning("touch_key_usage failed for %s: %s", key_id, e) - - -async def list_api_keys(pool, miner_id: str) -> list[dict]: - """Return non-secret metadata for every key of a miner.""" - rows = await pool.fetch( - "SELECT key_id, scope, label, created_at, last_used_at, revoked_at " - "FROM miner_api_keys WHERE miner_id = $1 ORDER BY created_at DESC", - miner_id, - ) - return [dict(r) for r in rows] - - -def hotkey_to_miner_id_fallback(hotkey: str) -> str: - """Deterministic placeholder ``miner_id`` for a hotkey when the - operator hasn't issued a real one yet — lets the dual-stack period - write ``experiments.prompt_id`` against a stable key derived from - the bittensor identity.""" - return f"hotkey:{hotkey}" if hotkey else "" diff --git a/shared/pg_schema.py b/shared/pg_schema.py index b5a23d1..24bd33d 100644 --- a/shared/pg_schema.py +++ b/shared/pg_schema.py @@ -247,52 +247,6 @@ ON miner_access_log(hotkey, round_id); """ -# Miner registry + bearer API keys for the non-competitive cutover. -# A ``miner_id`` is an opaque operator-issued identifier (UUID by default) -# that decouples the feedback API from the on-chain hotkey identity. -# During the dual-stack period a miner row carries both: ``hotkey`` is -# the bittensor SS58 (nullable, only set for chain-registered miners) and -# ``miner_id`` is the bearer-auth identifier scoped to /miners/me/*. -MINER_REGISTRY_SCHEMA = """ -CREATE TABLE IF NOT EXISTS miners ( - miner_id TEXT PRIMARY KEY, - name TEXT NOT NULL DEFAULT '', - hotkey TEXT NOT NULL DEFAULT '', - contact TEXT NOT NULL DEFAULT '', - created_at DOUBLE PRECISION NOT NULL DEFAULT extract(epoch from now()), - revoked_at DOUBLE PRECISION -); -CREATE INDEX IF NOT EXISTS idx_miners_hotkey ON miners(hotkey) - WHERE hotkey <> ''; - -CREATE TABLE IF NOT EXISTS miner_api_keys ( - key_id TEXT PRIMARY KEY, - key_hash TEXT NOT NULL, - miner_id TEXT NOT NULL REFERENCES miners(miner_id) ON DELETE CASCADE, - scope TEXT NOT NULL DEFAULT 'miner', - label TEXT NOT NULL DEFAULT '', - created_at DOUBLE PRECISION NOT NULL DEFAULT extract(epoch from now()), - last_used_at DOUBLE PRECISION, - revoked_at DOUBLE PRECISION -); -CREATE INDEX IF NOT EXISTS idx_miner_api_keys_hash - ON miner_api_keys(key_hash) WHERE revoked_at IS NULL; -CREATE INDEX IF NOT EXISTS idx_miner_api_keys_miner - ON miner_api_keys(miner_id); - --- ``prompt_id`` on experiments lets miners correlate Phase C scores --- with the prompt variant that produced the architecture (drives the --- GEPA / random_mutate feedback loop on the miner CLI). Opaque to --- the operator; only the issuing miner interprets it. -ALTER TABLE experiments - ADD COLUMN IF NOT EXISTS prompt_id TEXT; -CREATE INDEX IF NOT EXISTS idx_experiments_prompt_id - ON experiments(prompt_id) WHERE prompt_id IS NOT NULL; -CREATE INDEX IF NOT EXISTS idx_experiments_miner_prompt - ON experiments(miner_hotkey, prompt_id) - WHERE prompt_id IS NOT NULL; -""" - def _decode_jsonb(value, default): """Coerce a JSONB column value to a Python dict/list. @@ -360,9 +314,6 @@ def row_to_element(row) -> DataElement: _decode_jsonb(row["substrate_cids"], []) if "substrate_cids" in row_keys else [] ) - prompt_id = ( - (row["prompt_id"] or "") if "prompt_id" in row_keys else "" - ) return DataElement( index=row["id"], @@ -389,7 +340,6 @@ def row_to_element(row) -> DataElement: task=row["task"], round_id=row["round_id"] if row["round_id"] is not None else -1, substrate_cids=substrate_cids, - prompt_id=prompt_id, ) @@ -416,7 +366,7 @@ def _jsonb(value) -> str: def element_to_params(element: DataElement, next_id: int) -> tuple: - """Convert a DataElement to a positional tuple for INSERT ($1..$25). + """Convert a DataElement to a positional tuple for INSERT ($1..$24). JSONB columns (loss_curve, generated_samples, objectives, tool_calls, agent_behavior, substrate_cids) are explicitly serialised with @@ -425,7 +375,6 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: are replaced with ``null`` to keep PostgreSQL happy. """ round_id = element.round_id if element.round_id >= 0 else None - prompt_id = getattr(element, "prompt_id", "") or None return ( next_id, element.name, @@ -451,7 +400,6 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: _jsonb(element.tool_calls), _jsonb(element.agent_behavior), _jsonb(element.substrate_cids), - prompt_id, ) @@ -461,15 +409,13 @@ def element_to_params(element: DataElement, next_id: int) -> tuple: parent_index, generation, score, miner_uid, miner_hotkey, loss_curve, manifest_sha256, generated_samples, objectives, timestamp, round_id, task, - reasoning, tool_calls, agent_behavior, substrate_cids, - prompt_id + reasoning, tool_calls, agent_behavior, substrate_cids ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, - $21, $22, $23, $24, - $25 + $21, $22, $23, $24 ) """ diff --git a/shared/pg_store.py b/shared/pg_store.py index d2d925f..d0cf6fb 100644 --- a/shared/pg_store.py +++ b/shared/pg_store.py @@ -16,7 +16,6 @@ from shared.database import DataElement from shared.pg_schema import ( SCHEMA_TABLE_DDL, SCHEMA_INDEX_DDL, FTS_FUNCTION_DDL, FTS_TRIGGER_DDL, - MINER_REGISTRY_SCHEMA, INSERT_SQL, row_to_element, element_to_params, compute_diff, _finite_or, ) @@ -154,7 +153,6 @@ async def init_schema(self): await conn.execute(SCHEMA_INDEX_DDL) await conn.execute(FTS_FUNCTION_DDL) await conn.execute(FTS_TRIGGER_DDL) - await conn.execute(MINER_REGISTRY_SCHEMA) from shared.pg_provenance import PgProvenanceQuery self.provenance = PgProvenanceQuery(self.pool) await self.provenance.init_schema() diff --git a/shared/protocol.py b/shared/protocol.py index c337aea..07f1851 100644 --- a/shared/protocol.py +++ b/shared/protocol.py @@ -84,12 +84,6 @@ class Proposal: populates them. They are useful for explainability but trivially fakeable; pair with validator-observed signals (proxy access logs, pod wall-clock) for adversarial use. - - ``prompt_id`` is opaque to the operator — it's a miner-local - identifier (UUID by default) for the prompt variant that produced - this architecture. Round-tripped back via - ``experiments.prompt_id`` so the miner's GEPA/random_mutate loop - can correlate Phase C scores with the variant. """ code: str = "" @@ -101,9 +95,6 @@ class Proposal: # (e.g. {"tool": "llm", "prompt": "...", "summary": "..."}). Schema is # intentionally open so different agents can use different shapes. tool_calls: list = field(default_factory=list) - # Miner-local prompt-variant id; surfaced on the experiments row so - # the miner CLI's optimizer can attribute Phase C scores back. - prompt_id: str = "" def to_json(self) -> str: return json.dumps({ @@ -112,7 +103,6 @@ def to_json(self) -> str: "motivation": self.motivation, "reasoning": self.reasoning, "tool_calls": self.tool_calls, - "prompt_id": self.prompt_id, }) @classmethod diff --git a/shared/substrate.py b/shared/substrate.py index aa8c3a4..7055552 100644 --- a/shared/substrate.py +++ b/shared/substrate.py @@ -22,21 +22,13 @@ import json import logging from dataclasses import asdict, dataclass, fields -from typing import TYPE_CHECKING, Any +from typing import Any -if TYPE_CHECKING: # only the type stubs at static-check time - import bittensor as bt # noqa: F401 +import bittensor as bt logger = logging.getLogger(__name__) -def _bt(): - """Lazy bittensor import — keeps this module loadable in - non-competitive deployments that never sign/verify records.""" - import bittensor as bt - return bt - - SCHEMA_VERSION = "radar.substrate.v1" SIGNATURE_PREFIX = "sr25519:" @@ -157,7 +149,6 @@ def verify_record( if not record.validator_hotkey: return False, "validator_hotkey is empty" try: - bt = _bt() keypair = bt.Keypair(ss58_address=record.validator_hotkey) if not keypair.verify(canonical_bytes(record), sig_bytes): return False, "sr25519 signature does not match canonical bytes" diff --git a/tests/test_agent_prompt_loader.py b/tests/test_agent_prompt_loader.py deleted file mode 100644 index 4e811e8..0000000 --- a/tests/test_agent_prompt_loader.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Tests for the in-agent prompt loader (miner_template/agent.py).""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from miner_template import agent - - -def _write_active(tmp_path: Path, prompts: list[dict]) -> Path: - (tmp_path / "prompts").mkdir() - p = tmp_path / "prompts" / "active.json" - p.write_text(json.dumps({"prompts": prompts})) - return p - - -def test_loader_returns_empty_when_no_file(monkeypatch, tmp_path): - monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "missing")) - got = agent._load_active_prompt(round_id=0) - assert got == {"id": "", "template": ""} - - -def test_loader_returns_empty_when_population_empty(monkeypatch, tmp_path): - _write_active(tmp_path, []) - monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) - got = agent._load_active_prompt(round_id=0) - assert got == {"id": "", "template": ""} - - -def test_loader_returns_round_robin_pick(monkeypatch, tmp_path): - _write_active(tmp_path, [ - {"id": "a", "template": "A"}, - {"id": "b", "template": "B"}, - {"id": "c", "template": "C"}, - ]) - monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) - picks = [agent._load_active_prompt(i)["id"] for i in range(5)] - assert picks == ["a", "b", "c", "a", "b"] - - -def test_loader_handles_corrupt_json(monkeypatch, tmp_path): - (tmp_path / "prompts").mkdir() - (tmp_path / "prompts" / "active.json").write_text("{not json") - monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) - assert agent._load_active_prompt(0) == {"id": "", "template": ""} - - -def test_loader_handles_legacy_list_payload(monkeypatch, tmp_path): - """Older optimizer versions wrote a bare list — be permissive.""" - (tmp_path / "prompts").mkdir() - (tmp_path / "prompts" / "active.json").write_text(json.dumps([ - {"id": "x", "template": "T"}, - ])) - monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) - got = agent._load_active_prompt(0) - assert got["id"] == "x" - - -def test_design_architecture_emits_prompt_id(monkeypatch, tmp_path): - """End-to-end: design_architecture round-trips the prompt id.""" - _write_active(tmp_path, [{"id": "the-active-one", "template": "T"}]) - monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "prompts")) - - # design_architecture also calls load_scratchpad/save_scratchpad; - # those names are injected by the validator's sandbox at runtime, - # so we inject stubs into the module globals here. - agent.__dict__["load_scratchpad"] = lambda c: tmp_path - agent.__dict__["save_scratchpad"] = lambda c, d: None - - class _FakeClient: - def post_json(self, *a, **k): return {} - def get_json(self, *a, **k): return {} - - out = agent.design_architecture( - challenge={ - "round_id": 0, "min_flops_equivalent": 1_000_000, - "max_flops_equivalent": 5_000_000, "feasible_frontier": [], - }, - client=_FakeClient(), - ) - assert out["prompt_id"] == "the-active-one" - - -def test_design_architecture_empty_prompt_id_when_no_population( - monkeypatch, tmp_path, -): - monkeypatch.setenv("MINER_PROMPTS_DIR", str(tmp_path / "missing")) - monkeypatch.setattr(agent, "load_scratchpad", lambda c: tmp_path) - monkeypatch.setattr(agent, "save_scratchpad", lambda c, d: None) - - class _FakeClient: - def post_json(self, *a, **k): return {} - def get_json(self, *a, **k): return {} - - out = agent.design_architecture( - challenge={ - "round_id": 0, "min_flops_equivalent": 1_000_000, - "max_flops_equivalent": 5_000_000, "feasible_frontier": [], - }, - client=_FakeClient(), - ) - assert out["prompt_id"] == "" diff --git a/tests/test_auth_hmac.py b/tests/test_auth_hmac.py deleted file mode 100644 index a9cfaa3..0000000 --- a/tests/test_auth_hmac.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Tests for the HMAC + bearer-token primitives in shared/auth.py.""" - -from __future__ import annotations - -import time - -import pytest - -from shared import auth - - -SECRET = b"super-secret-shared-key-do-not-leak" -OTHER = b"another-secret" - - -def _headers_as_wire(d: dict[str, str]) -> dict[str, str]: - """Simulate the casing servers usually present headers in.""" - return {k.lower(): v for k, v in d.items()} - - -def test_sign_verify_roundtrip(): - body = b'{"hello": "world"}' - headers = auth.hmac_sign_request(SECRET, body, key_id="op") - ok, err, kid = auth.hmac_verify_request( - _headers_as_wire(headers), body, auth.static_key_lookup("op", SECRET) - ) - assert ok, err - assert kid == "op" - - -def test_verify_rejects_tampered_body(): - body = b'{"a": 1}' - headers = auth.hmac_sign_request(SECRET, body, key_id="op") - tampered = b'{"a": 2}' - ok, err, _ = auth.hmac_verify_request( - _headers_as_wire(headers), tampered, auth.static_key_lookup("op", SECRET) - ) - assert not ok - assert "signature" in err - - -def test_verify_rejects_wrong_key(): - body = b"x" - headers = auth.hmac_sign_request(SECRET, body, key_id="op") - ok, err, _ = auth.hmac_verify_request( - _headers_as_wire(headers), body, auth.static_key_lookup("op", OTHER) - ) - assert not ok - assert "signature" in err - - -def test_verify_rejects_unknown_key_id(): - body = b"x" - headers = auth.hmac_sign_request(SECRET, body, key_id="ghost") - ok, err, _ = auth.hmac_verify_request( - _headers_as_wire(headers), body, auth.static_key_lookup("op", SECRET) - ) - assert not ok - assert "unknown key_id" in err - - -def test_verify_rejects_stale_timestamp(): - body = b"x" - # Sign with a timestamp far outside tolerance. - old_ts = str(int(time.time()) - 10_000) - sig = auth.hmac_sign_request(SECRET, body, key_id="op") - sig["X-Radar-Timestamp"] = old_ts - # Recompute the signature against the old ts so we exercise the - # timestamp-window check rather than the signature check. - import hashlib - import hmac as _hmac - - digest = hashlib.sha256(body).hexdigest() - sig["X-Radar-Signature"] = _hmac.new( - SECRET, f"{old_ts}.{digest}".encode(), hashlib.sha256 - ).hexdigest() - - ok, err, _ = auth.hmac_verify_request( - _headers_as_wire(sig), body, auth.static_key_lookup("op", SECRET) - ) - assert not ok - assert "timestamp" in err - - -def test_verify_requires_all_headers(): - body = b"x" - headers = auth.hmac_sign_request(SECRET, body, key_id="op") - for missing in ("X-Radar-Signature", "X-Radar-Timestamp", "X-Radar-Key-Id"): - partial = {k: v for k, v in headers.items() if k != missing} - ok, err, _ = auth.hmac_verify_request( - _headers_as_wire(partial), body, auth.static_key_lookup("op", SECRET) - ) - assert not ok, f"missing {missing} should fail" - assert "missing" in err.lower() - - -def test_hmac_sign_request_rejects_string_secret(): - with pytest.raises(TypeError): - auth.hmac_sign_request("not-bytes", b"x") # type: ignore[arg-type] - - -def test_extract_bearer(): - assert auth.extract_bearer({"authorization": "Bearer abc123"}) == "abc123" - assert auth.extract_bearer({"Authorization": "bearer abc123"}) == "abc123" - assert auth.extract_bearer({"authorization": "Basic xx"}) is None - assert auth.extract_bearer({"authorization": "Bearer "}) is None - assert auth.extract_bearer({}) is None - assert auth.extract_bearer({"authorization": "Bearer"}) is None - - -def test_hash_api_key_stable(): - h1 = auth.hash_api_key("hunter2") - h2 = auth.hash_api_key("hunter2") - assert h1 == h2 - assert h1 != auth.hash_api_key("hunter3") - assert len(h1) == 64 # sha256 hex - - -def test_load_service_secret_unset(monkeypatch): - monkeypatch.delenv("RADAR_SERVICE_KEY", raising=False) - with pytest.raises(RuntimeError): - auth.load_service_secret() - - -def test_load_service_secret_set(monkeypatch): - monkeypatch.setenv("RADAR_SERVICE_KEY", "abc") - assert auth.load_service_secret() == b"abc" diff --git a/tests/test_cli_subcommands.py b/tests/test_cli_subcommands.py deleted file mode 100644 index 51505b3..0000000 --- a/tests/test_cli_subcommands.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Tests for the miner CLI subcommands. - -We exercise the CLI by importing ``miner.cli`` directly and calling -``dispatch(argv)`` with controlled argv. HTTP calls are intercepted via -``httpx.MockTransport`` on a fake ``MinerResultsClient``. - -GEPA itself isn't exercised here — we route to ``random_mutate`` to keep -the test fast and dependency-free. -""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path -from unittest.mock import patch - -import httpx -import pytest - -from miner import cli -from miner_template import prompts as prompts_mod -from miner_template.prompts import Prompt -from miner_template.results_client import MinerResultsClient - - -@pytest.fixture -def fake_results_client(monkeypatch): - """Install a MockTransport-backed MinerResultsClient. - - Returns a dict the test populates with ``submissions``, ``results``, - ``summary`` to control the fake server's responses. - """ - state: dict = {"submissions": [], "results": [], "summary": {}} - - def handler(request: httpx.Request) -> httpx.Response: - path = request.url.path - if path == "/miners/me/submissions": - return httpx.Response(200, json={"submissions": state["submissions"]}) - if path == "/miners/me/results": - return httpx.Response(200, json={"results": state["results"]}) - if path == "/miners/me/summary": - return httpx.Response(200, json=state["summary"]) - return httpx.Response(404, json={"error": "no route"}) - - original_init = MinerResultsClient.__init__ - - def patched_init(self, db_url, api_key, *, timeout=30.0, client=None): - if client is None: - transport = httpx.MockTransport(handler) - client = httpx.Client(transport=transport) - original_init(self, db_url, api_key, timeout=timeout, client=client) - - monkeypatch.setattr(MinerResultsClient, "__init__", patched_init) - return state - - -# ── results ───────────────────────────────────────────────────────── - - -def test_results_requires_db_url_and_key(capsys): - code = cli.dispatch([ - "miner", "results", "--db-url", "", "--api-key", "", - ]) - assert code == 2 - assert "missing --db-url or --api-key" in capsys.readouterr().err - - -def test_results_prints_table(fake_results_client, capsys): - fake_results_client["results"] = [{ - "round_id": 12, "submission_id": "s-1", "task_name": "ts", - "prompt_id": "p-abc", "architecture_code": "code", - "scores": {"raw_score": 0.42}, - }] - code = cli.dispatch([ - "miner", "results", - "--db-url", "http://db", "--api-key", "k", - ]) - out = capsys.readouterr().out - assert code == 0 - assert "round" in out - assert "s-1" in out - assert "0.4200" in out - - -def test_results_json_mode(fake_results_client, capsys): - fake_results_client["results"] = [{ - "round_id": 1, "submission_id": "s-1", "task_name": "t", - "prompt_id": "p", "scores": {"raw_score": 0.5}, - }] - code = cli.dispatch([ - "miner", "results", - "--db-url", "http://db", "--api-key", "k", "--json", - ]) - assert code == 0 - out = capsys.readouterr().out.strip().splitlines() - rec = json.loads(out[0]) - assert rec["submission_id"] == "s-1" - assert rec["scores"]["raw_score"] == 0.5 - - -def test_results_empty(fake_results_client, capsys): - code = cli.dispatch([ - "miner", "results", - "--db-url", "http://db", "--api-key", "k", - ]) - assert code == 0 - assert "(no results)" in capsys.readouterr().out - - -# ── optimize ───────────────────────────────────────────────────────── - - -def test_optimize_requires_db_url_and_key(capsys): - code = cli.dispatch([ - "miner", "optimize", "--db-url", "", "--api-key", "", - ]) - assert code == 2 - assert "missing --db-url" in capsys.readouterr().err - - -def test_optimize_without_seed_or_active_pop_fails( - fake_results_client, tmp_path, capsys, -): - code = cli.dispatch([ - "miner", "optimize", - "--db-url", "http://db", "--api-key", "k", - "--prompts-dir", str(tmp_path), - "--optimizer", "random_mutate", - ]) - assert code == 2 - assert "no active prompt population" in capsys.readouterr().err - - -def test_optimize_with_seed_creates_pop_and_writes_generation( - fake_results_client, tmp_path, capsys, -): - code = cli.dispatch([ - "miner", "optimize", - "--db-url", "http://db", "--api-key", "k", - "--prompts-dir", str(tmp_path), - "--optimizer", "random_mutate", - "--population", "4", - "--seed", - ]) - out = capsys.readouterr().out - assert code == 0 - assert "Seeded default prompt" in out - pop = prompts_mod.load_active(tmp_path) - assert len(pop) == 4 - assert prompts_mod.list_history(tmp_path) == [1] - - -def test_optimize_dry_run_does_not_write( - fake_results_client, tmp_path, capsys, -): - prompts_mod.save_active( - [Prompt.new(template="A"), Prompt.new(template="B")], tmp_path, - ) - fake_results_client["results"] = [{ - "round_id": 1, "submission_id": "s", "task_name": "t", - "prompt_id": "doesnt-matter", "scores": {"raw_score": 0.1}, - }] - before = prompts_mod.load_active(tmp_path) - code = cli.dispatch([ - "miner", "optimize", - "--db-url", "http://db", "--api-key", "k", - "--prompts-dir", str(tmp_path), - "--optimizer", "random_mutate", - "--population", "3", "--dry-run", - ]) - out = capsys.readouterr().out - assert code == 0 - assert "would write population" in out - after = prompts_mod.load_active(tmp_path) - assert [p.id for p in before] == [p.id for p in after] - assert prompts_mod.list_history(tmp_path) == [] - - -def test_optimize_unknown_optimizer(fake_results_client, tmp_path, capsys): - prompts_mod.save_active([Prompt.new(template="A")], tmp_path) - code = cli.dispatch([ - "miner", "optimize", - "--db-url", "http://db", "--api-key", "k", - "--prompts-dir", str(tmp_path), - "--optimizer", "no-such-optimizer", - ]) - assert code == 2 - assert "cannot resolve optimizer" in capsys.readouterr().err - - -# ── prompts ────────────────────────────────────────────────────────── - - -def test_prompts_list_empty(tmp_path, capsys): - code = cli.dispatch([ - "miner", "prompts", "list", "--prompts-dir", str(tmp_path), - ]) - assert code == 0 - assert "(no active prompt population)" in capsys.readouterr().out - - -def test_prompts_list_shows_population(tmp_path, capsys): - prompts_mod.save_active([ - Prompt(id="aaaaaaaa", template="hello world", generation=2), - ], tmp_path) - code = cli.dispatch([ - "miner", "prompts", "list", "--prompts-dir", str(tmp_path), - ]) - out = capsys.readouterr().out - assert code == 0 - assert "gen 2" in out - assert "aaaaaaaa" in out - - -def test_prompts_history_empty(tmp_path, capsys): - code = cli.dispatch([ - "miner", "prompts", "history", "--prompts-dir", str(tmp_path), - ]) - assert code == 0 - assert "(no archived generations)" in capsys.readouterr().out - - -def test_prompts_history_lists_archives(tmp_path, capsys): - prompts_mod.save_active([Prompt.new(template="a")], tmp_path) - prompts_mod.archive_current(1, tmp_path) - prompts_mod.archive_current(7, tmp_path) - code = cli.dispatch([ - "miner", "prompts", "history", "--prompts-dir", str(tmp_path), - ]) - out = capsys.readouterr().out - assert "gen_001" in out - assert "gen_007" in out - - -def test_prompts_rollback_missing_generation(tmp_path, capsys): - code = cli.dispatch([ - "miner", "prompts", "rollback", "99", "--prompts-dir", str(tmp_path), - ]) - assert code == 2 - assert "no archived generation" in capsys.readouterr().err - - -def test_prompts_rollback_restores_archived_pop(tmp_path): - prompts_mod.save_active([Prompt(id="x", template="A")], tmp_path) - prompts_mod.archive_current(1, tmp_path) - prompts_mod.save_active([Prompt(id="y", template="B")], tmp_path) - code = cli.dispatch([ - "miner", "prompts", "rollback", "1", "--prompts-dir", str(tmp_path), - ]) - assert code == 0 - cur = prompts_mod.load_active(tmp_path) - assert [p.id for p in cur] == ["x"] - - -# ── routing ────────────────────────────────────────────────────────── - - -def test_is_subcommand_matches_known_tokens(): - assert cli.is_subcommand(["miner", "results"]) - assert cli.is_subcommand(["miner", "optimize"]) - assert cli.is_subcommand(["miner", "prompts"]) - assert not cli.is_subcommand(["miner"]) - assert not cli.is_subcommand(["miner", "run"]) - assert not cli.is_subcommand(["miner", "--netuid", "1"]) - - -def test_dispatch_unknown_subcommand(capsys): - code = cli.dispatch(["miner", "wat"]) - assert code == 2 - assert "Unknown subcommand" in capsys.readouterr().err diff --git a/tests/test_config_noncompetitive.py b/tests/test_config_noncompetitive.py deleted file mode 100644 index 9fddbfd..0000000 --- a/tests/test_config_noncompetitive.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Tests for Config.NONCOMPETITIVE + service-key wiring.""" - -from __future__ import annotations - -import importlib - -import pytest - - -def _reload_config(monkeypatch, **env): - for k, v in env.items(): - if v is None: - monkeypatch.delenv(k, raising=False) - else: - monkeypatch.setenv(k, v) - import config as cfg - importlib.reload(cfg) - return cfg.Config - - -def test_noncompetitive_defaults_false(monkeypatch): - cfg = _reload_config(monkeypatch, RADAR_NONCOMPETITIVE=None) - assert cfg.NONCOMPETITIVE is False - - -def test_noncompetitive_truthy_variants(monkeypatch): - for val in ("true", "TRUE", "1", "yes"): - cfg = _reload_config(monkeypatch, RADAR_NONCOMPETITIVE=val) - assert cfg.NONCOMPETITIVE is True, f"failed for {val}" - - -def test_noncompetitive_falsy_variants(monkeypatch): - for val in ("false", "0", "no", ""): - cfg = _reload_config(monkeypatch, RADAR_NONCOMPETITIVE=val) - assert cfg.NONCOMPETITIVE is False, f"failed for {val}" - - -def test_service_key_id_defaults_operator(monkeypatch): - cfg = _reload_config(monkeypatch, RADAR_SERVICE_KEY_ID=None) - assert cfg.SERVICE_KEY_ID == "operator" - - -def test_service_key_id_overrideable(monkeypatch): - cfg = _reload_config(monkeypatch, RADAR_SERVICE_KEY_ID="my-op") - assert cfg.SERVICE_KEY_ID == "my-op" - - -def test_service_key_passes_through(monkeypatch): - cfg = _reload_config(monkeypatch, RADAR_SERVICE_KEY="a" * 64) - assert cfg.SERVICE_KEY == "a" * 64 diff --git a/tests/test_miner_auth_helpers.py b/tests/test_miner_auth_helpers.py deleted file mode 100644 index d7ae051..0000000 --- a/tests/test_miner_auth_helpers.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Unit tests for shared.miner_auth without a live Postgres. - -The async pool methods are exercised against a tiny in-memory fake so -the helpers' SQL shape, key minting, hashing, and dataclass mapping -are all covered. Real-DB integration is covered by the -``test_pg_*`` suite when ``TEST_PG_DSN`` is set. -""" - -from __future__ import annotations - -import re - -import pytest - -from shared import miner_auth -from shared.auth import hash_api_key - - -class _FakePool: - """Records execute() calls, answers fetch*/execute with scripted rows.""" - - def __init__(self): - self.calls: list[tuple] = [] - self.rows: dict[str, list[dict]] = {} - self.next_execute_result = "UPDATE 1" - - async def execute(self, sql, *args): - self.calls.append(("execute", sql, args)) - return self.next_execute_result - - async def fetchrow(self, sql, *args): - self.calls.append(("fetchrow", sql, args)) - rows = self.rows.get("fetchrow", []) - return rows.pop(0) if rows else None - - async def fetch(self, sql, *args): - self.calls.append(("fetch", sql, args)) - return self.rows.get("fetch", []) - - -@pytest.mark.asyncio -async def test_register_miner_assigns_id_when_omitted(): - pool = _FakePool() - mid = await miner_auth.register_miner(pool, name="alice") - assert isinstance(mid, str) and len(mid) == 32 - assert pool.calls[0][0] == "execute" - - -@pytest.mark.asyncio -async def test_register_miner_respects_supplied_id(): - pool = _FakePool() - mid = await miner_auth.register_miner( - pool, miner_id="custom", name="bob", - ) - assert mid == "custom" - - -@pytest.mark.asyncio -async def test_issue_api_key_returns_prefixed_plaintext_and_handle(): - pool = _FakePool() - plaintext, key_id = await miner_auth.issue_api_key( - pool, "m1", label="prod", - ) - assert plaintext.startswith(miner_auth.KEY_PREFIX) - assert re.match(r"^rdrk_[0-9a-f]{32}$", plaintext) - assert len(key_id) == 32 - # The INSERT stores the *hash*, never the plaintext. - _, sql, args = pool.calls[-1] - assert "INSERT INTO miner_api_keys" in sql - assert args[1] == hash_api_key(plaintext) - assert plaintext not in str(args) - - -@pytest.mark.asyncio -async def test_revoke_api_key_returns_true_when_row_updated(): - pool = _FakePool() - pool.next_execute_result = "UPDATE 1" - assert await miner_auth.revoke_api_key(pool, "k1") is True - - -@pytest.mark.asyncio -async def test_revoke_api_key_returns_false_when_no_row(): - pool = _FakePool() - pool.next_execute_result = "UPDATE 0" - assert await miner_auth.revoke_api_key(pool, "missing") is False - - -@pytest.mark.asyncio -async def test_lookup_bearer_returns_none_for_empty_token(): - pool = _FakePool() - assert await miner_auth.lookup_bearer(pool, "") is None - - -@pytest.mark.asyncio -async def test_lookup_bearer_returns_none_when_db_has_no_match(): - pool = _FakePool() - assert await miner_auth.lookup_bearer(pool, "any-token") is None - - -@pytest.mark.asyncio -async def test_lookup_bearer_maps_row_to_identity(): - pool = _FakePool() - pool.rows["fetchrow"] = [{ - "key_id": "k1", - "miner_id": "m1", - "scope": "miner", - "hotkey": "5xxx", - "name": "alice", - }] - ident = await miner_auth.lookup_bearer(pool, "secret-token") - assert ident is not None - assert ident.miner_id == "m1" - assert ident.key_id == "k1" - assert ident.scope == "miner" - assert ident.hotkey == "5xxx" - assert ident.name == "alice" - # The lookup hashes the token before hitting the DB. - _, sql, args = pool.calls[-1] - assert args[0] == hash_api_key("secret-token") - - -@pytest.mark.asyncio -async def test_lookup_bearer_handles_null_optional_fields(): - pool = _FakePool() - pool.rows["fetchrow"] = [{ - "key_id": "k", "miner_id": "m", "scope": "miner", - "hotkey": None, "name": None, - }] - ident = await miner_auth.lookup_bearer(pool, "x") - assert ident.hotkey == "" - assert ident.name == "" - - -@pytest.mark.asyncio -async def test_touch_key_usage_swallows_errors(): - class Bad(_FakePool): - async def execute(self, *args, **kwargs): - raise RuntimeError("db down") - - await miner_auth.touch_key_usage(Bad(), "k1") # must not raise - - -@pytest.mark.asyncio -async def test_list_api_keys_returns_dicts(): - pool = _FakePool() - pool.rows["fetch"] = [{"key_id": "k1"}, {"key_id": "k2"}] - rows = await miner_auth.list_api_keys(pool, "m1") - assert [r["key_id"] for r in rows] == ["k1", "k2"] - - -def test_hotkey_fallback(): - assert miner_auth.hotkey_to_miner_id_fallback("hk1") == "hotkey:hk1" - assert miner_auth.hotkey_to_miner_id_fallback("") == "" diff --git a/tests/test_miner_feedback_routes.py b/tests/test_miner_feedback_routes.py deleted file mode 100644 index 9f9967a..0000000 --- a/tests/test_miner_feedback_routes.py +++ /dev/null @@ -1,284 +0,0 @@ -"""End-to-end-ish tests for the /miners/me/* + /tasks/{task}/frontier -routes. - -We stub the asyncpg pool with a small fake so the test runs without -Postgres. The middleware's bearer-lookup path is exercised, plus the -SQL parameterisation and JSON shaping. -""" - -from __future__ import annotations - -from contextlib import contextmanager - -import pytest -from fastapi import FastAPI -from fastapi.testclient import TestClient - -from shared.auth import hash_api_key - - -# ── Fake pool ──────────────────────────────────────────────────────── - - -class _Row(dict): - """asyncpg.Record-like dict — supports both ``row[k]`` and ``row.keys()``.""" - - -class _FakePool: - """Records calls and answers fetch* with scripted rows.""" - - def __init__(self): - self.fetch_rows: list[list[_Row]] = [] - self.fetchrow_rows: list[_Row | None] = [] - self.fetchval_rows: list = [] - self.executed: list[tuple[str, tuple]] = [] - - async def fetch(self, sql, *args): - self.executed.append((sql, args)) - if self.fetch_rows: - return self.fetch_rows.pop(0) - return [] - - async def fetchrow(self, sql, *args): - self.executed.append((sql, args)) - if self.fetchrow_rows: - return self.fetchrow_rows.pop(0) - return None - - async def fetchval(self, sql, *args): - self.executed.append((sql, args)) - if self.fetchval_rows: - return self.fetchval_rows.pop(0) - return None - - async def execute(self, sql, *args): - self.executed.append((sql, args)) - return "UPDATE 1" - - -@contextmanager -def _wire_app(monkeypatch): - """Build a fresh FastAPI app with our routes + fake pool.""" - # Force a clean import so previous tests' state doesn't leak. - from database import miner_feedback as mf, server as srv - - pool = _FakePool() - monkeypatch.setattr(srv, "_pool", pool) - app = FastAPI() - mf.include_miner_feedback_routes(app) - - # Replicate the bearer middleware locally so we don't pull the full - # validator surface in. - @app.middleware("http") - async def _bearer(request, call_next): - if request.url.path.startswith("/miners/me/"): - return await srv._bearer_auth_then_call(request, call_next) - return await call_next(request) - - yield app, pool - - -# ── lookup_bearer test scaffolding ────────────────────────────────── - - -def _install_bearer_identity(monkeypatch, ident_dict): - """Patch shared.miner_auth.lookup_bearer to return ``ident_dict`` (or - ``None`` for unknown).""" - from shared import miner_auth - - async def fake_lookup(pool, token): - if token == "good": - return miner_auth.MinerIdentity(**ident_dict) - return None - - monkeypatch.setattr(miner_auth, "lookup_bearer", fake_lookup) - - -def _ident(hotkey="hk-1", miner_id="m1"): - return { - "miner_id": miner_id, "key_id": "k1", - "scope": "miner", "hotkey": hotkey, "name": "alice", - } - - -# ── /miners/me/submissions ────────────────────────────────────────── - - -def test_submissions_requires_bearer(monkeypatch): - _install_bearer_identity(monkeypatch, _ident()) - with _wire_app(monkeypatch) as (app, _): - client = TestClient(app) - r = client.get("/miners/me/submissions") - assert r.status_code == 401 - - -def test_submissions_rejects_unknown_token(monkeypatch): - _install_bearer_identity(monkeypatch, _ident()) - with _wire_app(monkeypatch) as (app, _): - client = TestClient(app) - r = client.get( - "/miners/me/submissions", - headers={"Authorization": "Bearer wrong"}, - ) - assert r.status_code == 403 - - -def test_submissions_returns_rows_scoped_to_hotkey(monkeypatch): - _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) - with _wire_app(monkeypatch) as (app, pool): - pool.fetch_rows = [[ - _Row( - submission_id=1, round_id=42, task_name="ts_forecasting", - prompt_id="p-1", architecture_code="class M: pass", - motivation="m", reasoning="r", tool_calls=[], - created_at=1700000000.0, - ), - ]] - client = TestClient(app) - r = client.get( - "/miners/me/submissions?limit=10&task=ts_forecasting", - headers={"Authorization": "Bearer good"}, - ) - assert r.status_code == 200 - payload = r.json() - assert len(payload["submissions"]) == 1 - sub = payload["submissions"][0] - assert sub["prompt_id"] == "p-1" - assert sub["task_name"] == "ts_forecasting" - # First arg of the query is the hotkey filter — confirm scoping. - sql, args = pool.executed[0] - assert args[0] == "hk-A" - - -def test_submissions_empty_when_no_hotkey_on_identity(monkeypatch): - _install_bearer_identity(monkeypatch, _ident(hotkey="")) - with _wire_app(monkeypatch) as (app, _): - client = TestClient(app) - r = client.get( - "/miners/me/submissions", - headers={"Authorization": "Bearer good"}, - ) - assert r.status_code == 200 - assert r.json() == {"submissions": []} - - -# ── /miners/me/results ────────────────────────────────────────────── - - -def test_results_join_includes_scores(monkeypatch): - _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) - with _wire_app(monkeypatch) as (app, pool): - pool.fetch_rows = [[ - _Row( - submission_id=7, round_id=12, task_name="ts_forecasting", - prompt_id="p-7", architecture_code="x", motivation="m", - metric=0.123, score=0.45, success=True, - objectives={"flops_equivalent_size": 1_000_000, "crps": 0.123}, - created_at=1700000000.0, - ), - ]] - client = TestClient(app) - r = client.get( - "/miners/me/results", - headers={"Authorization": "Bearer good"}, - ) - assert r.status_code == 200 - row = r.json()["results"][0] - assert row["prompt_id"] == "p-7" - assert row["scores"]["raw_score"] == 0.45 - assert row["scores"]["metric"] == 0.123 - assert row["scores"]["flops_equivalent_size"] == 1_000_000 - assert row["scores"]["success"] is True - - -def test_results_handles_objectives_as_json_string(monkeypatch): - _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) - with _wire_app(monkeypatch) as (app, pool): - pool.fetch_rows = [[ - _Row( - submission_id=1, round_id=1, task_name="t", - prompt_id=None, architecture_code="", motivation="", - metric=None, score=0.0, success=False, - objectives='{"flops_equivalent_size": 5}', - created_at=0.0, - ), - ]] - client = TestClient(app) - r = client.get( - "/miners/me/results", - headers={"Authorization": "Bearer good"}, - ) - assert r.status_code == 200 - assert r.json()["results"][0]["scores"]["flops_equivalent_size"] == 5 - - -# ── /miners/me/summary ────────────────────────────────────────────── - - -def test_summary_aggregates(monkeypatch): - _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) - with _wire_app(monkeypatch) as (app, pool): - pool.fetchrow_rows = [ - _Row(total=10, last_round=42), - _Row(mean_score=0.55, successes=3, n=8), - ] - client = TestClient(app) - r = client.get( - "/miners/me/summary", - headers={"Authorization": "Bearer good"}, - ) - assert r.status_code == 200 - s = r.json() - assert s["total_submissions"] == 10 - assert s["last_round_id"] == 42 - assert s["mean_score_recent"] == 0.55 - assert s["successes_recent"] == 3 - assert s["recent_window"] == 8 - - -def test_summary_zero_when_no_rows(monkeypatch): - _install_bearer_identity(monkeypatch, _ident(hotkey="hk-A")) - with _wire_app(monkeypatch) as (app, pool): - pool.fetchrow_rows = [ - _Row(total=0, last_round=None), - _Row(mean_score=None, successes=0, n=0), - ] - client = TestClient(app) - r = client.get( - "/miners/me/summary", - headers={"Authorization": "Bearer good"}, - ) - assert r.status_code == 200 - s = r.json() - assert s["total_submissions"] == 0 - assert s["last_round_id"] is None - assert s["mean_score_recent"] is None - - -# ── /tasks/{task}/frontier (public) ───────────────────────────────── - - -def test_task_frontier_public_no_auth(monkeypatch): - with _wire_app(monkeypatch) as (app, pool): - pool.fetch_rows = [[ - _Row(metric=0.1, objectives={"flops_equivalent_size": 1_000_000}, score=0.9), - _Row(metric=0.2, objectives='{"flops_equivalent_size": 5_000_000}', score=0.7), - ]] - client = TestClient(app) - r = client.get("/tasks/ts_forecasting/frontier?limit=2") - assert r.status_code == 200 - payload = r.json() - assert payload["task"] == "ts_forecasting" - assert len(payload["points"]) == 2 - assert payload["points"][0]["flops"] == 1_000_000 - assert payload["points"][0]["metric"] == 0.1 - - -def test_task_frontier_caps_limit(monkeypatch): - with _wire_app(monkeypatch) as (app, pool): - pool.fetch_rows = [[]] - client = TestClient(app) - client.get("/tasks/foo/frontier?limit=99999") - # Last arg of the SQL call is the LIMIT — confirm it's capped at 500. - _, args = pool.executed[-1] - assert args[-1] == 500 diff --git a/tests/test_noncompetitive_gates.py b/tests/test_noncompetitive_gates.py deleted file mode 100644 index b769e59..0000000 --- a/tests/test_noncompetitive_gates.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Regression tests: NONCOMPETITIVE early-returns + optional bittensor import. - -Confirms the chain-touching code paths are properly gated so a -non-competitive deployment can run without ever invoking bittensor in -the leaf modules (runner/server, miner/neuron, validator/neuron init, -DatabaseClient). -""" - -from __future__ import annotations - -import os -from unittest.mock import MagicMock - -import pytest - - -def test_runner_server_has_no_bittensor_reference(): - """runner/server.py was fully stripped of metagraph/bittensor.""" - import runner.server as srv - src = open(srv.__file__).read() - assert "import bittensor" not in src - assert not hasattr(srv, "_metagraph_cache") - assert not hasattr(srv, "_load_metagraph") - assert not hasattr(srv, "_metagraph_lock") - - -def test_miner_neuron_bittensor_import_is_optional(): - """``import bittensor`` is wrapped in try/except so the module - loads in non-competitive deployments that don't have bittensor - installed.""" - src = open("/home/user/radar/miner/neuron.py").read() - assert "try:\n import bittensor as bt" in src - assert "bt = None" in src - - -def test_validator_neuron_bittensor_import_is_optional(): - src = open("/home/user/radar/validator/neuron.py").read() - assert "try:\n import bittensor as bt" in src - assert "bt = None" in src - - -def test_validator_get_config_uses_argparse_in_noncompetitive(monkeypatch): - """get_config() returns a plain Namespace (no bittensor) when - NONCOMPETITIVE=true.""" - monkeypatch.setenv("RADAR_NONCOMPETITIVE", "true") - monkeypatch.setenv("RADAR_SERVICE_KEY", "a" * 64) - import importlib - import config - importlib.reload(config) - # Read the function source instead of importing the whole module - # (which has heavy transitive deps). We just need to confirm the - # branch shape. - src = open("/home/user/radar/validator/neuron.py").read() - assert "Config.NONCOMPETITIVE or bt is None" in src - assert "parser.parse_args()" in src - - -def test_miner_get_config_uses_argparse_in_noncompetitive(): - src = open("/home/user/radar/miner/neuron.py").read() - assert "Config.NONCOMPETITIVE or bt is None" in src - - -def test_set_weights_skipped_in_noncompetitive(): - """_set_weights() has an early-return on Config.NONCOMPETITIVE.""" - src = open("/home/user/radar/validator/neuron.py").read() - # Look for the gate right at the top of _set_weights. - assert ( - 'def _set_weights(self):\n """Set weights on chain' - in src - ) - assert 'NONCOMPETITIVE: skipping weight set' in src - - -def test_run_round_uses_walltime_in_noncompetitive(): - src = open("/home/user/radar/validator/neuron.py").read() - # Walltime path uses time.time() // 12 as a stand-in for block height. - assert "int(time.time() // 12)" in src - - -def test_validator_wallet_is_none_in_noncompetitive(): - """The validator init skips wallet/subtensor/metagraph creation - when NONCOMPETITIVE is set.""" - src = open("/home/user/radar/validator/neuron.py").read() - assert "self.wallet = None" in src - assert "self.subtensor = None" in src - assert "self.metagraph = None" in src - - -def test_miner_wallet_is_none_in_noncompetitive(): - src = open("/home/user/radar/miner/neuron.py").read() - assert "self.wallet = None" in src - assert "self.subtensor = None" in src - assert "self.metagraph = None" in src - - -def test_validator_db_client_uses_service_secret_in_noncompetitive(): - src = open("/home/user/radar/validator/neuron.py").read() - assert "service_secret=Config.SERVICE_KEY.encode()" in src - assert "key_id=Config.SERVICE_KEY_ID" in src - - -def test_database_client_constructor_rejects_no_auth(): - """At least one of wallet / service_secret / api_key required.""" - from shared.db_client import DatabaseClient - with pytest.raises(ValueError): - DatabaseClient("http://x") - - -def test_database_client_accepts_service_secret_only(): - from shared.db_client import DatabaseClient - c = DatabaseClient("http://x", service_secret=b"k" * 32) - assert c.service_secret == b"k" * 32 - assert c.wallet is None - - -def test_database_client_accepts_api_key_only(): - from shared.db_client import DatabaseClient - c = DatabaseClient("http://x", api_key="rdrk_abc") - assert c.api_key == "rdrk_abc" - assert c.wallet is None - assert c.service_secret is None - - -def test_database_client_signs_with_hmac_when_secret_set(): - from shared.db_client import DatabaseClient - c = DatabaseClient("http://x", service_secret=b"a" * 32, key_id="op") - headers = c._sign(b"body") - assert "X-Radar-Signature" in headers - assert "X-Radar-Timestamp" in headers - assert headers["X-Radar-Key-Id"] == "op" - - -def test_database_client_signs_with_bearer_when_key_set(): - from shared.db_client import DatabaseClient - c = DatabaseClient("http://x", api_key="rdrk_abc") - headers = c._sign(b"") - assert headers == {"Authorization": "Bearer rdrk_abc"} - - -def test_substrate_module_loads_without_bittensor(): - """shared/substrate.py uses TYPE_CHECKING + lazy _bt() so it loads - even when bittensor is missing from the environment.""" - src = open("/home/user/radar/shared/substrate.py").read() - assert "if TYPE_CHECKING" in src - assert "import bittensor as bt" in src # in the TYPE_CHECKING block - # The runtime path goes through _bt() — verify the helper exists. - assert "def _bt():" in src - # No bare top-level import outside TYPE_CHECKING. - lines = src.splitlines() - for i, line in enumerate(lines): - stripped = line.lstrip() - if stripped.startswith("import bittensor") or stripped.startswith( - "from bittensor" - ): - # Walk up to see if we're inside a TYPE_CHECKING block. - context = "\n".join(lines[max(0, i - 5):i + 1]) - assert ( - "TYPE_CHECKING" in context - or "def " in context # inside a function = lazy - ), f"bare bittensor import at line {i + 1}: {line}" - - -def test_validator_neuron_imports_without_bittensor(monkeypatch): - """validator/neuron.py + its transitive graph load when bittensor - is missing. Smoke test — confirms the chain-free deployment path - doesn't blow up at import time.""" - monkeypatch.setenv("RADAR_NONCOMPETITIVE", "true") - monkeypatch.setenv("RADAR_SERVICE_KEY", "a" * 64) - monkeypatch.setenv("RADAR_DB_API_URL", "http://localhost:8090") - import importlib - import config - importlib.reload(config) - import validator.neuron - importlib.reload(validator.neuron) - # bt is None when bittensor isn't installed; ImportError was caught - # at top-level by the try/except. In this container it IS None - # (bittensor not in pip set). - # The important assertion: the module loaded. - assert validator.neuron is not None - assert hasattr(validator.neuron, "Validator") - - -def test_no_top_level_bittensor_imports_in_main_code(): - """Sweep guard: no top-level ``import bittensor`` in code/, only - inside functions or under ``if TYPE_CHECKING:``.""" - import pathlib - - root = pathlib.Path("/home/user/radar") - bad: list[str] = [] - for path in root.rglob("*.py"): - if any(part in {"tests", "scripts"} for part in path.parts): - continue - text = path.read_text() - lines = text.splitlines() - for i, raw in enumerate(lines): - stripped = raw.lstrip() - if not ( - stripped.startswith("import bittensor") - or stripped.startswith("from bittensor") - ): - continue - indent = len(raw) - len(stripped) - # Anything indented is inside a function / try block → lazy. - if indent > 0: - continue - # Top-level — must be guarded by TYPE_CHECKING or try/except. - ctx = "\n".join(lines[max(0, i - 4):i]) - if "TYPE_CHECKING" in ctx or "try:" in ctx: - continue - bad.append(f"{path}:{i + 1}: {raw}") - assert not bad, "unguarded top-level bittensor imports:\n" + "\n".join(bad) diff --git a/tests/test_operator_cli.py b/tests/test_operator_cli.py deleted file mode 100644 index b597279..0000000 --- a/tests/test_operator_cli.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Tests for the operator CLI argv parsing + handler dispatch. - -Real-DB integration of the handlers is covered by the miner_auth -helper tests; here we just exercise the argparse surface, output -formatting, and rotate-service-key which has no DB dependency. -""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, patch - -import pytest - -from database import operator_cli - - -def test_parser_register_requires_no_args(): - args = operator_cli._build_parser().parse_args(["register"]) - assert args.command == "register" - assert args.name == "" - assert args.hotkey == "" - - -def test_parser_issue_key_requires_miner_id(): - with pytest.raises(SystemExit): - operator_cli._build_parser().parse_args(["issue-key"]) - - -def test_parser_revoke_key_requires_key_id(): - with pytest.raises(SystemExit): - operator_cli._build_parser().parse_args(["revoke-key"]) - - -def test_rotate_service_key_prints_long_hex(capsys): - code = operator_cli.main(["rotate-service-key"]) - assert code == 0 - out = capsys.readouterr().out - # 32 bytes hex = 64 chars somewhere in the output. - candidate = [w for line in out.splitlines() for w in line.split() - if len(w) == 64 and all(c in "0123456789abcdef" for c in w)] - assert candidate, f"no 64-hex key found in output:\n{out}" - - -def test_register_dispatch_calls_register_miner(monkeypatch, capsys): - fake_conn = AsyncMock() - fake_conn.close = AsyncMock() - - async def _connect_stub(): - return fake_conn - - async def _ensure_stub(conn): - pass - - monkeypatch.setattr(operator_cli, "_connect", _connect_stub) - monkeypatch.setattr(operator_cli, "_ensure_schema", _ensure_stub) - with patch( - "shared.miner_auth.register_miner", - new=AsyncMock(return_value="abc123"), - ) as mr: - code = operator_cli.main([ - "register", "--name", "alice", "--hotkey", "5xxx", - ]) - assert code == 0 - mr.assert_awaited_once() - out = capsys.readouterr().out - assert "abc123" in out - - -def test_issue_key_dispatch_prints_plaintext_once(monkeypatch, capsys): - fake_conn = AsyncMock() - fake_conn.close = AsyncMock() - - async def _connect_stub(): - return fake_conn - - async def _ensure_stub(conn): - pass - - monkeypatch.setattr(operator_cli, "_connect", _connect_stub) - monkeypatch.setattr(operator_cli, "_ensure_schema", _ensure_stub) - with patch( - "shared.miner_auth.issue_api_key", - new=AsyncMock(return_value=("rdrk_secret123", "kid-xyz")), - ): - code = operator_cli.main([ - "issue-key", "--miner-id", "m1", "--label", "prod", - ]) - assert code == 0 - out = capsys.readouterr().out - assert "rdrk_secret123" in out - assert "kid-xyz" in out - assert "shown ONCE" in out - - -def test_revoke_key_returns_1_on_miss(monkeypatch, capsys): - fake_conn = AsyncMock() - fake_conn.close = AsyncMock() - - async def _connect_stub(): - return fake_conn - - monkeypatch.setattr(operator_cli, "_connect", _connect_stub) - with patch( - "shared.miner_auth.revoke_api_key", - new=AsyncMock(return_value=False), - ): - code = operator_cli.main(["revoke-key", "--key-id", "ghost"]) - assert code == 1 - - -def test_revoke_key_returns_0_on_hit(monkeypatch, capsys): - fake_conn = AsyncMock() - fake_conn.close = AsyncMock() - - async def _connect_stub(): - return fake_conn - - monkeypatch.setattr(operator_cli, "_connect", _connect_stub) - with patch( - "shared.miner_auth.revoke_api_key", - new=AsyncMock(return_value=True), - ): - code = operator_cli.main(["revoke-key", "--key-id", "real"]) - assert code == 0 - - -def test_dotenv_is_loaded_at_import(tmp_path, monkeypatch): - """Regression: operator_cli used to read os.environ directly without - importing ``config``, which meant ``radar/.env`` was silently - ignored and every invocation printed ``RADAR_PG_DSN not set``. - Now ``import config`` triggers ``load_dotenv`` at module load — - verify the chain by writing a marker var to a fake .env, reloading - config, and asserting it shows up in ``os.environ``. - """ - import importlib - import os - - env_file = tmp_path / ".env" - env_file.write_text("RADAR_OPERATOR_CLI_DOTENV_PROBE=loaded\n") - - monkeypatch.delenv("RADAR_OPERATOR_CLI_DOTENV_PROBE", raising=False) - from dotenv import load_dotenv - load_dotenv(env_file, override=True) - - # Re-import operator_cli so its top-level ``import config`` runs - # again in this test's context (config itself is module-cached - # but the assertion is about the env var the dotenv chain set). - importlib.import_module("database.operator_cli") - assert os.environ.get("RADAR_OPERATOR_CLI_DOTENV_PROBE") == "loaded" - - -def test_list_keys_prints_table(monkeypatch, capsys): - fake_conn = AsyncMock() - fake_conn.close = AsyncMock() - - async def _connect_stub(): - return fake_conn - - monkeypatch.setattr(operator_cli, "_connect", _connect_stub) - with patch( - "shared.miner_auth.list_api_keys", - new=AsyncMock(return_value=[ - {"key_id": "k1", "scope": "miner", "label": "prod", - "created_at": 1700000000, "revoked_at": None, - "last_used_at": 1700001000}, - ]), - ): - code = operator_cli.main(["list-keys", "--miner-id", "m1"]) - assert code == 0 - out = capsys.readouterr().out - assert "k1" in out - assert "prod" in out diff --git a/tests/test_optimizer_random_mutate.py b/tests/test_optimizer_random_mutate.py deleted file mode 100644 index 61f24fc..0000000 --- a/tests/test_optimizer_random_mutate.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Tests for the random_mutate baseline optimizer.""" - -from __future__ import annotations - -import pytest - -from miner_template.optimizers import ResultRow -from miner_template.optimizers.random_mutate import optimize -from miner_template.prompts import Prompt - - -def _pop(*ids_and_templates: tuple[str, str]) -> list[Prompt]: - return [Prompt(id=i, template=t, generation=0) - for i, t in ids_and_templates] - - -def _result(prompt_id: str, score: float, round_id: int = 1) -> ResultRow: - return ResultRow( - round_id=round_id, - submission_id=f"sub-{round_id}-{prompt_id}", - task_name="ts_forecasting", - prompt_id=prompt_id, - architecture_code="class M: pass", - scores={"raw_score": score}, - ) - - -def test_empty_population_returns_empty(): - assert optimize([], [], {}) == [] - - -def test_no_results_clones_top_of_population(): - pop = _pop(("a", "A"), ("b", "B"), ("c", "C")) - new = optimize([], pop, {"population": 4, "elite_k": 2}) - assert len(new) == 4 - # Both elites must appear at the head (order = ranking, stable). - assert {p.id for p in new[:2]} == {"a", "b"} - - -def test_ranks_by_mean_score_and_promotes_winners(): - pop = _pop(("a", "A"), ("b", "B"), ("c", "C")) - results = [ - _result("a", 0.1), - _result("b", 0.9), _result("b", 0.9), - _result("c", 0.5), - ] - new = optimize(results, pop, {"population": 4, "elite_k": 2}) - assert len(new) == 4 - elite_ids = [p.id for p in new if p.parent_id is None] - assert elite_ids[0] == "b" # highest mean score - assert elite_ids[1] == "c" - - -def test_children_have_parent_pointing_at_elite(): - pop = _pop(("a", "A"), ("b", "B")) - results = [_result("a", 0.9), _result("b", 0.1)] - new = optimize(results, pop, {"population": 4, "elite_k": 1}) - children = [p for p in new if p.parent_id is not None] - assert len(children) == 3 - assert all(c.parent_id == "a" for c in children) - assert all(c.template.startswith("A") for c in children) - assert all(c.generation == 1 for c in children) - - -def test_population_size_enforced(): - pop = _pop(("a", "A"), ("b", "B")) - new = optimize([], pop, {"population": 8}) - assert len(new) == 8 - - -def test_population_below_one_raises(): - pop = _pop(("a", "A")) - with pytest.raises(ValueError): - optimize([], pop, {"population": 0}) - - -def test_deterministic_perturbation(): - """Same inputs -> same outputs (modulo new UUIDs). We can't compare - ids, but the chosen perturbation suffix is deterministic in the - parent + child_idx, so child templates should match across runs.""" - pop = _pop(("a", "Alpha")) - a = optimize([], pop, {"population": 4, "elite_k": 1}) - b = optimize([], pop, {"population": 4, "elite_k": 1}) - assert [p.template for p in a] == [p.template for p in b] - - -def test_unknown_prompt_ids_in_results_ignored(): - pop = _pop(("a", "A")) - results = [_result("ghost", 999.0)] # not in population - new = optimize(results, pop, {"population": 2, "elite_k": 1}) - assert len(new) == 2 - assert new[0].id == "a" - - -def test_custom_score_key(): - pop = _pop(("a", "A"), ("b", "B")) - rows = [ - ResultRow(round_id=1, submission_id="x", task_name="t", - prompt_id="a", architecture_code="", - scores={"raw_score": 0.0, "frontier_bonus": 1.0}), - ResultRow(round_id=1, submission_id="y", task_name="t", - prompt_id="b", architecture_code="", - scores={"raw_score": 1.0, "frontier_bonus": 0.0}), - ] - new = optimize(rows, pop, {"population": 2, "elite_k": 1, - "score_key": "frontier_bonus"}) - # Top elite should now be 'a' (high frontier_bonus) instead of 'b'. - assert new[0].id == "a" diff --git a/tests/test_optimizers_resolve.py b/tests/test_optimizers_resolve.py deleted file mode 100644 index 7ab14a5..0000000 --- a/tests/test_optimizers_resolve.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Tests for the optimizer plugin registry.""" - -from __future__ import annotations - -import pytest - -from miner_template import optimizers -from miner_template.optimizers import ResultRow - - -def test_list_builtins_has_gepa_and_random(): - names = optimizers.list_builtins() - assert "gepa" in names - assert "random_mutate" in names - - -def test_resolve_builtin_returns_callable(): - fn = optimizers.resolve("random_mutate") - assert callable(fn) - - -def test_resolve_unknown_alias_raises(): - with pytest.raises(ValueError): - optimizers.resolve("nonexistent_optimizer") - - -def test_resolve_dotted_path(): - fn = optimizers.resolve("miner_template.optimizers.random_mutate:optimize") - assert callable(fn) - - -def test_resolve_missing_module_raises_importerror(): - with pytest.raises(ImportError): - optimizers.resolve("not_a_real_module.somewhere:func") - - -def test_resolve_missing_function_raises_attributeerror(): - with pytest.raises(AttributeError): - optimizers.resolve("miner_template.optimizers.random_mutate:not_there") - - -def test_resultrow_from_dict_safe_with_missing_fields(): - r = ResultRow.from_dict({}) - assert r.round_id == 0 - assert r.prompt_id is None - assert r.scores == {} - - -def test_resultrow_from_dict_full(): - r = ResultRow.from_dict({ - "round_id": 12, - "submission_id": "sub-1", - "task_name": "ts_forecasting", - "prompt_id": "p-abc", - "architecture_code": "class M: pass", - "scores": {"raw_score": 0.7, "crps": 0.3}, - "created_at": "2026-05-19T00:00:00Z", - }) - assert r.round_id == 12 - assert r.prompt_id == "p-abc" - assert r.scores["raw_score"] == 0.7 diff --git a/tests/test_prompts_store.py b/tests/test_prompts_store.py deleted file mode 100644 index 7ad27f6..0000000 --- a/tests/test_prompts_store.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Tests for the miner-side prompt-population store.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from miner_template import prompts as ps -from miner_template.prompts import Prompt - - -def test_load_active_empty(tmp_path: Path): - assert ps.load_active(tmp_path) == [] - - -def test_save_then_load_roundtrip(tmp_path: Path): - pop = [ - Prompt.new(template="p1", generation=1), - Prompt.new(template="p2", generation=1), - ] - ps.save_active(pop, tmp_path) - got = ps.load_active(tmp_path) - assert len(got) == 2 - assert {p.template for p in got} == {"p1", "p2"} - assert all(p.generation == 1 for p in got) - - -def test_save_atomic_no_partial_on_failure(tmp_path: Path, monkeypatch): - """If os.fsync raises, ``active.json`` must not exist.""" - pop = [Prompt.new(template="ok")] - ps.save_active(pop, tmp_path) # establish baseline - original = ps.load_active(tmp_path) - - # Now break the write halfway through and confirm the original - # active.json is untouched. - real_replace = ps.os.replace - - def boom(*args, **kwargs): - raise OSError("disk full") - - monkeypatch.setattr(ps.os, "replace", boom) - with pytest.raises(OSError): - ps.save_active([Prompt.new(template="will-fail")], tmp_path) - - monkeypatch.setattr(ps.os, "replace", real_replace) - assert ps.load_active(tmp_path) == original - - -def test_archive_and_list_history(tmp_path: Path): - ps.save_active([Prompt.new(template="gen0")], tmp_path) - p = ps.archive_current(1, tmp_path) - assert p is not None and p.exists() - ps.save_active([Prompt.new(template="gen1")], tmp_path) - ps.archive_current(2, tmp_path) - assert ps.list_history(tmp_path) == [1, 2] - - -def test_archive_noop_when_no_active(tmp_path: Path): - assert ps.archive_current(1, tmp_path) is None - - -def test_load_generation_roundtrip(tmp_path: Path): - seed = [Prompt.new(template="archived")] - ps.save_active(seed, tmp_path) - ps.archive_current(5, tmp_path) - loaded = ps.load_generation(5, tmp_path) - assert len(loaded) == 1 - assert loaded[0].template == "archived" - - -def test_load_generation_missing(tmp_path: Path): - with pytest.raises(FileNotFoundError): - ps.load_generation(99, tmp_path) - - -def test_rollback_archives_current_then_restores(tmp_path: Path): - a = [Prompt.new(template="A")] - ps.save_active(a, tmp_path) - ps.archive_current(1, tmp_path) - b = [Prompt.new(template="B")] - ps.save_active(b, tmp_path) - # Rollback to gen 1 ('A'); the current 'B' should be archived too. - ps.rollback_to(1, tmp_path) - active = ps.load_active(tmp_path) - assert [p.template for p in active] == ["A"] - # Both 1 and 2 (the just-archived B) should now be in history. - assert ps.list_history(tmp_path) == [1, 2] - assert [p.template for p in ps.load_generation(2, tmp_path)] == ["B"] - - -def test_pick_for_round_deterministic_roundrobin(tmp_path: Path): - pop = [ - Prompt(id="a", template="A"), - Prompt(id="b", template="B"), - Prompt(id="c", template="C"), - ] - picks = [ps.pick_for_round(pop, i).id for i in range(7)] - assert picks == ["a", "b", "c", "a", "b", "c", "a"] - - -def test_pick_for_round_empty_raises(): - with pytest.raises(ValueError): - ps.pick_for_round([], 0) - - -def test_seed_default_creates_active(tmp_path: Path): - pop = ps.seed_default(tmp_path) - assert len(pop) == 1 - on_disk = ps.load_active(tmp_path) - assert len(on_disk) == 1 - assert on_disk[0].generation == 0 - - -def test_seed_default_is_noop_when_active_exists(tmp_path: Path): - existing = [Prompt.new(template="kept")] - ps.save_active(existing, tmp_path) - pop = ps.seed_default(tmp_path) - assert len(pop) == 1 - assert pop[0].template == "kept" - - -def test_prompt_from_dict_resilient_to_missing_fields(): - p = Prompt.from_dict({"template": "t"}) - assert p.template == "t" - assert p.id # auto-generated - assert p.generation == 0 - assert p.parent_id is None - assert p.metadata == {} - - -def test_save_uses_atomic_replace(tmp_path: Path): - """Sanity: the final file lands at ``active.json`` and no ``.tmp`` - file is left behind.""" - ps.save_active([Prompt.new(template="x")], tmp_path) - assert (tmp_path / "active.json").exists() - leftover = list(tmp_path.glob("*.tmp")) - assert leftover == [] - - -def test_archived_file_is_readable_json(tmp_path: Path): - """Make sure archived generations are still valid JSON, not just bytes.""" - ps.save_active([Prompt.new(template="t1"), Prompt.new(template="t2")], tmp_path) - dst = ps.archive_current(3, tmp_path) - assert dst is not None - data = json.loads(dst.read_text()) - assert isinstance(data, dict) - assert len(data["prompts"]) == 2 diff --git a/tests/test_protocol_prompt_id.py b/tests/test_protocol_prompt_id.py deleted file mode 100644 index 5081e2f..0000000 --- a/tests/test_protocol_prompt_id.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Tests for prompt_id round-trip through Proposal + DataElement + INSERT.""" - -from __future__ import annotations - -import json - -from shared.database import DataElement -from shared.pg_schema import INSERT_SQL, element_to_params -from shared.protocol import Proposal - - -def test_proposal_defaults_empty_prompt_id(): - p = Proposal() - assert p.prompt_id == "" - - -def test_proposal_to_json_includes_prompt_id(): - p = Proposal(code="x", prompt_id="abc123") - d = json.loads(p.to_json()) - assert d["prompt_id"] == "abc123" - - -def test_proposal_from_json_resilient_to_missing_prompt_id(): - p = Proposal.from_json(json.dumps({"code": "x"})) - assert p.prompt_id == "" - - -def test_proposal_from_json_roundtrips_prompt_id(): - p = Proposal.from_json(json.dumps({"code": "x", "prompt_id": "p1"})) - assert p.prompt_id == "p1" - - -def test_data_element_defaults_empty_prompt_id(): - e = DataElement() - assert e.prompt_id == "" - - -def test_data_element_carries_prompt_id(): - e = DataElement(name="x", prompt_id="p-abc") - assert e.prompt_id == "p-abc" - - -def test_element_to_params_includes_prompt_id_as_last_arg(): - e = DataElement(name="x", code="c", prompt_id="p-1") - params = element_to_params(e, next_id=42) - # INSERT_SQL has 25 placeholders; the new prompt_id is $25. - assert len(params) == 25 - assert params[-1] == "p-1" - - -def test_element_to_params_normalizes_empty_prompt_id_to_none(): - e = DataElement(name="x", code="c") # prompt_id default "" - params = element_to_params(e, next_id=0) - assert params[-1] is None # so the DB stores NULL not '' - - -def test_insert_sql_lists_prompt_id_column(): - assert "prompt_id" in INSERT_SQL - # 25 positional placeholders. - assert "$25" in INSERT_SQL diff --git a/tests/test_results_client.py b/tests/test_results_client.py deleted file mode 100644 index 1db250f..0000000 --- a/tests/test_results_client.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Tests for MinerResultsClient using httpx's MockTransport.""" - -from __future__ import annotations - -import json - -import httpx -import pytest - -from miner_template.optimizers import ResultRow -from miner_template.results_client import MinerResultsClient - - -def _make_client(handler, api_key: str = "test-key") -> MinerResultsClient: - transport = httpx.MockTransport(handler) - http = httpx.Client(transport=transport) - return MinerResultsClient("http://db.local", api_key, client=http) - - -def test_requires_db_url_and_api_key(): - with pytest.raises(ValueError): - MinerResultsClient("", "k") - with pytest.raises(ValueError): - MinerResultsClient("http://x", "") - - -def test_submissions_sends_bearer_and_returns_list(): - seen = {} - - def handler(request: httpx.Request) -> httpx.Response: - seen["url"] = str(request.url) - seen["auth"] = request.headers.get("authorization") - return httpx.Response(200, json={"submissions": [ - {"submission_id": "s1", "round_id": 1}, - ]}) - - c = _make_client(handler, api_key="secret") - rows = c.submissions(limit=10, task="ts_forecasting") - assert seen["auth"] == "Bearer secret" - assert "/miners/me/submissions" in seen["url"] - assert "limit=10" in seen["url"] - assert "task=ts_forecasting" in seen["url"] - assert len(rows) == 1 - assert rows[0]["submission_id"] == "s1" - - -def test_submissions_accepts_plain_list_response(): - def handler(request): - return httpx.Response(200, json=[{"submission_id": "x"}]) - - c = _make_client(handler) - rows = c.submissions() - assert rows[0]["submission_id"] == "x" - - -def test_results_returns_resultrow_objects(): - def handler(request): - return httpx.Response(200, json={"results": [{ - "round_id": 5, - "submission_id": "s5", - "task_name": "ts_forecasting", - "prompt_id": "p1", - "architecture_code": "code", - "scores": {"raw_score": 0.7}, - }]}) - - c = _make_client(handler) - rows = c.results() - assert len(rows) == 1 - assert isinstance(rows[0], ResultRow) - assert rows[0].prompt_id == "p1" - assert rows[0].scores["raw_score"] == 0.7 - - -def test_results_empty_payload_returns_empty_list(): - def handler(request): - return httpx.Response(200, json={"results": None}) - - c = _make_client(handler) - assert c.results() == [] - - -def test_summary_returns_dict(): - def handler(request): - return httpx.Response(200, json={"total": 42, "last_round_id": 7}) - - c = _make_client(handler) - s = c.summary() - assert s["total"] == 42 - assert s["last_round_id"] == 7 - - -def test_summary_handles_non_dict_response(): - def handler(request): - return httpx.Response(200, json=[1, 2, 3]) - - c = _make_client(handler) - assert c.summary() == {} - - -def test_frontier_requires_task(): - def handler(request): - return httpx.Response(200, json={"points": []}) - - c = _make_client(handler) - with pytest.raises(ValueError): - c.frontier("") - - -def test_frontier_returns_points(): - def handler(request): - assert "/tasks/ts_forecasting/frontier" in str(request.url) - return httpx.Response(200, json={"points": [ - {"flops": 1e6, "crps": 0.4}, - {"flops": 5e6, "crps": 0.3}, - ]}) - - c = _make_client(handler) - pts = c.frontier("ts_forecasting") - assert len(pts) == 2 - - -def test_4xx_raises(): - def handler(request): - return httpx.Response(403, json={"error": "bad key"}) - - c = _make_client(handler) - with pytest.raises(httpx.HTTPStatusError): - c.summary() - - -def test_context_manager_closes_owned_client(): - def handler(request): - return httpx.Response(200, json={}) - - transport = httpx.MockTransport(handler) - # Owned client path — no client passed in. - http = httpx.Client(transport=transport) - c = MinerResultsClient("http://x", "k", client=http) - with c: - c.summary() - # The owned-client logic only closes when the client was internally - # created; we passed one in so we must close manually. Just confirm - # close() is idempotent. - c.close() - c.close() diff --git a/tests/test_runner_auth.py b/tests/test_runner_auth.py deleted file mode 100644 index 5fdae1a..0000000 --- a/tests/test_runner_auth.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Auth tests for runner/server.py — confirms metagraph is gone and HMAC -service-key verification is in place.""" - -from __future__ import annotations - -import json -import os - -from fastapi.testclient import TestClient - -from shared.auth import hmac_sign_request - - -SECRET = b"a" * 32 -KEY_ID = "operator" - - -def _client(monkeypatch): - monkeypatch.setenv("RADAR_SERVICE_KEY", SECRET.decode()) - monkeypatch.setenv("RADAR_SERVICE_KEY_ID", KEY_ID) - monkeypatch.delenv("RADAR_LOCALNET", raising=False) - # Reload to pick up env. - import importlib - - import runner.server as srv - importlib.reload(srv) - return TestClient(srv.app) - - -def test_train_rejects_without_hmac_headers(monkeypatch): - client = _client(monkeypatch) - r = client.post("/train", content=b"{}", headers={"Content-Type": "application/json"}) - assert r.status_code == 403 - assert "missing" in r.json()["error"].lower() - - -def test_train_rejects_bad_signature(monkeypatch): - client = _client(monkeypatch) - body = b'{"architecture": "x", "task_name": "ts_forecasting"}' - headers = hmac_sign_request(b"WRONG_SECRET", body, key_id=KEY_ID) - headers["Content-Type"] = "application/json" - r = client.post("/train", content=body, headers=headers) - assert r.status_code == 403 - - -def test_train_rejects_unknown_key_id(monkeypatch): - client = _client(monkeypatch) - body = b'{"architecture": "x", "task_name": "ts_forecasting"}' - headers = hmac_sign_request(SECRET, body, key_id="not-operator") - headers["Content-Type"] = "application/json" - r = client.post("/train", content=body, headers=headers) - assert r.status_code == 403 - - -def test_train_503_when_service_key_unset(monkeypatch): - monkeypatch.delenv("RADAR_SERVICE_KEY", raising=False) - monkeypatch.delenv("RADAR_LOCALNET", raising=False) - import importlib - - import runner.server as srv - importlib.reload(srv) - client = TestClient(srv.app) - body = b'{"architecture": "x", "task_name": "ts_forecasting"}' - headers = hmac_sign_request(SECRET, body, key_id=KEY_ID) - headers["Content-Type"] = "application/json" - r = client.post("/train", content=body, headers=headers) - assert r.status_code == 503 - - -def test_health_no_auth(monkeypatch): - client = _client(monkeypatch) - r = client.get("/health") - assert r.status_code == 200 - assert r.json() == {"status": "ok"} - - -def test_localnet_skips_auth(monkeypatch): - monkeypatch.setenv("RADAR_LOCALNET", "1") - monkeypatch.delenv("RADAR_SERVICE_KEY", raising=False) - import importlib - - import runner.server as srv - importlib.reload(srv) - client = TestClient(srv.app) - body = json.dumps({ - "architecture": "class M: pass", - "task_name": "ts_forecasting", - }).encode() - r = client.post( - "/train", content=body, - headers={"Content-Type": "application/json"}, - ) - # 202 (accepted) — auth was skipped. We don't actually train - # anything because run_sandbox would need a real sandbox; the test - # just confirms auth doesn't reject. - assert r.status_code == 202 - - -def test_no_bittensor_or_metagraph_symbols(): - """Ensure the metagraph machinery is gone from the module.""" - import runner.server as srv - assert not hasattr(srv, "_load_metagraph") - assert not hasattr(srv, "_metagraph_cache") - src = open(srv.__file__).read() - assert "import bittensor" not in src diff --git a/validator/collection.py b/validator/collection.py index 3680f53..2e82f10 100644 --- a/validator/collection.py +++ b/validator/collection.py @@ -265,7 +265,6 @@ async def _run_single_agent( result.get("reasoning", "") or "", _REASONING_MAX_BYTES, ), tool_calls=_normalise_tool_calls(result.get("tool_calls", [])), - prompt_id=str(result.get("prompt_id", "") or "")[:64], ) agent_log = _truncate_text( result.get("agent_log", "") or "", _TRACE_MAX_BYTES, @@ -276,7 +275,6 @@ async def _run_single_agent( "reasoning": proposal.reasoning, "tool_calls": proposal.tool_calls, "agent_behavior": agent_behavior, - "prompt_id": proposal.prompt_id, } r2.upload_json( f"round_{round_id}/proposals/{uid}.json", @@ -286,7 +284,6 @@ async def _run_single_agent( "motivation": proposal.motivation, "reasoning": proposal.reasoning, "tool_calls": proposal.tool_calls, - "prompt_id": proposal.prompt_id, "agent_log": agent_log, "agent_behavior": agent_behavior, }, @@ -390,15 +387,7 @@ async def run_and_collect_agents( # Build DB client if not provided if db_client is None: - if Config.SERVICE_KEY: - db_client = DatabaseClient( - Config.DB_API_URL, - service_secret=Config.SERVICE_KEY.encode(), - key_id=Config.SERVICE_KEY_ID, - api_key=Config.DB_API_KEY, - ) - else: - db_client = DatabaseClient(Config.DB_API_URL, wallet) + db_client = DatabaseClient(Config.DB_API_URL, wallet) # Pre-fetch all agent bundles for my assigned miners (concurrently). async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: @@ -470,9 +459,6 @@ async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: tool_calls=_normalise_tool_calls( data.get("tool_calls", []), ), - prompt_id=str( - data.get("prompt_id", "") or "" - )[:64], ) agent_meta[uid] = { "agent_log": _truncate_text( @@ -481,7 +467,6 @@ async def _prefetch(uid: int) -> tuple[int, Optional[dict]]: ), "reasoning": proposals[uid].reasoning, "tool_calls": proposals[uid].tool_calls, - "prompt_id": proposals[uid].prompt_id, "agent_behavior": data.get("agent_behavior", {}) if isinstance(data.get("agent_behavior"), dict) else {}, diff --git a/validator/coordinator.py b/validator/coordinator.py index c68cd4a..6973bc4 100644 --- a/validator/coordinator.py +++ b/validator/coordinator.py @@ -18,7 +18,7 @@ import httpx from config import Config -from shared.auth import sign_request, hmac_sign_request +from shared.auth import sign_request from shared.protocol import Proposal, TrainerRequest, TrainerReady, TrainerRelease if TYPE_CHECKING: @@ -143,13 +143,6 @@ def __init__( self.metagraph = metagraph self.r2 = r2 self.my_uid = my_uid - # Off-chain deploy: when no wallet, sign outbound trainer / proxy - # requests with the shared HMAC service key. The trainer image - # (runner/server.py) verifies the same way. - self._service_secret = ( - Config.SERVICE_KEY.encode() if (wallet is None and Config.SERVICE_KEY) else None - ) - self._service_key_id = Config.SERVICE_KEY_ID # Optional dual-write store (TEN-240 Phase 7). When present, JSON # artifacts (dispatch records, frontier snapshots) fan out to both # R2 and Hippius. Falls back to R2-only when None — historical @@ -170,15 +163,6 @@ def __init__( self._reverify_tasks: dict[int, list[asyncio.Task]] = {} self._targon_client = None - def _sign(self, body: bytes) -> dict[str, str]: - """Sign an outbound HTTP body. HMAC in off-chain deploys, Epistula - otherwise.""" - if self._service_secret is not None: - return hmac_sign_request( - self._service_secret, body, key_id=self._service_key_id, - ) - return sign_request(self.wallet, body) - def compute_my_jobs( self, block_hash: str, @@ -310,7 +294,7 @@ async def _dispatch_one( # Sign fresh every attempt so the Epistula timestamp # stays inside the EPISTULA_TIMESTAMP_TOLERANCE window # (default 120s; tunable via RADAR_EPISTULA_TOLERANCE). - headers = self._sign(payload) + headers = sign_request(self.wallet, payload) headers["Content-Type"] = "application/json" resp = await client.post( f"{url.rstrip('/')}/train", @@ -622,10 +606,7 @@ async def write_dispatch_record(self, round_id: int, results: list[TrainingResul — the dispatching validator is the only entity that knows the opaque submission_id minted at dispatch. """ - hotkey = ( - self.wallet.hotkey.ss58_address - if self.wallet is not None else self._service_key_id - ) + hotkey = self.wallet.hotkey.ss58_address key = f"round_{round_id}/dispatch/vali_{hotkey}.json" hotkeys = self.metagraph.hotkeys if hasattr(self.metagraph, "hotkeys") else [] records = [] @@ -761,7 +742,7 @@ async def prepare_trainers( for uid, commitment in with_listener.items(): try: url = f"{commitment.listener_url.rstrip('/')}/prepare" - fresh_headers = self._sign(body) + fresh_headers = sign_request(self.wallet, body) fresh_headers["Content-Type"] = "application/json" resp = await client.post(url, content=body, headers=fresh_headers) if resp.status_code < 400: @@ -873,17 +854,16 @@ async def release_trainers( if not commitment or not commitment.listener_url: continue try: - mg_hotkeys = getattr(self.metagraph, "hotkeys", None) or [] release = TrainerRelease( round_id=round_id, miner_hotkey=( - mg_hotkeys[uid] - if uid < len(mg_hotkeys) + self.metagraph.hotkeys[uid] + if uid < len(self.metagraph.hotkeys) else "" ), ) body = release.to_json().encode() - headers = self._sign(body) + headers = sign_request(self.wallet, body) headers["Content-Type"] = "application/json" url = f"{commitment.listener_url.rstrip('/')}/release" await client.post(url, content=body, headers=headers) @@ -923,10 +903,9 @@ async def _verify_ready(self, ready_msg, uid: int) -> tuple[bool, str, bool]: """ if Config.HOSTING_BACKEND == "targon": from validator.trainer_verify import verify_trainer - mg_hotkeys = getattr(self.metagraph, "hotkeys", None) or [] hotkey = ( - mg_hotkeys[uid] - if uid < len(mg_hotkeys) else ready_msg.miner_hotkey + self.metagraph.hotkeys[uid] + if uid < len(self.metagraph.hotkeys) else ready_msg.miner_hotkey ) result = await verify_trainer( ready=ready_msg, @@ -959,10 +938,9 @@ async def reverify_running( if Config.HOSTING_BACKEND != "targon": return True # No mid-run check on Basilica. from validator.trainer_verify import reverify_workload - mg_hotkeys = getattr(self.metagraph, "hotkeys", None) or [] miner_hotkey = ( - mg_hotkeys[miner_uid] - if miner_uid < len(mg_hotkeys) else ready_msg.miner_hotkey + self.metagraph.hotkeys[miner_uid] + if miner_uid < len(self.metagraph.hotkeys) else ready_msg.miner_hotkey ) result = await reverify_workload( ready=ready_msg, diff --git a/validator/neuron.py b/validator/neuron.py index cd509e8..e08e40c 100644 --- a/validator/neuron.py +++ b/validator/neuron.py @@ -16,25 +16,28 @@ import importlib.util import logging import os +import random import threading -import time import traceback from typing import Optional +import bittensor as bt import uvicorn from config import Config from shared.challenge import ( generate_challenge, round_start_block, current_phase, select_task, ) +from shared.commitment import read_miner_commitments from shared.database import DataElement from shared.db_client import DatabaseClient -from shared.protocol import Proposal +from shared.protocol import Challenge, Proposal from shared.provenance import detect_components from shared.scoring import ( - apply_round_metadata, compute_penalties, ema_update, score_round, + apply_round_metadata, compute_penalties, ema_update, + score_round, scores_to_weights, ) -from shared.task import load_enabled_tasks +from shared.task import TaskSpec, load_task, load_enabled_tasks from validator.collection import run_and_collect_agents from validator.coordinator import ( TrainingCoordinator, compute_assignments, compute_fallback, @@ -50,25 +53,81 @@ logger = logging.getLogger(__name__) +def compute_live_validator_uids( + metagraph, + miner_uids: Optional[set[int]] = None, + current_block: Optional[int] = None, + stale_blocks: int = 600, +) -> list[int]: + """Filter ``metagraph.validator_permit`` to live, non-miner validators. + + A UID counts as a live validator iff: + * it has ``validator_permit=True`` on chain, AND + * it is not running as a miner this round (no commitment), AND + * its on-chain ``last_update`` is within ``stale_blocks`` of + ``current_block``. + + ``miner_uids`` / ``current_block`` are optional; when absent, the + corresponding check is skipped. A ``last_update`` of 0 is treated as + "never updated" and is not filtered (bootstrap case). + """ + permits = getattr(metagraph, "validator_permit", None) + if permits is None: + return [] + + last_update = getattr(metagraph, "last_update", None) + miners = miner_uids or set() + result: list[int] = [] + for uid in range(metagraph.n): + if uid >= len(permits) or not permits[uid]: + continue + if uid in miners: + continue + if ( + current_block is not None + and last_update is not None + and uid < len(last_update) + ): + try: + last = int(last_update[uid]) + except (TypeError, ValueError): + last = 0 + if last > 0 and current_block - last > stale_blocks: + continue + result.append(uid) + return result + + def get_my_assignments( all_uids: list[int], validator_uids: list[int], my_uid: int, seed: int, ) -> list[int]: - """Single-validator deployment: this validator owns every UID. + """Deterministic assignment of UIDs to validators. - Kept as a stub so collection.py's pass-through still works. + Shuffle all_uids with seed, round-robin across sorted validator_uids. + Returns the UIDs assigned to my_uid. """ - return list(all_uids) + if not validator_uids or my_uid not in validator_uids: + return list(all_uids) + + rng = random.Random(seed) + shuffled = list(all_uids) + rng.shuffle(shuffled) + + sorted_validators = sorted(validator_uids) + my_idx = sorted_validators.index(my_uid) + + return [uid for i, uid in enumerate(shuffled) if i % len(sorted_validators) == my_idx] class Validator: - """Radar validator — 3-phase pipeline (off-chain, single-validator).""" + """Radar subnet validator — 3-phase pipeline.""" - def __init__(self, config): + def __init__(self, config: bt.Config): self.config = config - self.netuid = getattr(config, "netuid", 0) + self.netuid = config.netuid # Fail-fast on Targon misconfig — operator gets a clear error at # startup instead of a stack trace mid-round when verify fires. @@ -84,11 +143,10 @@ def __init__(self, config): "Without a pinned digest the verify chain becomes a no-op — refusing to start." ) - if not Config.SERVICE_KEY: - raise RuntimeError( - "RADAR_SERVICE_KEY must be set (shared HMAC secret for " - "service-to-service auth)." - ) + # Bittensor components + self.wallet = bt.Wallet(config=config) + self.subtensor = bt.Subtensor(config=config) + self.metagraph = self.subtensor.metagraph(self.netuid) # Tasks — load all enabled tasks for multi-task rounds self.tasks = load_enabled_tasks(Config.ENABLED_TASKS) @@ -105,9 +163,7 @@ def __init__(self, config): "centralized DB server (e.g. http://:8090)." ) self.db_client = DatabaseClient( - db_url=Config.DB_API_URL, - service_secret=Config.SERVICE_KEY.encode(), - key_id=Config.SERVICE_KEY_ID, + db_url=Config.DB_API_URL, wallet=self.wallet, api_key=Config.DB_API_KEY, ) @@ -197,10 +253,11 @@ def __init__(self, config): ) # Training coordinator + my_uid = self._my_uid() if self.r2: self.coordinator = TrainingCoordinator( - wallet=None, metagraph=None, - r2=self.r2, my_uid=-1, + wallet=self.wallet, metagraph=self.metagraph, + r2=self.r2, my_uid=my_uid, artifact_store=self.artifact_store, ) else: @@ -215,8 +272,8 @@ def __init__(self, config): # Configure proxy set_proxy_config( db_api_url=Config.DB_API_URL, - wallet=None, - metagraph=None, + wallet=self.wallet, + metagraph=self.metagraph, api_key=Config.DB_API_KEY, ) @@ -226,6 +283,14 @@ def __init__(self, config): "enabled" if self.r2 else "disabled", ) + def _my_uid(self) -> int: + """Get this validator's UID.""" + hotkey = self.wallet.hotkey.ss58_address + hotkeys = self.metagraph.hotkeys + if hotkeys is not None and hotkey in hotkeys: + return hotkeys.index(hotkey) + return -1 + def _init_hippius(self): """Construct the Hippius client when opted in; return None otherwise.""" if not Config.HIPPIUS_ENABLED: @@ -248,6 +313,19 @@ def _init_hippius(self): logger.warning("Hippius client init failed: %s", e) return None + def _get_validator_uids( + self, + miner_uids: Optional[set[int]] = None, + current_block: Optional[int] = None, + ) -> list[int]: + """Instance wrapper for :func:`compute_live_validator_uids`.""" + return compute_live_validator_uids( + self.metagraph, + miner_uids=miner_uids, + current_block=current_block, + stale_blocks=Config.VALIDATOR_STALE_BLOCKS, + ) + def _proxy_base_url(self) -> str: """Return the externally-reachable base URL for the validator proxy. @@ -337,14 +415,9 @@ def _run(): logger.info("Proxy server started on port %d", self.proxy_port) async def _wait_until_block(self, target_block: int): - """Wait until the wall-clock-derived block counter reaches target_block. - - Off-chain deploys use ``time.time() // 12`` as a stand-in for block - height so timing stays compatible with the per-phase block-window - constants in ``config.py``. - """ + """Wait until the chain reaches target_block.""" while True: - current = int(time.time() // 12) + current = self.subtensor.block if current >= target_block: return remaining = (target_block - current) * 12 @@ -359,11 +432,15 @@ async def _cache_training_metas( if the cache write fails.""" if not metas: return + hotkeys = self.metagraph.hotkeys if hasattr(self.metagraph, "hotkeys") else [] cached = 0 for uid, meta in metas.items(): if not isinstance(meta, dict): continue - hk = meta.get("miner_hotkey", "") + hk = ( + meta.get("miner_hotkey") + or (hotkeys[uid] if uid < len(hotkeys) else "") + ) if not hk: continue try: @@ -432,12 +509,17 @@ async def _publish_submission_reveal( async def run_round(self): """Execute one full 3-phase round.""" # ── SETUP ────────────────────────────────────────────── - # No chain — round_start is a wall-clock-derived integer so the - # deterministic seed stays stable for ~12s like a block. - current_block = int(time.time() // 12) + self.metagraph.sync() + set_metagraph(self.metagraph) + hotkeys = self.metagraph.hotkeys or [] + hotkey_map = { + hotkeys[uid]: uid + for uid in range(self.metagraph.n) + if uid < len(hotkeys) + } + set_hotkey_map(hotkey_map) + current_block = self.subtensor.block round_start = round_start_block(current_block, Config.ROUND_INTERVAL_BLOCKS) - set_metagraph(None) - set_hotkey_map({}) block_hash = hashlib.sha256(str(round_start).encode()).hexdigest() # Select task for this round (deterministic from block hash) @@ -503,11 +585,20 @@ async def run_round(self): ) # ── PHASE A: RUN AGENTS + COLLECT ──────────────────── - # Off-chain deploy: miner identity comes from operator-CLI tokens, - # not on-chain ImageCommitments. Commitments stay empty; downstream - # code paths treat that as "every registered miner is eligible". - commitments: dict = {} - validator_uids: list[int] = [] + commitments = read_miner_commitments(self.subtensor, self.netuid, self.metagraph) + try: + current_block = self.subtensor.get_current_block() + except Exception as e: + logger.warning("Failed to read current block for liveness filter: %s", e) + current_block = None + validator_uids = self._get_validator_uids( + miner_uids=set(commitments.keys()), + current_block=current_block, + ) + logger.info( + "Live validators this round: %s (filtered miners + stale last_update)", + validator_uids, + ) if not self.r2: logger.warning("No R2 configured — skipping Phase A/B/C") @@ -525,17 +616,16 @@ async def _no_endpoints(): prepare_coro = _no_endpoints() collect_coro = run_and_collect_agents( - wallet=None, - metagraph=None, + wallet=self.wallet, + metagraph=self.metagraph, challenge_json=challenge.to_json(), round_id=challenge.round_id, seed=challenge.seed, r2=self.r2, - my_uid=-1, + my_uid=self._my_uid(), validator_uids=validator_uids, commitments=commitments, get_my_assignments_fn=get_my_assignments, - db_client=self.db_client, ) dynamic_endpoints, (submissions, agent_meta) = await asyncio.gather( @@ -573,17 +663,30 @@ async def _no_endpoints(): logger.warning("No coordinator configured — skipping Phase B/C") return - # Single-validator deploy: we own every dispatch. - my_uid = -1 - dispatch_validators = [my_uid] + my_uid = self._my_uid() + dispatch_validators = list(validator_uids) + if my_uid >= 0 and my_uid not in dispatch_validators: + logger.info( + "My UID %d not in validator_uids %s — adding self as safety-net dispatcher", + my_uid, dispatch_validators, + ) + dispatch_validators.append(my_uid) all_jobs = compute_assignments( block_hash, filtered, - miner_uids=list(filtered.keys()), + miner_uids=list(commitments.keys()), validator_uids=dispatch_validators, round_id=challenge.round_id, ) - my_jobs = list(all_jobs) + my_jobs = [j for j in all_jobs if j.dispatcher == my_uid] + + if not my_jobs and all_jobs: + logger.warning( + "Round-robin assigned 0 of %d jobs to my_uid=%d — " + "dispatching all jobs (other dispatchers may be offline)", + len(all_jobs), my_uid, + ) + my_jobs = list(all_jobs) trainer_endpoints = dynamic_endpoints logger.info( @@ -825,7 +928,7 @@ async def _no_endpoints(): await self._publish_submission_reveal( round_id=challenge.round_id, uid_to_sid=uid_to_sid, - hotkeys=[], + hotkeys=self.metagraph.hotkeys or [], ) primary_obj = round_task.primary_objective primary_name = primary_obj.name if primary_obj else "metric" @@ -914,10 +1017,10 @@ async def _no_endpoints(): # can attach the CID. Empty dict when disabled, no client, or # publish failed — never raises. substrate_cids = await run_substrate_publish_step( - hippius=self.hippius, wallet=None, + hippius=self.hippius, wallet=self.wallet, challenge=challenge, eval_results=eval_results, training_metas=training_metas, commitments=commitments, - metagraph=None, my_uid=-1, + metagraph=self.metagraph, my_uid=self._my_uid(), current_block=current_block, task_name=task_name, block_hash=block_hash, netuid=int(self.netuid), ) @@ -935,8 +1038,8 @@ async def _no_endpoints(): gate_passed += 1 proposal = filtered.get(uid, Proposal()) + miner_hk = self.metagraph.hotkeys[uid] if uid < len(self.metagraph.hotkeys) else "" meta = agent_meta.get(uid, {}) - miner_hk = meta.get("miner_hotkey", "") if isinstance(meta, dict) else "" element = DataElement( name=f"round_{challenge.round_id}_miner_{uid}", code=proposal.code, @@ -952,11 +1055,6 @@ async def _no_endpoints(): agent_behavior=meta.get("agent_behavior", {}), task=task_name, round_id=challenge.round_id, - prompt_id=( - meta.get("prompt_id", "") - or getattr(proposal, "prompt_id", "") - or "" - ), ) # Write to centralized DB. When this validator published @@ -965,7 +1063,7 @@ async def _no_endpoints(): new_idx = await self.db_client.add_experiment( element.to_dict(), substrate_cid=substrate_cids.get(uid, ""), - validator_hotkey="", + validator_hotkey=self.wallet.hotkey.ss58_address, ) if new_idx is not None: element.index = new_idx @@ -1021,9 +1119,8 @@ async def _no_endpoints(): else: logger.warning("Round scores empty — no eval results passed size gate") - # EMA on raw scores. No on-chain weight set — the EMA dict is the - # source of truth for the dashboard and operator-CLI reporting. - all_uids = sorted(set(self.ema_scores) | set(round_scores)) + # EMA on raw scores, softmax applied once in _set_weights() + all_uids = list(range(self.metagraph.n)) self.ema_scores = ema_update(self.ema_scores, round_scores, all_uids, Config.EMA_ALPHA) nonzero_ema = {uid: s for uid, s in self.ema_scores.items() if s > 0} logger.info( @@ -1031,6 +1128,7 @@ async def _no_endpoints(): Config.EMA_ALPHA, len(nonzero_ema), {uid: f"{s:.4f}" for uid, s in sorted(nonzero_ema.items())} if nonzero_ema else "all zero", ) + self._set_weights() # Write frontier to centralized DB + R2 await self.db_client.update_frontier( @@ -1050,6 +1148,34 @@ async def _no_endpoints(): len(round_scores), nonzero_scored, ) + def _set_weights(self): + """Set weights on chain — softmax on EMA scores.""" + if not self.ema_scores: + logger.warning("No EMA scores — skipping weight setting") + return + try: + uids, weights = scores_to_weights(self.ema_scores, Config.SOFTMAX_TEMPERATURE) + nonzero_weights = { + uid: f"{w:.4f}" for uid, w in zip(uids, weights) if w > 0 + } + if nonzero_weights: + logger.info( + "Setting weights for %d UIDs (%d nonzero): %s", + len(uids), len(nonzero_weights), nonzero_weights, + ) + else: + logger.warning( + "Setting all-zero weights for %d UIDs (no miner scored this round)", + len(uids), + ) + self.subtensor.set_weights( + netuid=self.netuid, wallet=self.wallet, + uids=uids, weights=weights, + ) + logger.info("Weights set successfully on chain") + except Exception as e: + logger.error("Failed to set weights: %s", e) + async def run(self): """Main validator loop.""" self.start_proxy_server() @@ -1064,7 +1190,7 @@ async def run(self): logger.warning("Startup orphan reap failed: %s", e) while True: try: - current_block = int(time.time() // 12) + current_block = self.subtensor.block round_start = round_start_block(current_block, Config.ROUND_INTERVAL_BLOCKS) phase = current_phase( current_block, round_start, @@ -1110,13 +1236,15 @@ async def run(self): await asyncio.sleep(60) -def get_config(): +def get_config() -> bt.Config: parser = argparse.ArgumentParser(description="Radar Validator") parser.add_argument("--netuid", type=int, default=1) parser.add_argument("--task", type=str, default="ml_training") parser.add_argument("--db_dir", type=str, default="./experiments") parser.add_argument("--db_port", type=int, default=Config.PROXY_PORT) - return parser.parse_args() + bt.Wallet.add_args(parser) + bt.Subtensor.add_args(parser) + return bt.Config(parser) def main(): diff --git a/validator/substrate_publisher.py b/validator/substrate_publisher.py index 7d36652..1454e71 100644 --- a/validator/substrate_publisher.py +++ b/validator/substrate_publisher.py @@ -261,11 +261,6 @@ async def run_substrate_publish_step( """ if hippius is None or not eval_results: return {} - if wallet is None: - # Off-chain deploy — Phase C bundles are SR25519-signed by the - # validator hotkey. Without a wallet there's nothing to sign with, - # so we skip substrate publishing entirely. - return {} try: records = await build_phase_c_records( wallet=wallet, challenge=challenge,