diff --git a/cascade/audit/main.py b/cascade/audit/main.py index b73b9ff5..882886f4 100644 --- a/cascade/audit/main.py +++ b/cascade/audit/main.py @@ -17,6 +17,10 @@ with torch + the ``timestral`` extra installed, re-forecasts a sample of windows from the public checkpoints. It reports; it never changes consensus. +* ``cascade-audit decided`` — the drift guard: ``chain.toml`` vs + ``decided_votes.toml`` (the fleet's decided on-chain votes and the live + image pins) and, with ``--state-dir``, vs this node's persisted lock-ins. + Prints the exact lines to type; exit 1 on drift. Reads only. ``--tier`` picks the depth (0 = seconds/CPU, 1 = +corpus re-derivation, 2 = +re-training, experimental). ``--json`` emits machine-readable results. @@ -116,6 +120,21 @@ def _build_parser() -> argparse.ArgumentParser: help="--reproduce forecast device (default cpu).") mkt_p.add_argument("--workdir", type=Path, default=Path("./_audit_work/marketcast")) mkt_p.add_argument("--json", action="store_true") + dec_p = sub.add_parser( + "decided", + help="Drift guard: compare chain.toml with decided_votes.toml (the fleet's decided " + "on-chain votes and live image pins) and, with --state-dir, with this node's " + "activation_*_state.json lock-ins. Prints the exact lines to type. Exit 1 on drift.") + dec_p.add_argument("--config", type=Path, default=None, + help="chain.toml path (default: the repo's).") + dec_p.add_argument("--decided", type=Path, default=None, + help="decided_votes.toml path (default: beside --config).") + dec_p.add_argument("--state-dir", type=Path, default=None, + help="A service's state dir (its activation_*_state.json files): also " + "report lock-ins this node holds that the record lacks or disputes.") + dec_p.add_argument("--as-of-block", type=int, default=None, + help="Judge 'applied' against this block (default: the record's).") + dec_p.add_argument("--json", action="store_true") round_p = sub.add_parser("round", help="Audit one round by id.") round_p.add_argument("round_id", help="Round id (the base seed / manifest round_id).") sub.add_parser("latest", help="Audit the newest published receipt.") @@ -638,6 +657,27 @@ def audit_receipt( return results +def _decided(args) -> int: + """``cascade-audit decided`` — the decided-votes drift guard (reads only).""" + from ..shared import decided as D + + cfg = load_chain_config(args.config) + decided_path = args.decided or D.default_decided_path(args.config) + record = D.load_decided(decided_path) + findings = D.drift_report(cfg, record, as_of_block=args.as_of_block) + if args.state_dir is not None: + findings += D.state_report(D.read_state_records(args.state_dir), record) + if args.json: + print(json.dumps({"ok": not any(f.fails for f in findings), + "as_of_block": int(args.as_of_block or record.as_of_block), + "decided": str(decided_path), + "findings": [f.as_dict() for f in findings]}, indent=2, + sort_keys=True)) + else: + print(D.render(findings)) + return 1 if any(f.fails for f in findings) else 0 + + def main(argv: list[str] | None = None) -> int: # DEC-CA-0067: this process votes on / acts on the v2b vote — a config # whose pinned feature is missing or wrong refuses to load @@ -653,6 +693,8 @@ def main(argv: list[str] | None = None) -> int: return _marketcast(args) if args.command == "bench": return _bench(args) + if args.command == "decided": + return _decided(args) logging.basicConfig(level=args.log_level, format="%(asctime)s %(levelname)s %(name)s: %(message)s") cfg = load_chain_config(args.config) diff --git a/cascade/shared/activation.py b/cascade/shared/activation.py index 5a25834f..d8b80e16 100644 --- a/cascade/shared/activation.py +++ b/cascade/shared/activation.py @@ -1163,6 +1163,12 @@ def resolve_activation( log.info("activation: %s LOCKED IN at boundary %d (%.1f%% of eligible stake, %d/%d " "validators) — rollover at block %d", feature, boundary, 100 * t.ratio, len(t.signed), len(t.eligible), act) + # the drift guard's "type this" (cascade.shared.decided): the decided + # block belongs in the release chain.toml + decided_votes.toml once it + # has applied — a hand edit on one node is how main drifted (2026-11-13) + from .decided import type_hint + + log.warning("activation: %s", type_hint(feature, act)) else: log.info("activation: %s at boundary %d: %.1f%% of eligible stake signed (%d/%d " "validators, threshold %.0f%%) — not yet", feature, boundary, 100 * t.ratio, diff --git a/cascade/shared/decided.py b/cascade/shared/decided.py new file mode 100644 index 00000000..2d0172d0 --- /dev/null +++ b/cascade/shared/decided.py @@ -0,0 +1,403 @@ +"""Decided on-chain votes vs the shipped ``chain.toml`` — the drift guard. + +Stake-weighted activation (DEC-CA-0045, :mod:`cascade.shared.activation`) +lets the fleet DECIDE a feature's block on chain. Once decided, the block is +meant to be TYPED into the release ``chain.toml`` (the owner override): a +typed block is behaviour-identical to the resolved one and frees the +feature's segment of the 128-byte readiness note. Until 2026-11-13 that typing +happened only on the operator's deployed file and never reached ``main`` — +an external validator that pulled ``main`` overflowed its note and refused to +start (OPSLOG 2026-11-13; PR #457 typed the v2b block). + +This module makes the gap visible in CI and on the box: + +* :data:`DECIDED_FILE` (``decided_votes.toml`` beside ``chain.toml``) is the + one place in git that records what the fleet decided — every vote's + feature name, lock block and activation block, which ``chain.toml`` key + types it, and the worker-image pins the network trains on. +* :func:`drift_report` compares a loaded config against that record and + returns findings: a decided block the config has not typed (with the + exact line to type), a typed block that disagrees with the decision, a + pin that drifted, a record entry whose feature name the config no longer + computes (changed vote params = a fresh vote; the record is stale). +* :func:`state_report` compares a node's persisted ``activation_*_state.json`` + records against the record: a vote this node resolved that the record + lacks, or resolved to another block (the rolling-era-king case: nodes + that restarted late adopted a different block). +* :func:`type_hint` is the one-line "type this" the resolver logs at lock-in. + +``cascade-audit decided`` runs both reports; ``tests/unit/test_decided_votes.py`` +runs :func:`drift_report` on the repo's own files, so a vote that resolves +and is not typed turns CI red on the next PR, whatever that PR is about. + +Nothing here changes consensus: it reads config and state and reports. +""" +from __future__ import annotations + +import json +import tomllib +from collections.abc import Iterable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from . import activation as A +from .config import ChainConfig + +DECIDED_FILE = "decided_votes.toml" + +# Feature-name prefix → the chain.toml key (dotted section path) that types it. +# ``cascade.shared.activation`` names every feature ``[-]``; +# a new voted feature adds one row here and one ``*_feature`` entry in +# :func:`configured_features`. +TYPED_KEY_BY_PREFIX: tuple[tuple[str, str], ...] = ( + ("rolling-era-king", "round.rolling_from_block"), + ("forfeit-", "scoring.forfeit_from_block"), + ("margin-v2-", "scoring.margin_v2_from_block"), + ("prelaunch-", "v2.prelaunch_burn_from_block"), + ("v2b-", "v2.image_decoupled_from_block"), + ("lad-", "eval.shared_ladder_from_block"), +) + +# The image pins the network trains on; a release that moves one without +# updating the record (or the record without the release) drifts. +PIN_KEYS: tuple[str, ...] = ( + "training.train_image_digest", + "round.funded_pod_image", + "round.worker_code_fingerprint", + "v2.training.train_image_digest", +) + +# The activation_*_state.json files a node keeps, by feature prefix. +STATE_FILES: tuple[tuple[str, str], ...] = ( + ("rolling-era-king", "activation_state.json"), + ("forfeit-", "activation_forfeit_state.json"), + ("margin-v2-", "activation_margin_v2_state.json"), + ("prelaunch-", "activation_prelaunch_burn_state.json"), + ("v2b-", "activation_image_decoupled_state.json"), + ("lad-", "activation_shared_ladder_state.json"), +) + + +def typed_key_for(feature: str) -> str: + """The dotted ``chain.toml`` key that types ``feature``, or ``""`` when the + code knows no such feature (a retired vote's state file, say).""" + for prefix, key in TYPED_KEY_BY_PREFIX: + if feature == prefix or (prefix.endswith("-") and feature.startswith(prefix)): + return key + return "" + + +def toml_line(key: str, value: Any) -> str: + """``[section]`` + ``key = value`` for a dotted key, as it is typed.""" + section, _, name = key.rpartition(".") + rendered = json.dumps(value) if isinstance(value, str) else str(int(value)) + return f"[{section}]\n{name} = {rendered}" + + +def type_hint(feature: str, block: int) -> str: + """The one-line instruction logged at lock-in and printed by the audit.""" + key = typed_key_for(feature) + if not key: + return (f"{feature} decided at block {int(block)} — no chain.toml key types this " + "feature in this release") + return (f"DECIDED: {feature} applies at block {int(block)} — once that block has passed, " + f"type `{key.rpartition('.')[2]} = {int(block)}` under [{key.rpartition('.')[0]}] " + f"in the release chain.toml and record it in {DECIDED_FILE} (PR to main); the " + "typed block is behaviour-identical and frees the feature's note segment") + + +def config_value(cfg: Any, key: str) -> Any: + """Read a dotted key off the loaded config (``v2.training.x`` → + ``cfg.training.v2_contract.x``; everything else walks attributes).""" + if key.startswith("v2.training."): + contract = getattr(getattr(cfg, "training", None), "v2_contract", None) + return getattr(contract, key[len("v2.training."):], None) + obj: Any = cfg + for part in key.split("."): + obj = getattr(obj, part, None) + if obj is None: + return None + return obj + + +# ── the record ────────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class DecidedVote: + feature: str + lock_block: int + activation_block: int + source: str = "" + # False = nodes resolved this vote to DIFFERENT blocks (or the loader + # cannot take the typed value yet); the guard never asks to type it and + # reports the known disagreement instead. ``reason`` says why. + type_it: bool = True + reason: str = "" + + @property + def key(self) -> str: + return typed_key_for(self.feature) + + +@dataclass(frozen=True) +class DecidedRecord: + as_of_block: int + votes: tuple[DecidedVote, ...] = () + pins: dict[str, str] = field(default_factory=dict) + + def vote(self, feature: str) -> DecidedVote | None: + for v in self.votes: + if v.feature == feature: + return v + return None + + +def parse_decided(text: str) -> DecidedRecord: + raw = tomllib.loads(text) + votes = [] + for row in raw.get("vote", ()) or (): + votes.append(DecidedVote( + feature=str(row["feature"]), lock_block=int(row.get("lock_block", 0) or 0), + activation_block=int(row["activation_block"]), source=str(row.get("source", "")), + type_it=bool(row.get("type_it", True)), reason=str(row.get("reason", "")))) + names = [v.feature for v in votes] + if len(set(names)) != len(names): + raise ValueError(f"{DECIDED_FILE}: duplicate feature rows") + for v in votes: + if v.activation_block <= 0: + raise ValueError(f"{DECIDED_FILE}: {v.feature}: activation_block must be > 0") + if v.lock_block and v.lock_block > v.activation_block: + raise ValueError(f"{DECIDED_FILE}: {v.feature}: lock_block {v.lock_block} after " + f"activation_block {v.activation_block}") + pins = {str(k): str(v) for k, v in (raw.get("pins", {}) or {}).items()} + unknown = sorted(set(pins) - set(PIN_KEYS)) + if unknown: + raise ValueError(f"{DECIDED_FILE}: unknown pin key(s) {unknown}; known: {list(PIN_KEYS)}") + return DecidedRecord(as_of_block=int(raw.get("as_of_block", 0) or 0), + votes=tuple(votes), pins=pins) + + +def load_decided(path: Path | str) -> DecidedRecord: + return parse_decided(Path(path).read_text(encoding="utf-8")) + + +def default_decided_path(chain_toml: Path | str | None = None) -> Path: + """``decided_votes.toml`` beside the given ``chain.toml`` (default: the + repo's / package's own).""" + from .config import DEFAULT_CHAIN_TOML + + shipped = Path(DEFAULT_CHAIN_TOML).resolve().parent / DECIDED_FILE + if chain_toml is None: + return shipped + beside = Path(chain_toml).resolve().parent / DECIDED_FILE + # a deployed chain.toml usually has no record beside it: the release's is + # the one to judge it by + return beside if beside.is_file() else shipped + + +# ── what the config itself says ───────────────────────────────────────────── + + +def configured_features(cfg: ChainConfig) -> dict[str, A.FeatureSpec]: + """Every voted feature this config computes, by name — the names the + record must match (changed vote params change the name: a fresh vote).""" + out: dict[str, A.FeatureSpec] = {} + primary = A.primary_feature(cfg) + if primary.name: + out[primary.name] = primary + for fn in (A.forfeit_feature, A.margin_v2_feature, A.prelaunch_burn_feature, + A.image_decoupled_feature, A.shared_ladder_feature): + spec = fn(cfg) + if spec is not None and spec.name: + out[spec.name] = spec + return out + + +# ── findings ──────────────────────────────────────────────────────────────── + +TYPE = "TYPE" # decided, applied, not typed → type it (CI FAIL) +MISMATCH = "MISMATCH" # typed to a block other than the decided one (CI FAIL) +PIN_DRIFT = "PIN_DRIFT" # a pin differs between config and record (CI FAIL) +STALE = "STALE" # record names a feature the config no longer computes (CI FAIL) +PENDING = "PENDING" # decided but not yet applied as of the record's block (info) +HELD = "HELD" # record says type_it = false (nodes disagree / owner decision pending) +DISAGREE = "DISAGREE" # this node resolved a vote to another block than the record (info) +UNRECORDED = "UNRECORDED" # this node resolved a vote the record lacks (CI FAIL on the box) +UNKNOWN = "UNKNOWN" # a state file for a feature this release does not know (info) +OK = "OK" + +FAILING = frozenset({TYPE, MISMATCH, PIN_DRIFT, STALE, UNRECORDED}) + + +@dataclass(frozen=True) +class Finding: + kind: str + feature: str + key: str = "" + expected: Any = None + actual: Any = None + detail: str = "" + fix: str = "" + + @property + def fails(self) -> bool: + return self.kind in FAILING + + def as_dict(self) -> dict[str, Any]: + return {"kind": self.kind, "feature": self.feature, "key": self.key, + "expected": self.expected, "actual": self.actual, "detail": self.detail, + "fix": self.fix} + + +def drift_report(cfg: ChainConfig, record: DecidedRecord, *, + as_of_block: int | None = None) -> list[Finding]: + """Compare a loaded ``chain.toml`` with the decided-votes record. + + ``as_of_block`` (default: the record's) is "now": a decided block at or + before it has APPLIED and must be typed; a later one is pending (typing + it early would give the fleet no notice — see + :func:`cascade.shared.activation.v2b_typed_notice_problem`).""" + now = int(record.as_of_block if as_of_block is None else as_of_block) + features = configured_features(cfg) + out: list[Finding] = [] + for v in record.votes: + key = v.key + if not key: + out.append(Finding(UNKNOWN, v.feature, detail="no chain.toml key types this " + "feature in this release; drop the row or add the mapping")) + continue + typed = int(config_value(cfg, key) or 0) + if v.feature not in features: + # a typed feature keeps its computed name (the spec carries the + # typed block), so a missing name means the params changed or the + # vote was retired — whatever the key says + out.append(Finding(STALE, v.feature, key, v.activation_block, typed, + detail="the config computes no feature of this name (vote " + "params changed = a fresh vote, or the vote was retired); " + f"configured: {sorted(features)}", + fix=f"update or drop the {v.feature!r} row in {DECIDED_FILE}")) + continue + if typed and typed != v.activation_block: + out.append(Finding(MISMATCH, v.feature, key, v.activation_block, typed, + detail="typed block differs from the decided one", + fix=toml_line(key, v.activation_block))) + continue + if typed: + out.append(Finding(OK, v.feature, key, v.activation_block, typed)) + continue + if not v.type_it: + out.append(Finding(HELD, v.feature, key, v.activation_block, typed, + detail=v.reason or "held: not to be typed (see the record)")) + continue + if v.activation_block > now: + out.append(Finding(PENDING, v.feature, key, v.activation_block, typed, + detail=f"applies at {v.activation_block} > as-of {now}; type it " + "once that block has passed")) + continue + out.append(Finding(TYPE, v.feature, key, v.activation_block, typed, + detail=f"decided (lock {v.lock_block}, applied {v.activation_block} " + f"<= as-of {now}) but the config still votes on it", + fix=toml_line(key, v.activation_block))) + recorded = {v.feature for v in record.votes} + for name, spec in features.items(): + if name not in recorded and spec.decided_on_chain: + out.append(Finding(PENDING, name, typed_key_for(name), + detail="open vote, no decision recorded")) + for key in PIN_KEYS: + want = record.pins.get(key) + if want is None: + continue + have = config_value(cfg, key) + have_s = "" if have is None else str(have) + if have_s != want: + out.append(Finding(PIN_DRIFT, "pins", key, want, have_s, + detail="the config's pin differs from the recorded live pin", + fix=toml_line(key, want))) + else: + out.append(Finding(OK, "pins", key, want, have_s)) + return out + + +def read_state_records(state_dir: Path | str) -> dict[str, A.ActivationRecord]: + """Every ``activation_*_state.json`` under ``state_dir`` that parses, by + file name. Reads only; a torn file is skipped (the services own repair).""" + out: dict[str, A.ActivationRecord] = {} + for p in sorted(Path(state_dir).glob("activation_*_state.json")) + \ + [Path(state_dir) / "activation_state.json"]: + if not p.is_file() or p.name in out: + continue + try: + rec = A.ActivationStore(p).load() + except (A.ActivationStateUnreadable, OSError, ValueError): + continue + if rec.feature: + out[p.name] = rec + return out + + +def state_report(records: Iterable[A.ActivationRecord] | dict[str, A.ActivationRecord], + record: DecidedRecord) -> list[Finding]: + """Compare a node's persisted lock-ins with the record: a locked vote the + record lacks must be added; one the record names at another block is a + known disagreement (or a stale record) and is reported, never resolved.""" + recs = list(records.values()) if isinstance(records, dict) else list(records) + out: list[Finding] = [] + for rec in recs: + if not rec.locked: + continue + key = typed_key_for(rec.feature) + if not key: + out.append(Finding(UNKNOWN, rec.feature, actual=rec.activation_block, + detail="this node holds a lock-in for a feature this release " + "does not know (retired vote); nothing to type")) + continue + known = record.vote(rec.feature) + if known is None: + out.append(Finding(UNRECORDED, rec.feature, key, actual=rec.activation_block, + detail=f"this node locked in {rec.feature} (lock " + f"{rec.lock_block}, applies {rec.activation_block}, via " + f"{rec.source}) but {DECIDED_FILE} has no row for it", + fix=record_row(rec))) + elif int(known.activation_block) != int(rec.activation_block): + out.append(Finding(DISAGREE, rec.feature, key, known.activation_block, + rec.activation_block, + detail="this node resolved the vote to a different block than " + "the record names — nodes disagree (or the record is stale); " + "do NOT type either value until the fleet agrees")) + else: + out.append(Finding(OK, rec.feature, key, known.activation_block, + rec.activation_block)) + return out + + +def record_row(rec: A.ActivationRecord) -> str: + """The ``[[vote]]`` row to add to the record for a node's lock-in.""" + return (f"[[vote]]\nfeature = {json.dumps(rec.feature)}\nlock_block = {int(rec.lock_block)}\n" + f"activation_block = {int(rec.activation_block)}\nsource = {json.dumps(rec.source)}") + + +def render(findings: Iterable[Finding]) -> str: + rows = list(findings) + lines = [] + for f in rows: + head = f"[{f.kind}] {f.feature}" + if f.key: + head += f" {f.key}" + if f.kind != OK: + if f.expected is not None: + head += f" decided={f.expected}" + if f.actual is not None: + head += f" config={f.actual}" + if f.detail: + head += f"\n {f.detail}" + if f.fix: + head += "\n → type:\n" + "\n".join(" " + ln for ln in f.fix.splitlines()) + lines.append(head) + bad = [f for f in rows if f.fails] + lines.append("" if not bad else f"{len(bad)} finding(s) need typing — the release chain.toml " + f"and {DECIDED_FILE} via a PR to main; a deployed file at its next quiet restart.") + if not bad: + lines.append("decided votes and pins: in sync.") + return "\n".join(lines) diff --git a/decided_votes.toml b/decided_votes.toml new file mode 100644 index 00000000..77bcb3f0 --- /dev/null +++ b/decided_votes.toml @@ -0,0 +1,71 @@ +# Decided on-chain votes and the live worker-image pins — the drift-guard record. +# +# Stake-weighted activation (DEC-CA-0045) lets the fleet decide a feature's +# block on chain. Once decided and applied, the block is TYPED into the release +# chain.toml: behaviour-identical, and it frees the feature's segment of the +# 128-byte readiness note. This file records what the fleet decided so that +# `tests/unit/test_decided_votes.py` (CI) and `cascade-audit decided` (the box) +# can tell when chain.toml has not caught up. See cascade/shared/decided.py. +# +# Maintenance, every time a vote locks in (the validator logs "DECIDED: …"): +# 1. add or update the [[vote]] row (feature, lock_block, activation_block, source); +# 2. once activation_block has passed, type the block in chain.toml (the row's +# key, from decided.TYPED_KEY_BY_PREFIX) in the same PR; +# 3. bump as_of_block to the boundary you are at. +# A row with type_it = false is HELD: the guard reports it and never asks to +# type it; `reason` says why (nodes disagree, or an owner decision is pending). +# +# Pins: the image digests the network trains on. Moving one in chain.toml +# without updating this file (or vice versa) fails CI. + +as_of_block = 9494100 # Fri 13 Nov 2026 01:23 UTC settlement (the last boundary passed when written) + +[[vote]] +feature = "rolling-era-king" +lock_block = 9133200 +activation_block = 9136800 +source = "config" +type_it = false +reason = "nodes resolved it differently: the operator node 9133200→9136800, one external validator 9374400→9378000 (it restarted late and adopted other notes). Typing either value in main would override a block some nodes decided; the operator types its own on its deployed file only." + +[[vote]] +feature = "forfeit-42b1a486" +lock_block = 9165600 +activation_block = 9166500 +source = "signals" +# typed in chain.toml 2026-11-06 + +[[vote]] +feature = "margin-v2-d30eb443" +lock_block = 9190800 +activation_block = 9194400 +source = "tally" +# typed in chain.toml 2026-11-06 + +[[vote]] +feature = "prelaunch-95-r2" +lock_block = 9486000 +activation_block = 9486900 +source = "tally" +type_it = false +reason = "HELD (owner decision pending, 2026-11-13): decided and applied on every node seen (operator + the external validator's note both carry 9486000:9486900); typing [v2] prelaunch_burn_from_block = 9486900 is behaviour-identical and frees another note segment, but it is a release chain.toml edit the owner has not yet approved. Next one-line PR." + +[[vote]] +feature = "v2b-69630f92" +lock_block = 9486000 +activation_block = 9493200 +source = "tally" +# typed in chain.toml 2026-11-13 (PR #457) + +[[vote]] +feature = "lad-556b1b98" +lock_block = 0 +activation_block = 9494100 +source = "config" +# typed from the release (rc35, PR #455, owner 2026-11-12): never an open vote + +[pins] +"training.train_image_digest" = "sha256:205ff68242ae78fc923c8a5c7d2c4a4eccdd8afc7c7739cae99187475ba96d6f" +"round.funded_pod_image" = "ghcr.io/tensorlink-ai/cascade-worker:worker-v2.0.0-rc15@sha256:5d41d94bc0c0d3ae3dfd042b0cc7e4d900c7d1f48173808436c12bf764a67641" +"round.worker_code_fingerprint" = "2d8261aa40e86d3821f8fbeac0d60a0ac7528434a686c17e204c2b20198e293c" +"v2.training.train_image_digest" = "sha256:5d41d94bc0c0d3ae3dfd042b0cc7e4d900c7d1f48173808436c12bf764a67641" diff --git a/docs/AUDIT.md b/docs/AUDIT.md index 34b1ec93..7c086848 100644 --- a/docs/AUDIT.md +++ b/docs/AUDIT.md @@ -494,6 +494,38 @@ cascade-audit direct-moe --journal-bucket BUCKET --trainer-hotkey TRAINER_SS58 - A private panel is only revealed after the trainer rotates it out, so a recent decision's private half WARNs until then; the public half reproduces at once. +## Decided votes vs `chain.toml` (the drift guard) + +Stake-weighted activation (DEC-CA-0045) lets the fleet decide a feature's +block on chain. Once decided and applied, the block is **typed** into the +release `chain.toml`: behaviour-identical, and it frees that feature's segment +of the 128-byte readiness note. Until 2026-11-13 the typing happened only on +the operator's deployed file; `main` lagged, and a validator that pulled `main` +overflowed its note and refused to start. + +`decided_votes.toml` (beside `chain.toml`) records what the fleet decided — +each vote's feature name, lock block, activation block and the key that types +it — plus the worker-image pins the network trains on. + +``` +cascade-audit decided # the repo's chain.toml vs the record +cascade-audit decided --config /path/chain.toml # a deployed file +cascade-audit decided --state-dir /path/to/state # + this node's activation_*_state.json +``` + +Findings: `TYPE` (decided, applied, not typed — the exact lines to type are +printed), `MISMATCH` (typed to another block), `PIN_DRIFT` (a pin moved on one +side), `STALE` (a row the config no longer computes: changed params = a fresh +vote), `UNRECORDED` (this node locked in a vote the record lacks — the +`[[vote]]` row to add is printed); informational: `PENDING` (not yet applied), +`HELD` (`type_it = false`, with the reason — nodes disagree, or an owner +decision is pending), `DISAGREE` (this node resolved a vote to a different +block than the record), `UNKNOWN` (a retired feature's state file). Exit 1 on +any of the first five. `tests/unit/test_decided_votes.py` runs the same check +on the repo's own files, so CI goes red when a vote resolves and is not typed. +The validator logs `DECIDED: … type … in the release chain.toml` at every +lock-in. Everything here reads; nothing changes consensus. + ## Trust anchors Two ss58 addresses in `chain.toml [manifest]` anchor the audit: diff --git a/tests/unit/test_decided_votes.py b/tests/unit/test_decided_votes.py new file mode 100644 index 00000000..3d5fb702 --- /dev/null +++ b/tests/unit/test_decided_votes.py @@ -0,0 +1,236 @@ +"""The decided-votes drift guard (``cascade.shared.decided``, ``cascade-audit +decided``, ``decided_votes.toml``). + +* The repo's own ``chain.toml`` and ``decided_votes.toml`` are IN SYNC — this + is the CI guard: a vote that locks in and is not typed (or a pin moved on + one side only) fails here on the next PR, whatever it is about. +* Every record row names a feature the shipped config actually computes + (changed vote params = a fresh vote = a stale row). +* A decided, applied, untyped block is a TYPE finding carrying the exact + lines to type; a typed block that disagrees is MISMATCH; a pin that moved + is PIN_DRIFT; a row the config no longer computes is STALE; a decided block + not yet applied is PENDING (never asked for early); a held row is HELD. +* A node's persisted lock-ins vs the record: UNRECORDED (add the row), + DISAGREE (nodes differ — never typed), UNKNOWN (retired feature). +* The CLI exits 1 on drift, 0 in sync, and emits JSON. +""" +from __future__ import annotations + +import json +import re +from dataclasses import replace +from pathlib import Path + +import pytest + +import cascade.shared.decided as D +from cascade.shared.activation import ActivationRecord, ActivationStore +from cascade.shared.config import load_chain_config + +REPO = Path(__file__).resolve().parents[2] + + +@pytest.fixture(scope="module") +def shipped(): + return load_chain_config(REPO / "chain.toml") + + +@pytest.fixture(scope="module") +def record(): + return D.load_decided(REPO / D.DECIDED_FILE) + + +def _by(findings, feature, key=None): + return [f for f in findings if f.feature == feature and (key is None or f.key == key)] + + +# ── the guard itself ───────────────────────────────────────────────────────── + + +def test_repo_chain_toml_and_record_are_in_sync(shipped, record): + findings = D.drift_report(shipped, record) + bad = [f for f in findings if f.fails] + assert not bad, "chain.toml has drifted from decided_votes.toml:\n" + D.render(bad) + # every row's feature is one the shipped config computes by that exact + # name — a changed param set would be a fresh vote and a stale row + names = set(D.configured_features(shipped)) + for v in record.votes: + assert v.feature in names, (v.feature, sorted(names)) + # the decided-and-typed ones read OK, the held ones HELD with a reason + kinds = {f.feature: f.kind for f in findings if f.feature != "pins"} + assert kinds["v2b-69630f92"] == D.OK + assert kinds["forfeit-42b1a486"] == D.OK + assert kinds["margin-v2-d30eb443"] == D.OK + assert kinds["lad-556b1b98"] == D.OK + for held in ("rolling-era-king", "prelaunch-95-r2"): + assert kinds[held] == D.HELD + assert record.vote(held).reason + assert all(f.kind == D.OK for f in _by(findings, "pins")) + assert {f.key for f in _by(findings, "pins")} == set(D.PIN_KEYS) + assert record.as_of_block >= max(v.activation_block for v in record.votes + if v.type_it or v.feature.startswith("v2b-")) + + +def test_every_pin_key_is_recorded(record): + assert set(record.pins) == set(D.PIN_KEYS) + + +def test_record_rejects_bad_rows(): + with pytest.raises(ValueError, match="duplicate"): + D.parse_decided('[[vote]]\nfeature="a"\nactivation_block=1\n' + '[[vote]]\nfeature="a"\nactivation_block=2\n') + with pytest.raises(ValueError, match="activation_block"): + D.parse_decided('[[vote]]\nfeature="a"\nactivation_block=0\n') + with pytest.raises(ValueError, match="lock_block"): + D.parse_decided('[[vote]]\nfeature="a"\nlock_block=5\nactivation_block=2\n') + with pytest.raises(ValueError, match="unknown pin"): + D.parse_decided('[pins]\n"round.nope" = "x"\n') + + +# ── findings ───────────────────────────────────────────────────────────────── + + +def test_untyped_decided_block_in_the_shipped_file_is_a_type_finding(tmp_path, record): + # the file exactly as a validator loads it, with the v2b line untyped + # (= main before PR #457): the guard names the line to type + src = re.sub(r"(?m)^image_decoupled_from_block\s*=\s*\d+", "image_decoupled_from_block = 0", + (REPO / "chain.toml").read_text(encoding="utf-8")) + p = tmp_path / "chain.toml" + p.write_text(src, encoding="utf-8") + cfg = load_chain_config(p) + (f,) = _by(D.drift_report(cfg, record), "v2b-69630f92") + assert f.kind == D.TYPE and f.fails + assert f.key == "v2.image_decoupled_from_block" + assert f.expected == 9493200 and f.actual == 0 + assert f.fix == "[v2]\nimage_decoupled_from_block = 9493200" + assert "→ type:" in D.render([f]) and "image_decoupled_from_block = 9493200" in D.render([f]) + + +def test_not_yet_applied_is_pending_not_type(shipped, record): + cfg = replace(shipped, v2=replace(shipped.v2, image_decoupled_from_block=0)) + (f,) = _by(D.drift_report(cfg, record, as_of_block=9490000), "v2b-69630f92") + assert f.kind == D.PENDING and not f.fails + (f,) = _by(D.drift_report(cfg, record, as_of_block=9493200), "v2b-69630f92") + assert f.kind == D.TYPE + + +def test_typed_block_that_disagrees_with_the_decision_is_mismatch(shipped, record): + cfg = replace(shipped, v2=replace(shipped.v2, image_decoupled_from_block=9500400)) + (f,) = _by(D.drift_report(cfg, record), "v2b-69630f92") + assert f.kind == D.MISMATCH and f.fails + assert f.expected == 9493200 and f.actual == 9500400 + assert f.fix.endswith("image_decoupled_from_block = 9493200") + + +def test_held_row_is_never_asked_for(shipped, record): + # rolling is open in the shipped file and HELD in the record: no TYPE + (f,) = _by(D.drift_report(shipped, record), "rolling-era-king") + assert f.kind == D.HELD and not f.fails + assert "9374400" in f.detail # the known disagreement + # ... but a typed value that is not the recorded one is still a MISMATCH + cfg = replace(shipped, round=replace(shipped.round, rolling_from_block=9378000, + epoch_blocks=900, epoch_blocks_prev=3600, + epoch_activation_block=9378000)) + (f,) = _by(D.drift_report(cfg, record), "rolling-era-king") + assert f.kind == D.MISMATCH + + +def test_pin_drift(shipped, record): + cfg = replace(shipped, round=replace(shipped.round, worker_code_fingerprint="0" * 64)) + (f,) = _by(D.drift_report(cfg, record), "pins", "round.worker_code_fingerprint") + assert f.kind == D.PIN_DRIFT and f.fails + assert f.fix == f'[round]\nworker_code_fingerprint = "{record.pins[f.key]}"' + # the v2 contract's pin is read through [v2.training] + (ok,) = _by(D.drift_report(shipped, record), "pins", "v2.training.train_image_digest") + assert ok.kind == D.OK and ok.actual == shipped.training.v2_contract.train_image_digest + + +def test_stale_row_and_unknown_feature(shipped, record): + stale = D.DecidedRecord(as_of_block=record.as_of_block, votes=( + D.DecidedVote("v2b-deadbeef", 1, 9493200, "tally"), + D.DecidedVote("promotion-carry", 1, 9291600, "signals"))) + findings = D.drift_report(shipped, stale) + kinds = {f.feature: f.kind for f in findings} + assert kinds["v2b-deadbeef"] == D.STALE + assert kinds["promotion-carry"] == D.UNKNOWN + # the open votes of the config with no row at all are reported as pending + assert kinds["rolling-era-king"] == D.PENDING and kinds["prelaunch-95-r2"] == D.PENDING + + +def test_state_report(record): + recs = [ + ActivationRecord("v2b-69630f92", 9486000, 9493200, 9486000, "tally"), # agrees + ActivationRecord("rolling-era-king", 9374400, 9378000, 9374400, "signals"), # differs + ActivationRecord("margin-v2-cafebabe", 9190800, 9194400, 9190800, "tally"), # new vote + ActivationRecord("promotion-carry", 9290700, 9291600, 9290700, "signals"), # retired + ActivationRecord("lad-556b1b98", 0, 0, 9494100, ""), # not locked + ] + kinds = {f.feature: f for f in D.state_report(recs, record)} + assert kinds["v2b-69630f92"].kind == D.OK + assert kinds["rolling-era-king"].kind == D.DISAGREE and not kinds["rolling-era-king"].fails + assert kinds["rolling-era-king"].expected == 9136800 + assert kinds["rolling-era-king"].actual == 9378000 + un = kinds["margin-v2-cafebabe"] + assert un.kind == D.UNRECORDED and un.fails + assert un.fix == ('[[vote]]\nfeature = "margin-v2-cafebabe"\nlock_block = 9190800\n' + 'activation_block = 9194400\nsource = "tally"') + assert kinds["promotion-carry"].kind == D.UNKNOWN + assert "lad-556b1b98" not in kinds + + +def test_read_state_records(tmp_path): + ActivationStore(tmp_path / "activation_state.json").save( + ActivationRecord("rolling-era-king", 9133200, 9136800, 9133200, "config")) + ActivationStore(tmp_path / "activation_image_decoupled_state.json").save( + ActivationRecord("v2b-69630f92", 9486000, 9493200, 9486000, "tally")) + (tmp_path / "activation_forfeit_state.json").write_text("{not json", encoding="utf-8") + recs = D.read_state_records(tmp_path) + assert set(recs) == {"activation_state.json", "activation_image_decoupled_state.json"} + assert recs["activation_state.json"].feature == "rolling-era-king" + + +def test_type_hint_and_key_mapping(): + hint = D.type_hint("v2b-69630f92", 9493200) + assert "image_decoupled_from_block = 9493200" in hint and "[v2]" in hint + assert D.DECIDED_FILE in hint + assert D.typed_key_for("forfeit-42b1a486") == "scoring.forfeit_from_block" + assert D.typed_key_for("lad-556b1b98") == "eval.shared_ladder_from_block" + assert D.typed_key_for("rolling-era-king") == "round.rolling_from_block" + assert D.typed_key_for("rolling-era-kingdom") == "" + assert D.typed_key_for("promotion-carry") == "" + assert "no chain.toml key" in D.type_hint("promotion-carry", 1) + assert D.toml_line("v2.training.train_image_digest", "sha256:ab") == \ + '[v2.training]\ntrain_image_digest = "sha256:ab"' + + +# ── the CLI ────────────────────────────────────────────────────────────────── + + +def test_cli_in_sync_and_drifted(tmp_path, capsys): + from cascade.audit.main import main as audit_main + + rc = audit_main(["decided", "--config", str(REPO / "chain.toml")]) + out = capsys.readouterr().out + assert rc == 0 and "in sync" in out and "[HELD] prelaunch-95-r2" in out + + src = re.sub(r"(?m)^image_decoupled_from_block\s*=\s*\d+", "image_decoupled_from_block = 0", + (REPO / "chain.toml").read_text(encoding="utf-8")) + p = tmp_path / "chain.toml" + p.write_text(src, encoding="utf-8") + rc = audit_main(["decided", "--config", str(p), "--decided", str(REPO / D.DECIDED_FILE)]) + out = capsys.readouterr().out + assert rc == 1 and "[TYPE] v2b-69630f92" in out and "image_decoupled_from_block = 9493200" in out + + # --state-dir: a lock-in the record lacks fails; a retired feature is informational + ActivationStore(tmp_path / "activation_margin_v2_state.json").save( + ActivationRecord("margin-v2-cafebabe", 9190800, 9194400, 9190800, "tally")) + ActivationStore(tmp_path / "activation_promotion_carry_state.json").save( + ActivationRecord("promotion-carry", 9290700, 9291600, 9290700, "signals")) + rc = audit_main(["decided", "--config", str(REPO / "chain.toml"), "--decided", + str(REPO / D.DECIDED_FILE), "--state-dir", str(tmp_path), "--json"]) + out = json.loads(capsys.readouterr().out) + assert rc == 1 and out["ok"] is False + kinds = {(f["feature"], f["kind"]) for f in out["findings"]} + assert ("margin-v2-cafebabe", D.UNRECORDED) in kinds + assert ("promotion-carry", D.UNKNOWN) in kinds + assert out["as_of_block"] == 9494100