Report per-lane cost and run spend in --json and transcripts (#79), v1.3.0

BACKLOG.md

@@ -84,9 +84,13 @@ The UI lives in its own private repo,
       TLS bypass that also weakens the OpenRouter lanes
       ([#78](https://github.com/CryptoJones/FlatlineRoundtable/issues/78)) — fixed on feat/per-lane-ca-bundle (v1.2.0):
       `ca_bundle` and a per-lane-only `insecure`
-- [ ] Report per-lane `cost` and run-level spend in `--json` (and under `--diff`) —
+- [x] Report per-lane `cost` and run-level spend in `--json` (and under `--diff`) —
       the number cron/CI callers need is computed but not surfaced machine-readably
-      ([#79](https://github.com/CryptoJones/FlatlineRoundtable/issues/79))
+      ([#79](https://github.com/CryptoJones/FlatlineRoundtable/issues/79)) — fixed on feat/json-spend (v1.3.0);
+      plain `--json` stays a list, see #131
+- [ ] Plain `--json` emits an object with `results` + `spend` — a breaking
+      shape change held for 2.0; #79 could not add a top-level key to a bare list
+      ([#131](https://github.com/CryptoJones/FlatlineRoundtable/issues/131))
 - [x] Discussion mode: `--discuss` lets lanes share one context and take turns,
       one lane in flight at a time — the lightweight stand-in for a Buzz-style
       conversation, no resident runtimes or bus

README.md

@@ -238,6 +238,20 @@ resolved is refused outright rather than estimated at $0 — a guard that
 cannot see a lane cannot bind on it. `cli`/`acp` lanes, `:free` models and an
 explicit `price_per_mtok: 0` are free, not unknown.
 
+What a run spent is machine-readable too. Every `--json` result carries a
+`cost`: dollars when the lane was priced and returned usage, `0.0` when it is
+free, and `null` when it is unknown. Do not read `null` as free. `--diff --json`,
+`--discuss --json`, `--synthesize --json` and every transcript add a `spend`
+object:
+
+```json
+"spend": {"actual": 0.0412, "estimated_worst_case": 0.31, "budget": 0.5, "unpriced": []}
+```
+
+`actual` includes the `--diff` readers. If `unpriced` names any lanes, `actual`
+is a lower bound, not the bill. Plain `--json` stays a bare list of results so
+existing callers keep working. Sum it with `jq 'map(.cost // 0) | add'`.
+
 ## Usage
 
 ```console

roundtable

@@ -97,7 +97,7 @@ from pathlib import Path
 
 # Semantic version. Bump it in the PR that changes behaviour: patch for a fix,
 # minor for a new flag or mode, major for a change that breaks a caller.
-__version__ = "1.2.0"
+__version__ = "1.3.0"
 
 TRANSCRIPT_DIR = Path.home() / ".local" / "share" / "flatline-roundtable" / "transcripts"
 RUN_ID_RE = re.compile(r"[A-Za-z0-9_-][A-Za-z0-9._-]{0,63}")
@@ -340,6 +340,29 @@ def lane_prices(lane: dict, table: dict) -> tuple[float, float]:
     return table.get(lane.get("model"), (0.0, 0.0))
 
 
+def spend_report(spent: float, estimate: float, budget, lanes: list[dict],
+                 table: dict) -> dict:
+    """The run's spend, machine-readable (#79): what the human summary prints,
+    plus the lanes whose cost could not be priced. When `unpriced` is non-empty,
+    `actual` is a lower bound, not the bill."""
+    return {"actual": round(spent, 6),
+            "estimated_worst_case": round(estimate, 6),
+            "budget": budget,
+            "unpriced": [l["name"] for l in lanes if not lane_price_known(l, table)]}
+
+
+def fill_costs(results: list[dict], lanes: list[dict], table: dict) -> None:
+    """Give every result a `cost` (#79). ask() sets it only when it priced a
+    usage block. A lane that is free or on a subscription is 0.0. Anything
+    else is null, meaning unknown: an unpriced model, or no usage returned.
+    Null is the honest value there. A zero would read as free."""
+    for r, l in zip(results, lanes):
+        if r is None or "cost" in r:
+            continue
+        free = lane_price_known(l, table) and lane_prices(l, table) == (0.0, 0.0)
+        r["cost"] = 0.0 if free else None
+
+
 def lane_price_known(lane: dict, table: dict) -> bool:
     """Whether lane_prices' (0, 0) would mean "free" rather than "no idea" (#72).
 
@@ -3371,13 +3394,19 @@ def synthesize_run(a, cfg: dict, lanes: list[dict]) -> int:
                          "synthesis": synth, "synthesis_error": synth_err,
                          "mode": "synthesis", "round": int(prior.get("round") or 1),
                          "parent": [str(p) for p in paths],
-                         "cost": round(spent[0], 6)}, path)
+                         "cost": round(spent[0], 6),
+                         "spend": spend_report(spent[0], estimate, budget, readers, prices)},
+                        path)
     if a.json:
         print(json.dumps({"run_id": run_id, "synthesis": synth, "error": synth_err,
-                          "cost": round(spent[0], 6)}, indent=2))
+                          "cost": round(spent[0], 6),
+                          "spend": spend_report(spent[0], estimate, budget, readers, prices)},
+                         indent=2))
     else:
         print(f"SYNTHESIS of {len(answered)} answers from {len(paths)} transcript(s)")
         print_synthesis(synth, synth_err)
+        if spent[0] or estimate:
+            print(f"spend: ${spent[0]:.4f} actual  (est. worst case ${estimate:.4f})")
         if path:
             print(f"transcript: {path}")
     return 0 if synth else 1
@@ -3989,7 +4018,7 @@ def main() -> int:
 
     thread: list[dict] = []
 
-    def save(results_so_far, synthesis=None, collisions=None, partial=True):
+    def save(results_so_far, synthesis=None, collisions=None, partial=True, spend=None):
         """Write the transcript. Called after every lane, and again at the end.
 
         Atomic via write-then-replace: a kill during the write leaves the previous
@@ -4003,6 +4032,8 @@ def main() -> int:
         # round does -- so a partial revision transcript reads like a full one.
         payload = {"run_id": run_id, "brief": brief_on_record, "results": results_so_far,
                    "synthesis": synthesis, "collisions": collisions}
+        if spend is not None:
+            payload["spend"] = spend
         if prompts or a.discuss:
             payload.update({"round": round_no,
                             "parent": [str(p) for p in prior_paths],
@@ -4104,18 +4135,25 @@ def main() -> int:
     # Final write: same file the fan-out has been updating, now complete and with
     # the synthesis attached. Clears the `partial` marker. Revision metadata
     # (round/parent/aliases) is folded in by save() when this is a revision round.
-    save(results, synthesis=synth, collisions=collisions, partial=False)
+    fill_costs(results, lanes, prices)
+    # spent[0] includes the --diff readers, which are picked from these lanes.
+    spend = spend_report(spent[0], estimate, budget, lanes, prices)
+    save(results, synthesis=synth, collisions=collisions, partial=False, spend=spend)
 
     if a.json:
         if a.discuss:
-            out = {"discussion": thread, "results": results}
+            out = {"discussion": thread, "results": results, "spend": spend}
             if a.diff:
                 out["synthesis"] = synth
         else:
-            out = {"results": results, "synthesis": synth} if a.diff else results
+            # The no-flag form stays a bare list: wrapping it would break every
+            # caller indexing into it. Each result carries its own `cost`.
+            out = {"results": results, "synthesis": synth, "spend": spend} if a.diff else results
         print(json.dumps(out, indent=2))
     elif a.diff:
         print_synthesis(synth, synth_err)
+        if spent[0] or estimate:
+            print(f"spend: ${spent[0]:.4f} actual  (est. worst case ${estimate:.4f})")
     else:
         if a.discuss:
             print(f"\n{'=' * 74}\nCLOSINGS — each lane's position after the discussion")

tests/test_roundtable.py

@@ -2708,6 +2708,61 @@ def test_each_gates_the_whole_panel_before_spawning_children(self):
             self.assertEqual(r.returncode, 0, r.stderr)
             self.assertIsNotNone(s.srv.last_request)
 
+    def test_json_gives_every_lane_a_cost(self):
+        """#79: cost was set only when a lane was priced and returned usage, so a
+        caller summing `--json` could not tell a free lane from an unknown one."""
+        with StubServer() as s:
+            r = self._run([
+                {"name": "Paid", "harness": "http", "model": "m", "vendor": "a",
+                 "base_url": s.url, "price_per_mtok": 1000.0},
+                {"name": "Free", "harness": "http", "model": "vendor/m:free", "vendor": "b",
+                 "base_url": s.url},
+                {"name": "Unknown", "harness": "http", "model": "vendor/not-in-table",
+                 "vendor": "c", "base_url": s.url},
+            ], "--json")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        out = json.loads(r.stdout)
+        self.assertIsInstance(out, list, "the plain --json shape must not change")
+        cost = {x["lane"]: x["cost"] for x in out}
+        # Stub usage: 100 prompt + 200 completion tokens at $1000/Mtok.
+        self.assertAlmostEqual(cost["Paid"], 0.3)
+        self.assertEqual(cost["Free"], 0.0)
+        self.assertIsNone(cost["Unknown"], "an unpriced lane must not read as free")
+
+    def test_diff_json_reports_run_spend(self):
+        """#79: the run-level number the human summary prints, machine-readable,
+        and including what the readers cost."""
+        with StubServer() as s:
+            lanes = [{"name": n, "harness": "http", "model": "m", "vendor": n,
+                      "base_url": s.url, "price_per_mtok": 1000.0} for n in ("A", "B")]
+            r = self._run(lanes, "--diff", "--json", "--max-spend", "50")
+            self.assertEqual(r.returncode, 0, r.stderr)
+            spend = json.loads(r.stdout)["spend"]
+            readers = len(s.srv.requests) - 2
+        self.assertGreaterEqual(readers, 1)
+        lane_total = 0.6
+        self.assertAlmostEqual(spend["actual"], lane_total + 0.3 * readers, places=4)
+        self.assertGreater(spend["estimated_worst_case"], 0)
+        self.assertEqual(spend["budget"], 50)
+        self.assertEqual(spend["unpriced"], [])
+
+    def test_diff_prints_the_spend_line(self):
+        with StubServer() as s:
+            lanes = [{"name": n, "harness": "http", "model": "m", "vendor": n,
+                      "base_url": s.url, "price_per_mtok": 1000.0} for n in ("A", "B")]
+            r = self._run(lanes, "--diff")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertIn("spend: $", r.stdout)
+
+    def test_spend_names_the_lanes_it_could_not_price(self):
+        with StubServer() as s:
+            r = self._run([{"name": n, "harness": "http", "model": m, "vendor": n,
+                            "base_url": s.url}
+                           for n, m in (("A", "vendor/not-in-table"), ("B", "vendor/m:free"))],
+                          "--diff", "--json")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        self.assertEqual(json.loads(r.stdout)["spend"]["unpriced"], ["A"])
+
     def test_list_makes_no_network_calls(self):
         with StubServer() as s:
             r = self._run([{"name": "A", "harness": "http", "model": "m", "base_url": s.url}],
@@ -3175,6 +3230,22 @@ def test_latest_n_works_and_json_reports_the_run(self):
         self.assertEqual(out["run_id"], "r2")
         self.assertIsNone(out["error"])
 
+    def test_spend_lands_in_the_transcript_and_synthesize_json(self):
+        """#79: a cron caller reading the transcript or --synthesize --json gets
+        the same spend object as --diff --json."""
+        with StubServer() as s:
+            lanes = self._lanes(s.url, 2, price_per_mtok=1000.0)
+            self.assertEqual(self._run(lanes, "--each", "--run-id", "r79").returncode, 0)
+            rec = json.loads(sorted(self.tdir.glob("*.json"))[0].read_text())
+            self.assertAlmostEqual(rec["spend"]["actual"], 0.3)
+            self.assertAlmostEqual(rec["results"][0]["cost"], 0.3)
+            r = self._run(lanes, "--synthesize", "latest-run", "--json", brief="")
+        self.assertEqual(r.returncode, 0, r.stderr)
+        out = json.loads(r.stdout)
+        self.assertAlmostEqual(out["spend"]["actual"], out["cost"])
+        self.assertAlmostEqual(out["spend"]["actual"], 0.6)
+        self.assertIsNone(out["spend"]["budget"])
+
     def test_refuses_to_start_a_run_alongside(self):
         with StubServer() as s:
             lanes = self._lanes(s.url, 2)