diff --git a/BACKLOG.md b/BACKLOG.md index 3d8caec..d7f8e8a 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -84,9 +84,13 @@ The UI lives in its own private repo, TLS bypass that also weakens the OpenRouter lanes ([#78](https://github.com/CryptoJones/FlatlineRoundtable/issues/78)) — fixed on feat/per-lane-ca-bundle (v1.2.0): `ca_bundle` and a per-lane-only `insecure` -- [ ] Report per-lane `cost` and run-level spend in `--json` (and under `--diff`) — +- [x] Report per-lane `cost` and run-level spend in `--json` (and under `--diff`) — the number cron/CI callers need is computed but not surfaced machine-readably - ([#79](https://github.com/CryptoJones/FlatlineRoundtable/issues/79)) + ([#79](https://github.com/CryptoJones/FlatlineRoundtable/issues/79)) — fixed on feat/json-spend (v1.3.0); + plain `--json` stays a list, see #131 +- [ ] Plain `--json` emits an object with `results` + `spend` — a breaking + shape change held for 2.0; #79 could not add a top-level key to a bare list + ([#131](https://github.com/CryptoJones/FlatlineRoundtable/issues/131)) - [x] Discussion mode: `--discuss` lets lanes share one context and take turns, one lane in flight at a time — the lightweight stand-in for a Buzz-style conversation, no resident runtimes or bus diff --git a/README.md b/README.md index 5a92458..60f62a8 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,20 @@ resolved is refused outright rather than estimated at $0 — a guard that cannot see a lane cannot bind on it. `cli`/`acp` lanes, `:free` models and an explicit `price_per_mtok: 0` are free, not unknown. +What a run spent is machine-readable too. Every `--json` result carries a +`cost`: dollars when the lane was priced and returned usage, `0.0` when it is +free, and `null` when it is unknown. Do not read `null` as free. `--diff --json`, +`--discuss --json`, `--synthesize --json` and every transcript add a `spend` +object: + +```json +"spend": {"actual": 0.0412, "estimated_worst_case": 0.31, "budget": 0.5, "unpriced": []} +``` + +`actual` includes the `--diff` readers. If `unpriced` names any lanes, `actual` +is a lower bound, not the bill. Plain `--json` stays a bare list of results so +existing callers keep working. Sum it with `jq 'map(.cost // 0) | add'`. + ## Usage ```console diff --git a/roundtable b/roundtable index f49994b..09961a5 100755 --- a/roundtable +++ b/roundtable @@ -97,7 +97,7 @@ from pathlib import Path # Semantic version. Bump it in the PR that changes behaviour: patch for a fix, # minor for a new flag or mode, major for a change that breaks a caller. -__version__ = "1.2.0" +__version__ = "1.3.0" TRANSCRIPT_DIR = Path.home() / ".local" / "share" / "flatline-roundtable" / "transcripts" RUN_ID_RE = re.compile(r"[A-Za-z0-9_-][A-Za-z0-9._-]{0,63}") @@ -340,6 +340,29 @@ def lane_prices(lane: dict, table: dict) -> tuple[float, float]: return table.get(lane.get("model"), (0.0, 0.0)) +def spend_report(spent: float, estimate: float, budget, lanes: list[dict], + table: dict) -> dict: + """The run's spend, machine-readable (#79): what the human summary prints, + plus the lanes whose cost could not be priced. When `unpriced` is non-empty, + `actual` is a lower bound, not the bill.""" + return {"actual": round(spent, 6), + "estimated_worst_case": round(estimate, 6), + "budget": budget, + "unpriced": [l["name"] for l in lanes if not lane_price_known(l, table)]} + + +def fill_costs(results: list[dict], lanes: list[dict], table: dict) -> None: + """Give every result a `cost` (#79). ask() sets it only when it priced a + usage block. A lane that is free or on a subscription is 0.0. Anything + else is null, meaning unknown: an unpriced model, or no usage returned. + Null is the honest value there. A zero would read as free.""" + for r, l in zip(results, lanes): + if r is None or "cost" in r: + continue + free = lane_price_known(l, table) and lane_prices(l, table) == (0.0, 0.0) + r["cost"] = 0.0 if free else None + + def lane_price_known(lane: dict, table: dict) -> bool: """Whether lane_prices' (0, 0) would mean "free" rather than "no idea" (#72). @@ -3371,13 +3394,19 @@ def synthesize_run(a, cfg: dict, lanes: list[dict]) -> int: "synthesis": synth, "synthesis_error": synth_err, "mode": "synthesis", "round": int(prior.get("round") or 1), "parent": [str(p) for p in paths], - "cost": round(spent[0], 6)}, path) + "cost": round(spent[0], 6), + "spend": spend_report(spent[0], estimate, budget, readers, prices)}, + path) if a.json: print(json.dumps({"run_id": run_id, "synthesis": synth, "error": synth_err, - "cost": round(spent[0], 6)}, indent=2)) + "cost": round(spent[0], 6), + "spend": spend_report(spent[0], estimate, budget, readers, prices)}, + indent=2)) else: print(f"SYNTHESIS of {len(answered)} answers from {len(paths)} transcript(s)") print_synthesis(synth, synth_err) + if spent[0] or estimate: + print(f"spend: ${spent[0]:.4f} actual (est. worst case ${estimate:.4f})") if path: print(f"transcript: {path}") return 0 if synth else 1 @@ -3989,7 +4018,7 @@ def main() -> int: thread: list[dict] = [] - def save(results_so_far, synthesis=None, collisions=None, partial=True): + def save(results_so_far, synthesis=None, collisions=None, partial=True, spend=None): """Write the transcript. Called after every lane, and again at the end. Atomic via write-then-replace: a kill during the write leaves the previous @@ -4003,6 +4032,8 @@ def main() -> int: # round does -- so a partial revision transcript reads like a full one. payload = {"run_id": run_id, "brief": brief_on_record, "results": results_so_far, "synthesis": synthesis, "collisions": collisions} + if spend is not None: + payload["spend"] = spend if prompts or a.discuss: payload.update({"round": round_no, "parent": [str(p) for p in prior_paths], @@ -4104,18 +4135,25 @@ def main() -> int: # Final write: same file the fan-out has been updating, now complete and with # the synthesis attached. Clears the `partial` marker. Revision metadata # (round/parent/aliases) is folded in by save() when this is a revision round. - save(results, synthesis=synth, collisions=collisions, partial=False) + fill_costs(results, lanes, prices) + # spent[0] includes the --diff readers, which are picked from these lanes. + spend = spend_report(spent[0], estimate, budget, lanes, prices) + save(results, synthesis=synth, collisions=collisions, partial=False, spend=spend) if a.json: if a.discuss: - out = {"discussion": thread, "results": results} + out = {"discussion": thread, "results": results, "spend": spend} if a.diff: out["synthesis"] = synth else: - out = {"results": results, "synthesis": synth} if a.diff else results + # The no-flag form stays a bare list: wrapping it would break every + # caller indexing into it. Each result carries its own `cost`. + out = {"results": results, "synthesis": synth, "spend": spend} if a.diff else results print(json.dumps(out, indent=2)) elif a.diff: print_synthesis(synth, synth_err) + if spent[0] or estimate: + print(f"spend: ${spent[0]:.4f} actual (est. worst case ${estimate:.4f})") else: if a.discuss: print(f"\n{'=' * 74}\nCLOSINGS — each lane's position after the discussion") diff --git a/tests/test_roundtable.py b/tests/test_roundtable.py index 078d686..daa7740 100644 --- a/tests/test_roundtable.py +++ b/tests/test_roundtable.py @@ -2708,6 +2708,61 @@ def test_each_gates_the_whole_panel_before_spawning_children(self): self.assertEqual(r.returncode, 0, r.stderr) self.assertIsNotNone(s.srv.last_request) + def test_json_gives_every_lane_a_cost(self): + """#79: cost was set only when a lane was priced and returned usage, so a + caller summing `--json` could not tell a free lane from an unknown one.""" + with StubServer() as s: + r = self._run([ + {"name": "Paid", "harness": "http", "model": "m", "vendor": "a", + "base_url": s.url, "price_per_mtok": 1000.0}, + {"name": "Free", "harness": "http", "model": "vendor/m:free", "vendor": "b", + "base_url": s.url}, + {"name": "Unknown", "harness": "http", "model": "vendor/not-in-table", + "vendor": "c", "base_url": s.url}, + ], "--json") + self.assertEqual(r.returncode, 0, r.stderr) + out = json.loads(r.stdout) + self.assertIsInstance(out, list, "the plain --json shape must not change") + cost = {x["lane"]: x["cost"] for x in out} + # Stub usage: 100 prompt + 200 completion tokens at $1000/Mtok. + self.assertAlmostEqual(cost["Paid"], 0.3) + self.assertEqual(cost["Free"], 0.0) + self.assertIsNone(cost["Unknown"], "an unpriced lane must not read as free") + + def test_diff_json_reports_run_spend(self): + """#79: the run-level number the human summary prints, machine-readable, + and including what the readers cost.""" + with StubServer() as s: + lanes = [{"name": n, "harness": "http", "model": "m", "vendor": n, + "base_url": s.url, "price_per_mtok": 1000.0} for n in ("A", "B")] + r = self._run(lanes, "--diff", "--json", "--max-spend", "50") + self.assertEqual(r.returncode, 0, r.stderr) + spend = json.loads(r.stdout)["spend"] + readers = len(s.srv.requests) - 2 + self.assertGreaterEqual(readers, 1) + lane_total = 0.6 + self.assertAlmostEqual(spend["actual"], lane_total + 0.3 * readers, places=4) + self.assertGreater(spend["estimated_worst_case"], 0) + self.assertEqual(spend["budget"], 50) + self.assertEqual(spend["unpriced"], []) + + def test_diff_prints_the_spend_line(self): + with StubServer() as s: + lanes = [{"name": n, "harness": "http", "model": "m", "vendor": n, + "base_url": s.url, "price_per_mtok": 1000.0} for n in ("A", "B")] + r = self._run(lanes, "--diff") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn("spend: $", r.stdout) + + def test_spend_names_the_lanes_it_could_not_price(self): + with StubServer() as s: + r = self._run([{"name": n, "harness": "http", "model": m, "vendor": n, + "base_url": s.url} + for n, m in (("A", "vendor/not-in-table"), ("B", "vendor/m:free"))], + "--diff", "--json") + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout)["spend"]["unpriced"], ["A"]) + def test_list_makes_no_network_calls(self): with StubServer() as s: r = self._run([{"name": "A", "harness": "http", "model": "m", "base_url": s.url}], @@ -3175,6 +3230,22 @@ def test_latest_n_works_and_json_reports_the_run(self): self.assertEqual(out["run_id"], "r2") self.assertIsNone(out["error"]) + def test_spend_lands_in_the_transcript_and_synthesize_json(self): + """#79: a cron caller reading the transcript or --synthesize --json gets + the same spend object as --diff --json.""" + with StubServer() as s: + lanes = self._lanes(s.url, 2, price_per_mtok=1000.0) + self.assertEqual(self._run(lanes, "--each", "--run-id", "r79").returncode, 0) + rec = json.loads(sorted(self.tdir.glob("*.json"))[0].read_text()) + self.assertAlmostEqual(rec["spend"]["actual"], 0.3) + self.assertAlmostEqual(rec["results"][0]["cost"], 0.3) + r = self._run(lanes, "--synthesize", "latest-run", "--json", brief="") + self.assertEqual(r.returncode, 0, r.stderr) + out = json.loads(r.stdout) + self.assertAlmostEqual(out["spend"]["actual"], out["cost"]) + self.assertAlmostEqual(out["spend"]["actual"], 0.6) + self.assertIsNone(out["spend"]["budget"]) + def test_refuses_to_start_a_run_alongside(self): with StubServer() as s: lanes = self._lanes(s.url, 2)