-
Notifications
You must be signed in to change notification settings - Fork 2
GPU receipt accounting: remote runs emit spend rows; ledger gpu sums them (#425) #430
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -267,6 +267,22 @@ export interface PlanCompiledRecord { | |
|
|
||
| /** One ADVISORY todo transition. `open` is the absence of a row; multiple rows per id | ||
| * resolve last-ts-wins. No `actor` field — no trustworthy actor identity exists here. */ | ||
| /** GPU/compute accounting for a remote run (#425, GPU-plane spec): pure | ||
| * spend record — no fidelity, never feeds priors. The campaign warrant fold | ||
| * sums these (unit-keyed bounds). Emitted by the remote executor at settle. */ | ||
| export interface ReceiptRecord { | ||
| type: "receipt"; | ||
| ts: string; | ||
| task_id: string; | ||
| executor: "remote"; | ||
| gpu_sku?: string; | ||
| gpu_seconds?: number; | ||
| cost_usd?: number; | ||
| problem?: string; | ||
| session?: string; | ||
| status?: "completed" | "failed" | "aborted"; | ||
| } | ||
|
|
||
| export interface TodoRecord { | ||
| type: "todo"; | ||
| ts: string; | ||
|
|
@@ -279,6 +295,7 @@ export interface TodoRecord { | |
|
|
||
| export type LedgerRecord = | ||
| | SolveRecord | ||
| | ReceiptRecord | ||
| | VerdictRecord | ||
| | AttemptErrorRecord | ||
| | FallbackRecord | ||
|
|
@@ -297,6 +314,49 @@ export function ledgerPath(): string { | |
| return join(studioPathsOrLegacy().ledger, "runs.jsonl"); | ||
| } | ||
|
|
||
| /** GPU spend totals over receipt rows (#425) — the warrant fold's view. | ||
| * Pure aggregation: never throws, zeros on empty. by_sku/by_status keyed | ||
| * breakdowns; cost_usd omitted per-row when the runner didn't report it. */ | ||
| export interface GpuTotals { | ||
| receipts: number; | ||
| gpu_seconds: number; | ||
| cost_usd: number; | ||
| by_sku: Record<string, { gpu_seconds?: number; cost_usd?: number }>; | ||
| by_status: Record<string, number>; | ||
| } | ||
|
|
||
| export function gpuTotals(file = ledgerPath()): GpuTotals { | ||
| const t: GpuTotals = { receipts: 0, gpu_seconds: 0, cost_usd: 0, by_sku: {}, by_status: {} }; | ||
| let raw: string; | ||
| try { | ||
| raw = readFileSync(file, "utf8"); | ||
| } catch { | ||
| return t; // absent ledger = zero spend, not an error | ||
| } | ||
| for (const line of raw.split("\n")) { | ||
| if (!line.trim()) continue; | ||
| let rec: LedgerRecord; | ||
| try { | ||
| rec = JSON.parse(line) as LedgerRecord; | ||
| } catch { | ||
| continue; // a torn line never breaks accounting | ||
| } | ||
| if (rec.type !== "receipt") continue; | ||
| t.receipts += 1; | ||
| if (rec.gpu_seconds !== undefined) t.gpu_seconds += rec.gpu_seconds; | ||
| if (rec.cost_usd !== undefined) t.cost_usd += rec.cost_usd; | ||
| if (rec.gpu_sku !== undefined) { | ||
| const b = t.by_sku[rec.gpu_sku] ?? {}; | ||
| if (rec.gpu_seconds !== undefined) b.gpu_seconds = (b.gpu_seconds ?? 0) + rec.gpu_seconds; | ||
| if (rec.cost_usd !== undefined) b.cost_usd = (b.cost_usd ?? 0) + rec.cost_usd; | ||
| t.by_sku[rec.gpu_sku] = b; | ||
| } | ||
| if (rec.status !== undefined) t.by_status[rec.status] = (t.by_status[rec.status] ?? 0) + 1; | ||
|
Comment on lines
+338
to
+354
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win Validate parsed rows before aggregation.
🤖 Prompt for AI Agents |
||
| } | ||
| t.cost_usd = Math.round(t.cost_usd * 1e4) / 1e4; | ||
| return t; | ||
| } | ||
|
|
||
| /** Append one record as a single JSONL line. Validates against the `ledger-record` | ||
| * schema (Task 2 wires this in) before writing so no malformed line ever lands. | ||
| * Throws when the serialized line exceeds PIPE_BUF (would break O_APPEND | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -9,6 +9,7 @@ | |||||||||||||||||||||||||||||||||
| import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, utimesSync, writeFileSync } from "node:fs"; | ||||||||||||||||||||||||||||||||||
| import { basename, join, resolve } from "node:path"; | ||||||||||||||||||||||||||||||||||
| import { EventQueue } from "./event_queue.js"; | ||||||||||||||||||||||||||||||||||
| import { appendRecord, type ReceiptRecord } from "./ledger.js"; | ||||||||||||||||||||||||||||||||||
| import { classifyLine } from "./telemetry.js"; | ||||||||||||||||||||||||||||||||||
| import { | ||||||||||||||||||||||||||||||||||
| appendIndex, | ||||||||||||||||||||||||||||||||||
|
|
@@ -160,6 +161,12 @@ export class RemoteExecutor implements Executor { | |||||||||||||||||||||||||||||||||
| events.push(classifyLine(line, "stdout")); | ||||||||||||||||||||||||||||||||||
| }; | ||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||
| // #425 GPU receipt: the runner-contract fields ride the finished payload; | ||||||||||||||||||||||||||||||||||
| // stashed here, consumed at settle. Absent fields → null → no receipt row | ||||||||||||||||||||||||||||||||||
| // (nothing to account — the pre-contract runner emits none). | ||||||||||||||||||||||||||||||||||
| type GpuReceipt = { gpu_sku?: string; gpu_seconds?: number; cost_usd?: number }; | ||||||||||||||||||||||||||||||||||
| let gpuReceipt: GpuReceipt | null = null; | ||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||
| const settle = (status: RunStatus, exitCode: number): void => { | ||||||||||||||||||||||||||||||||||
| if (settled) return; | ||||||||||||||||||||||||||||||||||
| settled = true; | ||||||||||||||||||||||||||||||||||
|
|
@@ -168,6 +175,32 @@ export class RemoteExecutor implements Executor { | |||||||||||||||||||||||||||||||||
| } catch (e) { | ||||||||||||||||||||||||||||||||||
| process.stderr.write(`amico-run: failed to write FINISHED: ${(e as Error).message}\n`); | ||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||
| // GPU accounting (#425): the receipt row is spend, not science — | ||||||||||||||||||||||||||||||||||
| // emitted on FAILED runs too (they burn the same GPU time), never | ||||||||||||||||||||||||||||||||||
| // feeding priors (no fidelity field exists on it by design). | ||||||||||||||||||||||||||||||||||
| if (gpuReceipt) { | ||||||||||||||||||||||||||||||||||
| try { | ||||||||||||||||||||||||||||||||||
| const rec: ReceiptRecord = { | ||||||||||||||||||||||||||||||||||
| type: "receipt", | ||||||||||||||||||||||||||||||||||
| ts: new Date().toISOString(), | ||||||||||||||||||||||||||||||||||
| task_id: taskId, | ||||||||||||||||||||||||||||||||||
| executor: "remote", | ||||||||||||||||||||||||||||||||||
| ...gpuReceipt, | ||||||||||||||||||||||||||||||||||
| }; | ||||||||||||||||||||||||||||||||||
| if (status === "completed" || status === "failed" || status === "aborted") rec.status = status; | ||||||||||||||||||||||||||||||||||
| appendRecord(rec); // validates against the ledger-record schema | ||||||||||||||||||||||||||||||||||
| atomicWriteFile(runDir, "receipt.toml", [ | ||||||||||||||||||||||||||||||||||
| "# GPU receipt (runner contract #424) — mirrored from the cloud finished payload", | ||||||||||||||||||||||||||||||||||
| `task_id = "${taskId}"`, | ||||||||||||||||||||||||||||||||||
| ...(rec.gpu_sku ? [`gpu_sku = ${JSON.stringify(rec.gpu_sku)}`] : []), | ||||||||||||||||||||||||||||||||||
| ...(rec.gpu_seconds !== undefined ? [`gpu_seconds = ${rec.gpu_seconds}`] : []), | ||||||||||||||||||||||||||||||||||
| ...(rec.cost_usd !== undefined ? [`cost_usd = ${rec.cost_usd}`] : []), | ||||||||||||||||||||||||||||||||||
| `status = "${status}"`, | ||||||||||||||||||||||||||||||||||
| ].join("\n") + "\n"); | ||||||||||||||||||||||||||||||||||
|
Comment on lines
+192
to
+199
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🗄️ Data Integrity & Integration | 🟡 Minor | ⚡ Quick win Escape Line 194 interpolates a remote response value into a TOML string. A task ID containing a quote or newline makes Proposed fix- `task_id = "${taskId}"`,
+ `task_id = ${JSON.stringify(taskId)}`,📝 Committable suggestion
Suggested change
🤖 Prompt for AI Agents |
||||||||||||||||||||||||||||||||||
| } catch (e) { | ||||||||||||||||||||||||||||||||||
| process.stderr.write(`amico-run: failed to emit GPU receipt stanza: ${(e as Error).message}\n`); | ||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||
| events.push({ kind: "finished", status, exitCode }); | ||||||||||||||||||||||||||||||||||
| events.close(); | ||||||||||||||||||||||||||||||||||
| resolveFinished({ status, exitCode }); | ||||||||||||||||||||||||||||||||||
|
|
@@ -322,6 +355,16 @@ export class RemoteExecutor implements Executor { | |||||||||||||||||||||||||||||||||
| settle("failed", EXIT_INFERRED); | ||||||||||||||||||||||||||||||||||
| return; | ||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||
| const fin = s.finished as | ||||||||||||||||||||||||||||||||||
| | { status?: string; gpu_sku?: unknown; gpu_seconds?: unknown; cost_usd?: unknown } | ||||||||||||||||||||||||||||||||||
| | undefined; | ||||||||||||||||||||||||||||||||||
| if (fin && gpuReceipt === null) { | ||||||||||||||||||||||||||||||||||
| const sku = typeof fin.gpu_sku === "string" && fin.gpu_sku !== "" ? fin.gpu_sku : undefined; | ||||||||||||||||||||||||||||||||||
| const secs = typeof fin.gpu_seconds === "number" && Number.isFinite(fin.gpu_seconds) && fin.gpu_seconds > 0 ? fin.gpu_seconds : undefined; | ||||||||||||||||||||||||||||||||||
| const cost = typeof fin.cost_usd === "number" && Number.isFinite(fin.cost_usd) && fin.cost_usd > 0 ? fin.cost_usd : undefined; | ||||||||||||||||||||||||||||||||||
| if (sku !== undefined || secs !== undefined || cost !== undefined) | ||||||||||||||||||||||||||||||||||
| gpuReceipt = { ...(sku !== undefined ? { gpu_sku: sku } : {}), ...(secs !== undefined ? { gpu_seconds: secs } : {}), ...(cost !== undefined ? { cost_usd: cost } : {}) }; | ||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||
| const f = s.finished?.status; | ||||||||||||||||||||||||||||||||||
| if (f === "completed" || f === "failed" || f === "aborted") settle(f, EXIT[f]); | ||||||||||||||||||||||||||||||||||
| }; | ||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -397,3 +397,48 @@ describe("stats records arrive in either shape", () => { | |
| }); | ||
| }); | ||
| }); | ||
|
|
||
| // ── GPU receipt accounting (#425): remote settle → receipt ledger row ────── | ||
| describe("RemoteExecutor — GPU receipt accounting (#425)", () => { | ||
| it("a completed run whose finished payload carries GPU fields emits a receipt ledger row + mirror receipt.toml", async () => { | ||
| await withCloud(async (fake) => { | ||
| fake.state.finished = { status: "completed", gpu_sku: "H100-80GB", gpu_seconds: 900, cost_usd: 2.7 }; | ||
| const ledgerFile = join(tmpRoot(), `ledger-${Date.now()}.jsonl`); | ||
| process.env.AMICO_LEDGER = ledgerFile; | ||
| try { | ||
| const h = await ex(fake, { pollMs: 5, warmingBudgetMs: 5000, lostAfterMs: 5000 }) | ||
| .submit(fakeJulia(tmpRoot(), "solve.jl", "// julia body"), { runsRoot: join(tmpRoot(), "runs"), lab: "t" }); | ||
| for await (const _ of h.events) void _; | ||
| const rows = readFileSync(ledgerFile, "utf8").trim().split("\n").map((l) => JSON.parse(l)); | ||
| const receipt = rows.find((r) => r.type === "receipt"); | ||
| expect(receipt).toMatchObject({ | ||
| task_id: fake.taskId, executor: "remote", | ||
| gpu_sku: "H100-80GB", gpu_seconds: 900, cost_usd: 2.7, | ||
| }); | ||
| // the mirror carries the durable artifact too | ||
| const mirror = join(h.runDir, "receipt.toml"); | ||
| expect(readFileSync(mirror, "utf8")).toContain('gpu_sku = "H100-80GB"'); | ||
| } finally { | ||
| delete process.env.AMICO_LEDGER; | ||
| } | ||
| }); | ||
| }); | ||
|
Comment on lines
+403
to
+425
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win Add failed-run receipt coverage. The PR requires accounting for failed remote runs, but this test only covers 🤖 Prompt for AI Agents |
||
| it("a finished payload WITHOUT GPU fields emits no receipt row (nothing to account)", async () => { | ||
| await withCloud(async (fake) => { | ||
| fake.state.finished = { status: "completed" }; | ||
| const ledgerFile = join(tmpRoot(), `ledger-${Date.now()}.jsonl`); | ||
| process.env.AMICO_LEDGER = ledgerFile; | ||
| try { | ||
| const h = await ex(fake, { pollMs: 5, warmingBudgetMs: 5000, lostAfterMs: 5000 }) | ||
| .submit(fakeJulia(tmpRoot(), "solve.jl", "// julia body"), { runsRoot: join(tmpRoot(), "runs"), lab: "t2" }); | ||
| for await (const _ of h.events) void _; | ||
| if (existsSync(ledgerFile)) { | ||
| const rows = readFileSync(ledgerFile, "utf8").trim().split("\n").map((l) => JSON.parse(l)); | ||
| expect(rows.some((r) => r.type === "receipt")).toBe(false); | ||
| } | ||
| } finally { | ||
| delete process.env.AMICO_LEDGER; | ||
| } | ||
| }); | ||
| }); | ||
| }); | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🗄️ Data Integrity & Integration | 🟠 Major | ⚡ Quick win
Do not report zero spend for every ledger read error.
Lines 331-335 return zero totals for permission errors, I/O errors, and directory paths.
ledger gpucan then report no spend while receipt data exists but is unreadable. Return zero only forENOENT. Surface other read failures.Proposed fix
📝 Committable suggestion
🤖 Prompt for AI Agents