From dcac2c4cf5ce886173146319fd827fc06d99132c Mon Sep 17 00:00:00 2001 From: Jiun Bae Date: Mon, 10 Aug 2026 08:59:14 +0900 Subject: [PATCH] feat: price new models by default and explain token totals Fleet reports read as inflated and undercount cost. Three causes, all fixed here. Pricing defaulted to offline, unlike ccusage itself, so every model newer than the snapshot bundled with ccusage was priced at $0 with no indication. On a four-host fleet that silently dropped 9.7% of tokens (11.9B, mostly claude-opus-5) out of the cost column. Default to online pricing to match ccusage; --offline still selects the bundled snapshot. Total tokens is dominated by cache reads -- 96.6% on the same fleet -- because every turn replays the cached context. Bars scaled by total tokens therefore track context size rather than work, and diverge sharply from cost. Add --graph-metric output alongside tokens and cost. Print Note: lines under the table covering the cache-read share, models missing from the pricing table, and the reasoning tokens ccusage folds into total tokens without a column. --json gains a notes array; schemaVersion is unchanged. Also bump ccusage to 20.0.19 to refresh the bundled pricing snapshot. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 23 +++++++++++-- ccusage-fleet.schema.json | 2 +- package-lock.json | 56 +++++++++++++++--------------- package.json | 2 +- src/args.js | 4 +-- src/cli.js | 7 +++- src/config.js | 6 ++-- src/format.js | 12 +++++++ src/graph.js | 32 +++++++---------- src/insights.js | 72 +++++++++++++++++++++++++++++++++++++++ test/args-config.test.js | 15 ++++++++ test/graph.test.js | 14 ++++++-- test/insights.test.js | 57 +++++++++++++++++++++++++++++++ 13 files changed, 243 insertions(+), 59 deletions(-) create mode 100644 src/format.js create mode 100644 src/insights.js create mode 100644 test/insights.test.js diff --git a/README.md b/README.md index 6c89d92..9d802b9 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,9 @@ npx ccusage-fleet daily --hosts localhost,rtzr --group-by none # Append a report-bucket token distribution graph npx ccusage-fleet daily --hosts localhost,rtzr --graph -# Scale the graph by estimated cost +# Scale the graph by estimated cost, or by output tokens npx ccusage-fleet daily --hosts localhost,rtzr --graph --graph-metric cost +npx ccusage-fleet daily --hosts localhost,rtzr --graph --graph-metric output # One consistent date range and timezone on every device npx ccusage-fleet daily \ @@ -101,10 +102,28 @@ The former `--by-host` and `--no-by-host` options remain as aliases for `--group ## Graph -Add `--graph` to append a proportional distribution chart after the table. Bars use total tokens by default; choose `--graph-metric cost` to scale them by estimated cost. Buckets follow the report command: days for `daily`, weeks for `weekly`, and months for `monthly`. +Add `--graph` to append a proportional distribution chart after the table. Buckets follow the report command: days for `daily`, weeks for `weekly`, and months for `monthly`. + +| `--graph-metric` | Bars scale by | +| --- | --- | +| `tokens` (default) | Total tokens, which cache reads usually dominate | +| `output` | Output tokens, the closest proxy for work performed | +| `cost` | Estimated cost | Hour buckets will require a future normalized event export from ccusage; ccusage-fleet does not copy raw transcripts to approximate them. +## Reading the numbers + +Total tokens counts every token an agent sent or received, and cache reads are typically over 90% of that: each turn replays the whole cached context, billed at a fraction of the input rate. A fleet-wide total spanning months across several machines therefore reaches billions of tokens without anything being double counted. Use `--graph-metric output` or the `Output` column to compare actual work, and `--since` to narrow the `Total` row to a period you care about. + +ccusage-fleet prints a `Note:` line under the table whenever the report needs that context: + +- how much of the total is cache reads +- models with no entry in the pricing table, which count as `$0` and make the reported cost a lower bound +- reasoning tokens that ccusage folds into total tokens without a column of their own + +Costs are list-price estimates from ccusage's pricing table, not billed amounts, and they are meaningless on subscription plans. Pricing is fetched online by default so that recently released models are priced; `--offline` uses the snapshot bundled with ccusage instead, which reports `$0` for any model newer than that snapshot. + ## Failure handling By default, reachable hosts are reported even when another host fails. Add `--strict` to return a non-zero exit status if any host fails. `--timeout` is applied per host. diff --git a/ccusage-fleet.schema.json b/ccusage-fleet.schema.json index b0daef8..10a67bf 100644 --- a/ccusage-fleet.schema.json +++ b/ccusage-fleet.schema.json @@ -27,7 +27,7 @@ "timezone": { "type": "string" }, "groupBy": { "enum": ["agent", "device", "none"] }, "graph": { "type": "boolean" }, - "graphMetric": { "enum": ["tokens", "cost"] }, + "graphMetric": { "enum": ["tokens", "output", "cost"] }, "byHost": { "type": "boolean" }, "offline": { "type": "boolean" }, "noCost": { "type": "boolean" }, diff --git a/package-lock.json b/package-lock.json index 7e036ef..09794ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.3.0", "license": "MIT", "dependencies": { - "ccusage": "20.0.17" + "ccusage": "20.0.19" }, "bin": { "ccusage-fleet": "bin/ccusage-fleet.js" @@ -19,9 +19,9 @@ } }, "node_modules/@ccusage/ccusage-darwin-arm64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-arm64/-/ccusage-darwin-arm64-20.0.17.tgz", - "integrity": "sha512-rQTuzovMtfbtBAmEMmC8ABzkd66E+jJI+E+WJ/MbEeRTGS/7cL50BpP0jI87FWTbGoY/ovlSV6gcmFyd+EvzXA==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-arm64/-/ccusage-darwin-arm64-20.0.19.tgz", + "integrity": "sha512-8b69h4tuMH1KNW+EYjFRJWpbWHXbMUtFu8cOCyFOQsR6lGxq37g0kzEGHSqG+iZEoUTHaabR5CE7ZHLhW9Q64A==", "cpu": [ "arm64" ], @@ -32,9 +32,9 @@ ] }, "node_modules/@ccusage/ccusage-darwin-x64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-x64/-/ccusage-darwin-x64-20.0.17.tgz", - "integrity": "sha512-FyNVZ1I7LZpkuIXB/pUE+e7OiUeEKKM1h/3Sq0bfq+zZL2SIGmWLTH5RiANcVFIhlEJYE2J2g32z2wy6zYJ9vQ==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-x64/-/ccusage-darwin-x64-20.0.19.tgz", + "integrity": "sha512-YTsqIfsPo3qR9OMrwJtnJoq/fil7Jk5mzXV8k0Iejpiq7zysw4FfkjCeHr0UVJG8ihdqtE18o+xIk99p3v+5Ew==", "cpu": [ "x64" ], @@ -45,9 +45,9 @@ ] }, "node_modules/@ccusage/ccusage-linux-arm64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-arm64/-/ccusage-linux-arm64-20.0.17.tgz", - "integrity": "sha512-evdCI+3G6tDoXubPFDQju5KNc+fy2NTjZnUs8nBIwh/oYIdo6LVYlB/Tpvf5lz3WLy8+VPfcc0UuXIjY/X1vkg==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-arm64/-/ccusage-linux-arm64-20.0.19.tgz", + "integrity": "sha512-4e+7hV3WrEpM7hQPQkh4/zNLTUSqzpd+vSSWl2y659+xQ+JCDLWqE6gzvXLMOAT/LTtcFtlrduIWDZt8VPceqQ==", "cpu": [ "arm64" ], @@ -58,9 +58,9 @@ ] }, "node_modules/@ccusage/ccusage-linux-x64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-x64/-/ccusage-linux-x64-20.0.17.tgz", - "integrity": "sha512-o9CFzPbRT+WEyhKV6Y0GUdxcxdrRnToBY7qclXMe+doKon4gjzZvAzHXR6qao/ZHdfg6cmLkGvBJx5ZcG/8bEw==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-x64/-/ccusage-linux-x64-20.0.19.tgz", + "integrity": "sha512-VPbB6onrwOGojTKutFzeahttb98ZgmdsiQpOr/BJet1i3QdrmUGXGmJSt591xXrvVOVVgsC7/5wBELdWM9/bKA==", "cpu": [ "x64" ], @@ -71,9 +71,9 @@ ] }, "node_modules/@ccusage/ccusage-win32-arm64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-arm64/-/ccusage-win32-arm64-20.0.17.tgz", - "integrity": "sha512-+cucn4UnhXxd3FQElpAu/k0dgAIN+Sh36dBEfymyuaSHKRvkG7QjbEWkuqsnXwjAy2Z0BIM5uq6gZkrhl0c3aA==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-arm64/-/ccusage-win32-arm64-20.0.19.tgz", + "integrity": "sha512-oOv4mZkapxzvQeAMI+jqfTOv/zN2x2AYPAD+F4AnYHQQL4n898axhH9uPd2Ls0U5CDlvfh1zDTyNILz8X9M15A==", "cpu": [ "arm64" ], @@ -84,9 +84,9 @@ ] }, "node_modules/@ccusage/ccusage-win32-x64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-x64/-/ccusage-win32-x64-20.0.17.tgz", - "integrity": "sha512-y+rbb5D3lUQwu6kKtRk2yVGZn3Az3+9TipRQAyk9vs+K4hp3TM+ToqciVRAZ3u+tap5VhPSj2bZt4J6gGXg18w==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-x64/-/ccusage-win32-x64-20.0.19.tgz", + "integrity": "sha512-KtDlTk07uv8cu6TlizB/RptcDiQePE1YSQe6uNNZTUnwVy9zq5R3fvUUFr9/d3FYHwimSGduoHXRS8Ta+Kmkrw==", "cpu": [ "x64" ], @@ -97,9 +97,9 @@ ] }, "node_modules/ccusage": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/ccusage/-/ccusage-20.0.17.tgz", - "integrity": "sha512-MJU4qDs6DOMdam0PXWWFgo0dw/kXAK05rX586DdxIARTzj/zJylWfVlIywgJwW094nNbqwJyGRZvzO2ZsczXDg==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/ccusage/-/ccusage-20.0.19.tgz", + "integrity": "sha512-vyNIyctcvTKFbrsU/WJuWiII3CVw8WopU/0NcR+lKmRbh62wm0jccmkWrFCG6j7R8agc8mIBz5+YlmGbXH1RnQ==", "license": "MIT", "bin": { "ccusage": "src/cli.js" @@ -108,12 +108,12 @@ "url": "https://github.com/sponsors/ryoppippi" }, "optionalDependencies": { - "@ccusage/ccusage-darwin-arm64": "20.0.17", - "@ccusage/ccusage-darwin-x64": "20.0.17", - "@ccusage/ccusage-linux-arm64": "20.0.17", - "@ccusage/ccusage-linux-x64": "20.0.17", - "@ccusage/ccusage-win32-arm64": "20.0.17", - "@ccusage/ccusage-win32-x64": "20.0.17" + "@ccusage/ccusage-darwin-arm64": "20.0.19", + "@ccusage/ccusage-darwin-x64": "20.0.19", + "@ccusage/ccusage-linux-arm64": "20.0.19", + "@ccusage/ccusage-linux-x64": "20.0.19", + "@ccusage/ccusage-win32-arm64": "20.0.19", + "@ccusage/ccusage-win32-x64": "20.0.19" } } } diff --git a/package.json b/package.json index 9af9097..3e645bc 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,6 @@ "node": ">=20" }, "dependencies": { - "ccusage": "20.0.17" + "ccusage": "20.0.19" } } diff --git a/src/args.js b/src/args.js index 8ead174..cff8aba 100644 --- a/src/args.js +++ b/src/args.js @@ -107,9 +107,9 @@ REPORT --group-by Group detail rows by agent, device, or none (default: agent) --by-host / --no-by-host Compatibility aliases for --group-by device/none --graph / --no-graph Show a report-bucket distribution graph - --graph-metric Scale graph bars by tokens or cost (default: tokens) + --graph-metric Scale graph bars by tokens, output, or cost (default: tokens) --no-cost Hide costs - --offline / --online Use bundled or online ccusage pricing (default: offline) + --offline / --online Use bundled or online ccusage pricing (default: online) EXECUTION --timeout Per-host timeout (default: 120000) diff --git a/src/cli.js b/src/cli.js index 6d7eea8..82e7d40 100644 --- a/src/cli.js +++ b/src/cli.js @@ -8,6 +8,7 @@ import { discoverConfigPath, loadConfig, resolveSettings } from './config.js'; import { mapConcurrent, runHost } from './runner.js'; import { renderFleetTable } from './table.js'; import { renderFleetGraph } from './graph.js'; +import { fleetNotes, renderNotes } from './insights.js'; const here = dirname(fileURLToPath(import.meta.url)); const packageJson = JSON.parse(readFileSync(resolve(here, '..', 'package.json'), 'utf8')); @@ -60,10 +61,14 @@ export async function runCli(argv, dependencies = {}) { } const fleet = aggregateFleet(results, settings); + const notes = fleetNotes(fleet, settings); if (settings.json) { - process.stdout.write(`${JSON.stringify(fleet, null, 2)}\n`); + process.stdout.write(`${JSON.stringify({ ...fleet, notes }, null, 2)}\n`); } else { const sections = [renderFleetTable(fleet, settings)]; + if (notes.length > 0) { + sections.push(renderNotes(notes)); + } if (settings.graph) { sections.push(renderFleetGraph(fleet, settings)); } diff --git a/src/config.js b/src/config.js index 488665b..c2ed891 100644 --- a/src/config.js +++ b/src/config.js @@ -144,8 +144,8 @@ export function resolveSettings(parsedOptions, config, defaults) { throw new Error(`group-by must be agent, device, or none (received '${groupBy}')`); } const graphMetric = parsedOptions.graphMetric ?? config.graphMetric ?? 'tokens'; - if (!['tokens', 'cost'].includes(graphMetric)) { - throw new Error(`graph-metric must be tokens or cost (received '${graphMetric}')`); + if (!['tokens', 'output', 'cost'].includes(graphMetric)) { + throw new Error(`graph-metric must be tokens, output, or cost (received '${graphMetric}')`); } const noCost = parsedOptions.noCost ?? config.noCost ?? false; if (graphMetric === 'cost' && noCost) { @@ -163,7 +163,7 @@ export function resolveSettings(parsedOptions, config, defaults) { hosts, json: parsedOptions.json ?? false, noCost, - offline: parsedOptions.offline ?? config.offline ?? true, + offline: parsedOptions.offline ?? config.offline ?? false, since: parsedOptions.since, sshConnectTimeoutSeconds: integer( parsedOptions.sshConnectTimeoutSeconds ?? config.sshConnectTimeoutSeconds, diff --git a/src/format.js b/src/format.js new file mode 100644 index 0000000..d825997 --- /dev/null +++ b/src/format.js @@ -0,0 +1,12 @@ +export function number(value) { + return typeof value === 'number' && Number.isFinite(value) ? value : 0; +} + +export function compact(value, prefix = '') { + const numeric = number(value); + const absolute = Math.abs(numeric); + if (absolute >= 1_000_000_000) return `${prefix}${(numeric / 1_000_000_000).toFixed(1)}B`; + if (absolute >= 1_000_000) return `${prefix}${(numeric / 1_000_000).toFixed(1)}M`; + if (absolute >= 1_000) return `${prefix}${(numeric / 1_000).toFixed(1)}K`; + return `${prefix}${Math.round(numeric)}`; +} diff --git a/src/graph.js b/src/graph.js index adc9ce9..86a798a 100644 --- a/src/graph.js +++ b/src/graph.js @@ -1,15 +1,10 @@ -function number(value) { - return typeof value === 'number' && Number.isFinite(value) ? value : 0; -} +import { compact, number } from './format.js'; -function compact(value, prefix = '') { - const numeric = number(value); - const absolute = Math.abs(numeric); - if (absolute >= 1_000_000_000) return `${prefix}${(numeric / 1_000_000_000).toFixed(1)}B`; - if (absolute >= 1_000_000) return `${prefix}${(numeric / 1_000_000).toFixed(1)}M`; - if (absolute >= 1_000) return `${prefix}${(numeric / 1_000).toFixed(1)}K`; - return `${prefix}${Math.round(numeric)}`; -} +const METRICS = new Map([ + ['cost', { key: 'totalCost', prefix: '$', title: 'Cost' }], + ['output', { key: 'outputTokens', prefix: '', title: 'Output tokens' }], + ['tokens', { key: 'totalTokens', prefix: '', title: 'Total tokens' }], +]); function bar(value, peak, width) { if (value <= 0 || peak <= 0) return ' '.repeat(width); @@ -38,25 +33,24 @@ function border(left, middle, right, widths, fill = '─') { export function renderFleetGraph(fleet, settings, terminalWidth = process.stdout.columns ?? 120) { const rows = fleet[fleet.command]; const bucketName = fleet.command === 'weekly' ? 'week' : fleet.command === 'monthly' ? 'month' : 'day'; - const metricKey = settings.graphMetric === 'cost' ? 'totalCost' : 'totalTokens'; - const values = rows.map((row) => number(row[metricKey])); + const metric = METRICS.get(settings.graphMetric) ?? METRICS.get('tokens'); + const values = rows.map((row) => number(row[metric.key])); const total = values.reduce((sum, value) => sum + value, 0); const peak = Math.max(0, ...values); - const metricTitle = settings.graphMetric === 'cost' ? 'Cost' : 'Total tokens'; - const metricValue = (value) => settings.graphMetric === 'cost' ? compact(value, '$') : compact(value); - const title = `${metricTitle} over time · ${bucketName} buckets · total ${metricValue(total)} · peak ${metricValue(peak)}`; + const metricValue = (value) => compact(value, metric.prefix); + const title = `${metric.title} over time · ${bucketName} buckets · total ${metricValue(total)} · peak ${metricValue(peak)}`; const widths = [10, 10, 10, Math.max(16, Math.min(60, terminalWidth - 46))]; const output = [title, border('╭', '┬', '╮', widths)]; - const headers = ['BUCKET', 'TOKENS', 'COST', 'DISTRIBUTION']; + const headers = ['BUCKET', settings.graphMetric === 'output' ? 'OUTPUT' : 'TOKENS', 'COST', 'DISTRIBUTION']; output.push(`│ ${headers.map((header, index) => pad(header, widths[index])).join(' ┆ ')} │`); output.push(border('╞', '╪', '╡', widths, '═')); for (const row of rows) { const cells = [ pad(row.period, widths[0]), - pad(compact(row.totalTokens), widths[1], true), + pad(compact(settings.graphMetric === 'output' ? row.outputTokens : row.totalTokens), widths[1], true), pad(settings.noCost ? '—' : compact(row.totalCost, '$'), widths[2], true), - bar(number(row[metricKey]), peak, widths[3]), + bar(number(row[metric.key]), peak, widths[3]), ]; output.push(`│ ${cells.join(' ┆ ')} │`); } diff --git a/src/insights.js b/src/insights.js new file mode 100644 index 0000000..4b4bfee --- /dev/null +++ b/src/insights.js @@ -0,0 +1,72 @@ +import { compact, number } from './format.js'; + +const COMPONENT_FIELDS = ['inputTokens', 'outputTokens', 'cacheCreationTokens', 'cacheReadTokens']; +const LISTED_MODELS = 3; +const CACHE_READ_NOTE_THRESHOLD = 0.5; + +function componentSum(usage) { + return COMPONENT_FIELDS.reduce((sum, field) => sum + number(usage[field]), 0); +} + +// ccusage reports one cost per model, so a model priced at exactly zero while it +// still moved tokens means the pricing table had no entry for it. +export function unpricedModels(totals) { + return (totals?.modelBreakdowns ?? []) + .map((item) => ({ modelName: item.modelName ?? 'unknown', tokens: componentSum(item), cost: number(item.cost) })) + .filter((item) => item.tokens > 0 && item.cost === 0) + .sort((a, b) => b.tokens - a.tokens); +} + +// ccusage folds agent-specific reasoning tokens into totalTokens without giving +// them a column of their own, so the visible columns can add up to slightly less. +export function reasoningDrift(totals) { + return number(totals?.totalTokens) - componentSum(totals); +} + +export function fleetNotes(fleet, settings = {}) { + const totals = fleet?.totals ?? {}; + const totalTokens = number(totals.totalTokens); + const notes = []; + if (totalTokens <= 0) { + return notes; + } + + const cacheRead = number(totals.cacheReadTokens); + if (cacheRead / totalTokens > CACHE_READ_NOTE_THRESHOLD) { + notes.push( + `Cache reads are ${((cacheRead / totalTokens) * 100).toFixed(1)}% of Total Tokens ` + + `(${compact(cacheRead)} of ${compact(totalTokens)}); output is ${compact(totals.outputTokens)}. ` + + 'Total Tokens tracks context replay, not work performed.', + ); + } + + if (!settings.noCost) { + const unpriced = unpricedModels(totals); + if (unpriced.length > 0) { + const unpricedTokens = unpriced.reduce((sum, item) => sum + item.tokens, 0); + const listed = unpriced.slice(0, LISTED_MODELS).map((item) => `${item.modelName} ${compact(item.tokens)}`); + const remaining = unpriced.length - listed.length; + const suffix = remaining > 0 ? `, +${remaining} more` : ''; + notes.push( + `${unpriced.length} model(s) have no pricing data and count as $0 ` + + `(${((unpricedTokens / totalTokens) * 100).toFixed(1)}% of tokens): ${listed.join(', ')}${suffix}. ` + + 'Reported cost is a lower bound.' + + (settings.offline ? ' Re-run with --online to fetch current pricing.' : ''), + ); + } + } + + const drift = reasoningDrift(totals); + if (drift !== 0) { + notes.push( + `${compact(drift)} tokens (${((Math.abs(drift) / totalTokens) * 100).toFixed(4)}%) are reasoning tokens ` + + 'counted in Total Tokens but not in the Input/Output/Cache columns.', + ); + } + + return notes; +} + +export function renderNotes(notes) { + return notes.map((note) => `Note: ${note}`).join('\n'); +} diff --git a/test/args-config.test.js b/test/args-config.test.js index 79ad223..d9fec3f 100644 --- a/test/args-config.test.js +++ b/test/args-config.test.js @@ -56,10 +56,25 @@ test('configures token and cost graphs', () => { const cost = resolveSettings(parseArgs(['--graph', '--graph-metric', 'cost']).options, {}, defaults); assert.equal(cost.graphMetric, 'cost'); + + const outputTokens = resolveSettings(parseArgs(['--graph', '--graph-metric', 'output']).options, {}, defaults); + assert.equal(outputTokens.graphMetric, 'output'); + assert.throws( () => resolveSettings(parseArgs(['--graph-metric', 'cost', '--no-cost']).options, {}, defaults), /cannot be combined with --no-cost/, ); + assert.throws( + () => resolveSettings(parseArgs(['--graph-metric', 'reads']).options, {}, defaults), + /graph-metric must be tokens, output, or cost/, + ); +}); + +test('fetches pricing online by default so new models are not counted as $0', () => { + assert.equal(resolveSettings(parseArgs([]).options, {}, defaults).offline, false); + assert.equal(resolveSettings(parseArgs(['--offline']).options, {}, defaults).offline, true); + assert.equal(resolveSettings(parseArgs([]).options, { offline: true }, defaults).offline, true); + assert.equal(resolveSettings(parseArgs(['--online']).options, { offline: true }, defaults).offline, false); }); test('rejects SSH option and shell injection', () => { diff --git a/test/graph.test.js b/test/graph.test.js index ce0f2ef..352c47f 100644 --- a/test/graph.test.js +++ b/test/graph.test.js @@ -6,8 +6,8 @@ import { renderFleetGraph } from '../src/graph.js'; const fleet = { command: 'daily', daily: [ - { period: '2026-07-14', totalCost: 25, totalTokens: 500 }, - { period: '2026-07-15', totalCost: 50, totalTokens: 1000 }, + { outputTokens: 20, period: '2026-07-14', totalCost: 25, totalTokens: 500 }, + { outputTokens: 10, period: '2026-07-15', totalCost: 50, totalTokens: 1000 }, ], }; @@ -26,6 +26,16 @@ test('supports cost-scaled graphs', () => { assert.match(output, /total \$75/); }); +test('supports output-scaled graphs so cache reads cannot dominate', () => { + const output = renderFleetGraph(fleet, { graphMetric: 'output', noCost: false }, 100); + assert.match(output, /Output tokens over time · day buckets · total 30 · peak 20/); + assert.match(output, /OUTPUT/); + // 2026-07-14 has fewer total tokens but more output, so it must own the peak bar. + const blocks = (line) => (line.match(/█/g) ?? []).length; + const [first, second] = output.split('\n').filter((line) => line.includes('2026-07-1')); + assert.ok(blocks(first) > blocks(second), `${blocks(first)} should exceed ${blocks(second)}`); +}); + test('uses the active report bucket label', () => { for (const [command, bucket] of [['weekly', 'week'], ['monthly', 'month']]) { const report = { command, [command]: fleet.daily }; diff --git a/test/insights.test.js b/test/insights.test.js new file mode 100644 index 0000000..cb8c06e --- /dev/null +++ b/test/insights.test.js @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { fleetNotes, reasoningDrift, unpricedModels } from '../src/insights.js'; + +function fleetWith(totals) { + return { command: 'daily', daily: [], totals }; +} + +const totals = { + cacheCreationTokens: 1_000, + cacheReadTokens: 960_000, + inputTokens: 20_000, + modelBreakdowns: [ + { cacheCreationTokens: 500, cacheReadTokens: 460_000, cost: 12.5, inputTokens: 10_000, modelName: 'priced-model', outputTokens: 9_000 }, + { cacheCreationTokens: 500, cacheReadTokens: 500_000, cost: 0, inputTokens: 10_000, modelName: 'new-model', outputTokens: 10_000 }, + ], + outputTokens: 19_000, + totalCost: 12.5, + totalTokens: 1_000_000, +}; + +test('flags models that moved tokens but carry no price', () => { + const unpriced = unpricedModels(totals); + assert.equal(unpriced.length, 1); + assert.equal(unpriced[0].modelName, 'new-model'); + assert.equal(unpriced[0].tokens, 520_500); +}); + +test('reports the gap between total tokens and the visible columns', () => { + assert.equal(reasoningDrift(totals), 0); + assert.equal(reasoningDrift({ ...totals, totalTokens: 1_002_500 }), 2_500); +}); + +test('explains cache-read share, unpriced models, and reasoning tokens', () => { + const notes = fleetNotes(fleetWith({ ...totals, totalTokens: 1_002_500 }), { offline: true }); + assert.equal(notes.length, 3); + assert.match(notes[0], /Cache reads are 95\.8% of Total Tokens/); + assert.match(notes[1], /1 model\(s\) have no pricing data and count as \$0 \(51\.9% of tokens\): new-model 520\.5K/); + assert.match(notes[1], /Re-run with --online/); + assert.match(notes[2], /2\.5K tokens .* are reasoning tokens/); +}); + +test('omits the --online hint when pricing is already fetched online', () => { + const notes = fleetNotes(fleetWith(totals), { offline: false }); + assert.ok(notes.some((note) => note.includes('no pricing data'))); + assert.ok(notes.every((note) => !note.includes('--online'))); +}); + +test('suppresses pricing notes when costs are hidden', () => { + const notes = fleetNotes(fleetWith(totals), { noCost: true }); + assert.ok(notes.every((note) => !note.includes('no pricing data'))); +}); + +test('returns no notes for an empty fleet', () => { + assert.deepEqual(fleetNotes(fleetWith({ totalTokens: 0 }), {}), []); +});