diff --git a/README.md b/README.md index 6c89d92..9d802b9 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,9 @@ npx ccusage-fleet daily --hosts localhost,rtzr --group-by none # Append a report-bucket token distribution graph npx ccusage-fleet daily --hosts localhost,rtzr --graph -# Scale the graph by estimated cost +# Scale the graph by estimated cost, or by output tokens npx ccusage-fleet daily --hosts localhost,rtzr --graph --graph-metric cost +npx ccusage-fleet daily --hosts localhost,rtzr --graph --graph-metric output # One consistent date range and timezone on every device npx ccusage-fleet daily \ @@ -101,10 +102,28 @@ The former `--by-host` and `--no-by-host` options remain as aliases for `--group ## Graph -Add `--graph` to append a proportional distribution chart after the table. Bars use total tokens by default; choose `--graph-metric cost` to scale them by estimated cost. Buckets follow the report command: days for `daily`, weeks for `weekly`, and months for `monthly`. +Add `--graph` to append a proportional distribution chart after the table. Buckets follow the report command: days for `daily`, weeks for `weekly`, and months for `monthly`. + +| `--graph-metric` | Bars scale by | +| --- | --- | +| `tokens` (default) | Total tokens, which cache reads usually dominate | +| `output` | Output tokens, the closest proxy for work performed | +| `cost` | Estimated cost | Hour buckets will require a future normalized event export from ccusage; ccusage-fleet does not copy raw transcripts to approximate them. +## Reading the numbers + +Total tokens counts every token an agent sent or received, and cache reads are typically over 90% of that: each turn replays the whole cached context, billed at a fraction of the input rate. A fleet-wide total spanning months across several machines therefore reaches billions of tokens without anything being double counted. Use `--graph-metric output` or the `Output` column to compare actual work, and `--since` to narrow the `Total` row to a period you care about. + +ccusage-fleet prints a `Note:` line under the table whenever the report needs that context: + +- how much of the total is cache reads +- models with no entry in the pricing table, which count as `$0` and make the reported cost a lower bound +- reasoning tokens that ccusage folds into total tokens without a column of their own + +Costs are list-price estimates from ccusage's pricing table, not billed amounts, and they are meaningless on subscription plans. Pricing is fetched online by default so that recently released models are priced; `--offline` uses the snapshot bundled with ccusage instead, which reports `$0` for any model newer than that snapshot. + ## Failure handling By default, reachable hosts are reported even when another host fails. Add `--strict` to return a non-zero exit status if any host fails. `--timeout` is applied per host. diff --git a/ccusage-fleet.schema.json b/ccusage-fleet.schema.json index b0daef8..10a67bf 100644 --- a/ccusage-fleet.schema.json +++ b/ccusage-fleet.schema.json @@ -27,7 +27,7 @@ "timezone": { "type": "string" }, "groupBy": { "enum": ["agent", "device", "none"] }, "graph": { "type": "boolean" }, - "graphMetric": { "enum": ["tokens", "cost"] }, + "graphMetric": { "enum": ["tokens", "output", "cost"] }, "byHost": { "type": "boolean" }, "offline": { "type": "boolean" }, "noCost": { "type": "boolean" }, diff --git a/package-lock.json b/package-lock.json index 7e036ef..09794ae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "0.3.0", "license": "MIT", "dependencies": { - "ccusage": "20.0.17" + "ccusage": "20.0.19" }, "bin": { "ccusage-fleet": "bin/ccusage-fleet.js" @@ -19,9 +19,9 @@ } }, "node_modules/@ccusage/ccusage-darwin-arm64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-arm64/-/ccusage-darwin-arm64-20.0.17.tgz", - "integrity": "sha512-rQTuzovMtfbtBAmEMmC8ABzkd66E+jJI+E+WJ/MbEeRTGS/7cL50BpP0jI87FWTbGoY/ovlSV6gcmFyd+EvzXA==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-arm64/-/ccusage-darwin-arm64-20.0.19.tgz", + "integrity": "sha512-8b69h4tuMH1KNW+EYjFRJWpbWHXbMUtFu8cOCyFOQsR6lGxq37g0kzEGHSqG+iZEoUTHaabR5CE7ZHLhW9Q64A==", "cpu": [ "arm64" ], @@ -32,9 +32,9 @@ ] }, "node_modules/@ccusage/ccusage-darwin-x64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-x64/-/ccusage-darwin-x64-20.0.17.tgz", - "integrity": "sha512-FyNVZ1I7LZpkuIXB/pUE+e7OiUeEKKM1h/3Sq0bfq+zZL2SIGmWLTH5RiANcVFIhlEJYE2J2g32z2wy6zYJ9vQ==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-x64/-/ccusage-darwin-x64-20.0.19.tgz", + "integrity": "sha512-YTsqIfsPo3qR9OMrwJtnJoq/fil7Jk5mzXV8k0Iejpiq7zysw4FfkjCeHr0UVJG8ihdqtE18o+xIk99p3v+5Ew==", "cpu": [ "x64" ], @@ -45,9 +45,9 @@ ] }, "node_modules/@ccusage/ccusage-linux-arm64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-arm64/-/ccusage-linux-arm64-20.0.17.tgz", - "integrity": "sha512-evdCI+3G6tDoXubPFDQju5KNc+fy2NTjZnUs8nBIwh/oYIdo6LVYlB/Tpvf5lz3WLy8+VPfcc0UuXIjY/X1vkg==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-arm64/-/ccusage-linux-arm64-20.0.19.tgz", + "integrity": "sha512-4e+7hV3WrEpM7hQPQkh4/zNLTUSqzpd+vSSWl2y659+xQ+JCDLWqE6gzvXLMOAT/LTtcFtlrduIWDZt8VPceqQ==", "cpu": [ "arm64" ], @@ -58,9 +58,9 @@ ] }, "node_modules/@ccusage/ccusage-linux-x64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-x64/-/ccusage-linux-x64-20.0.17.tgz", - "integrity": "sha512-o9CFzPbRT+WEyhKV6Y0GUdxcxdrRnToBY7qclXMe+doKon4gjzZvAzHXR6qao/ZHdfg6cmLkGvBJx5ZcG/8bEw==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-x64/-/ccusage-linux-x64-20.0.19.tgz", + "integrity": "sha512-VPbB6onrwOGojTKutFzeahttb98ZgmdsiQpOr/BJet1i3QdrmUGXGmJSt591xXrvVOVVgsC7/5wBELdWM9/bKA==", "cpu": [ "x64" ], @@ -71,9 +71,9 @@ ] }, "node_modules/@ccusage/ccusage-win32-arm64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-arm64/-/ccusage-win32-arm64-20.0.17.tgz", - "integrity": "sha512-+cucn4UnhXxd3FQElpAu/k0dgAIN+Sh36dBEfymyuaSHKRvkG7QjbEWkuqsnXwjAy2Z0BIM5uq6gZkrhl0c3aA==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-arm64/-/ccusage-win32-arm64-20.0.19.tgz", + "integrity": "sha512-oOv4mZkapxzvQeAMI+jqfTOv/zN2x2AYPAD+F4AnYHQQL4n898axhH9uPd2Ls0U5CDlvfh1zDTyNILz8X9M15A==", "cpu": [ "arm64" ], @@ -84,9 +84,9 @@ ] }, "node_modules/@ccusage/ccusage-win32-x64": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-x64/-/ccusage-win32-x64-20.0.17.tgz", - "integrity": "sha512-y+rbb5D3lUQwu6kKtRk2yVGZn3Az3+9TipRQAyk9vs+K4hp3TM+ToqciVRAZ3u+tap5VhPSj2bZt4J6gGXg18w==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-x64/-/ccusage-win32-x64-20.0.19.tgz", + "integrity": "sha512-KtDlTk07uv8cu6TlizB/RptcDiQePE1YSQe6uNNZTUnwVy9zq5R3fvUUFr9/d3FYHwimSGduoHXRS8Ta+Kmkrw==", "cpu": [ "x64" ], @@ -97,9 +97,9 @@ ] }, "node_modules/ccusage": { - "version": "20.0.17", - "resolved": "https://registry.npmjs.org/ccusage/-/ccusage-20.0.17.tgz", - "integrity": "sha512-MJU4qDs6DOMdam0PXWWFgo0dw/kXAK05rX586DdxIARTzj/zJylWfVlIywgJwW094nNbqwJyGRZvzO2ZsczXDg==", + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/ccusage/-/ccusage-20.0.19.tgz", + "integrity": "sha512-vyNIyctcvTKFbrsU/WJuWiII3CVw8WopU/0NcR+lKmRbh62wm0jccmkWrFCG6j7R8agc8mIBz5+YlmGbXH1RnQ==", "license": "MIT", "bin": { "ccusage": "src/cli.js" @@ -108,12 +108,12 @@ "url": "https://github.com/sponsors/ryoppippi" }, "optionalDependencies": { - "@ccusage/ccusage-darwin-arm64": "20.0.17", - "@ccusage/ccusage-darwin-x64": "20.0.17", - "@ccusage/ccusage-linux-arm64": "20.0.17", - "@ccusage/ccusage-linux-x64": "20.0.17", - "@ccusage/ccusage-win32-arm64": "20.0.17", - "@ccusage/ccusage-win32-x64": "20.0.17" + "@ccusage/ccusage-darwin-arm64": "20.0.19", + "@ccusage/ccusage-darwin-x64": "20.0.19", + "@ccusage/ccusage-linux-arm64": "20.0.19", + "@ccusage/ccusage-linux-x64": "20.0.19", + "@ccusage/ccusage-win32-arm64": "20.0.19", + "@ccusage/ccusage-win32-x64": "20.0.19" } } } diff --git a/package.json b/package.json index 9af9097..3e645bc 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,6 @@ "node": ">=20" }, "dependencies": { - "ccusage": "20.0.17" + "ccusage": "20.0.19" } } diff --git a/src/args.js b/src/args.js index 8ead174..cff8aba 100644 --- a/src/args.js +++ b/src/args.js @@ -107,9 +107,9 @@ REPORT --group-by Group detail rows by agent, device, or none (default: agent) --by-host / --no-by-host Compatibility aliases for --group-by device/none --graph / --no-graph Show a report-bucket distribution graph - --graph-metric Scale graph bars by tokens or cost (default: tokens) + --graph-metric Scale graph bars by tokens, output, or cost (default: tokens) --no-cost Hide costs - --offline / --online Use bundled or online ccusage pricing (default: offline) + --offline / --online Use bundled or online ccusage pricing (default: online) EXECUTION --timeout Per-host timeout (default: 120000) diff --git a/src/cli.js b/src/cli.js index 6d7eea8..82e7d40 100644 --- a/src/cli.js +++ b/src/cli.js @@ -8,6 +8,7 @@ import { discoverConfigPath, loadConfig, resolveSettings } from './config.js'; import { mapConcurrent, runHost } from './runner.js'; import { renderFleetTable } from './table.js'; import { renderFleetGraph } from './graph.js'; +import { fleetNotes, renderNotes } from './insights.js'; const here = dirname(fileURLToPath(import.meta.url)); const packageJson = JSON.parse(readFileSync(resolve(here, '..', 'package.json'), 'utf8')); @@ -60,10 +61,14 @@ export async function runCli(argv, dependencies = {}) { } const fleet = aggregateFleet(results, settings); + const notes = fleetNotes(fleet, settings); if (settings.json) { - process.stdout.write(`${JSON.stringify(fleet, null, 2)}\n`); + process.stdout.write(`${JSON.stringify({ ...fleet, notes }, null, 2)}\n`); } else { const sections = [renderFleetTable(fleet, settings)]; + if (notes.length > 0) { + sections.push(renderNotes(notes)); + } if (settings.graph) { sections.push(renderFleetGraph(fleet, settings)); } diff --git a/src/config.js b/src/config.js index 488665b..c2ed891 100644 --- a/src/config.js +++ b/src/config.js @@ -144,8 +144,8 @@ export function resolveSettings(parsedOptions, config, defaults) { throw new Error(`group-by must be agent, device, or none (received '${groupBy}')`); } const graphMetric = parsedOptions.graphMetric ?? config.graphMetric ?? 'tokens'; - if (!['tokens', 'cost'].includes(graphMetric)) { - throw new Error(`graph-metric must be tokens or cost (received '${graphMetric}')`); + if (!['tokens', 'output', 'cost'].includes(graphMetric)) { + throw new Error(`graph-metric must be tokens, output, or cost (received '${graphMetric}')`); } const noCost = parsedOptions.noCost ?? config.noCost ?? false; if (graphMetric === 'cost' && noCost) { @@ -163,7 +163,7 @@ export function resolveSettings(parsedOptions, config, defaults) { hosts, json: parsedOptions.json ?? false, noCost, - offline: parsedOptions.offline ?? config.offline ?? true, + offline: parsedOptions.offline ?? config.offline ?? false, since: parsedOptions.since, sshConnectTimeoutSeconds: integer( parsedOptions.sshConnectTimeoutSeconds ?? config.sshConnectTimeoutSeconds, diff --git a/src/format.js b/src/format.js new file mode 100644 index 0000000..d825997 --- /dev/null +++ b/src/format.js @@ -0,0 +1,12 @@ +export function number(value) { + return typeof value === 'number' && Number.isFinite(value) ? value : 0; +} + +export function compact(value, prefix = '') { + const numeric = number(value); + const absolute = Math.abs(numeric); + if (absolute >= 1_000_000_000) return `${prefix}${(numeric / 1_000_000_000).toFixed(1)}B`; + if (absolute >= 1_000_000) return `${prefix}${(numeric / 1_000_000).toFixed(1)}M`; + if (absolute >= 1_000) return `${prefix}${(numeric / 1_000).toFixed(1)}K`; + return `${prefix}${Math.round(numeric)}`; +} diff --git a/src/graph.js b/src/graph.js index adc9ce9..86a798a 100644 --- a/src/graph.js +++ b/src/graph.js @@ -1,15 +1,10 @@ -function number(value) { - return typeof value === 'number' && Number.isFinite(value) ? value : 0; -} +import { compact, number } from './format.js'; -function compact(value, prefix = '') { - const numeric = number(value); - const absolute = Math.abs(numeric); - if (absolute >= 1_000_000_000) return `${prefix}${(numeric / 1_000_000_000).toFixed(1)}B`; - if (absolute >= 1_000_000) return `${prefix}${(numeric / 1_000_000).toFixed(1)}M`; - if (absolute >= 1_000) return `${prefix}${(numeric / 1_000).toFixed(1)}K`; - return `${prefix}${Math.round(numeric)}`; -} +const METRICS = new Map([ + ['cost', { key: 'totalCost', prefix: '$', title: 'Cost' }], + ['output', { key: 'outputTokens', prefix: '', title: 'Output tokens' }], + ['tokens', { key: 'totalTokens', prefix: '', title: 'Total tokens' }], +]); function bar(value, peak, width) { if (value <= 0 || peak <= 0) return ' '.repeat(width); @@ -38,25 +33,24 @@ function border(left, middle, right, widths, fill = '─') { export function renderFleetGraph(fleet, settings, terminalWidth = process.stdout.columns ?? 120) { const rows = fleet[fleet.command]; const bucketName = fleet.command === 'weekly' ? 'week' : fleet.command === 'monthly' ? 'month' : 'day'; - const metricKey = settings.graphMetric === 'cost' ? 'totalCost' : 'totalTokens'; - const values = rows.map((row) => number(row[metricKey])); + const metric = METRICS.get(settings.graphMetric) ?? METRICS.get('tokens'); + const values = rows.map((row) => number(row[metric.key])); const total = values.reduce((sum, value) => sum + value, 0); const peak = Math.max(0, ...values); - const metricTitle = settings.graphMetric === 'cost' ? 'Cost' : 'Total tokens'; - const metricValue = (value) => settings.graphMetric === 'cost' ? compact(value, '$') : compact(value); - const title = `${metricTitle} over time · ${bucketName} buckets · total ${metricValue(total)} · peak ${metricValue(peak)}`; + const metricValue = (value) => compact(value, metric.prefix); + const title = `${metric.title} over time · ${bucketName} buckets · total ${metricValue(total)} · peak ${metricValue(peak)}`; const widths = [10, 10, 10, Math.max(16, Math.min(60, terminalWidth - 46))]; const output = [title, border('╭', '┬', '╮', widths)]; - const headers = ['BUCKET', 'TOKENS', 'COST', 'DISTRIBUTION']; + const headers = ['BUCKET', settings.graphMetric === 'output' ? 'OUTPUT' : 'TOKENS', 'COST', 'DISTRIBUTION']; output.push(`│ ${headers.map((header, index) => pad(header, widths[index])).join(' ┆ ')} │`); output.push(border('╞', '╪', '╡', widths, '═')); for (const row of rows) { const cells = [ pad(row.period, widths[0]), - pad(compact(row.totalTokens), widths[1], true), + pad(compact(settings.graphMetric === 'output' ? row.outputTokens : row.totalTokens), widths[1], true), pad(settings.noCost ? '—' : compact(row.totalCost, '$'), widths[2], true), - bar(number(row[metricKey]), peak, widths[3]), + bar(number(row[metric.key]), peak, widths[3]), ]; output.push(`│ ${cells.join(' ┆ ')} │`); } diff --git a/src/insights.js b/src/insights.js new file mode 100644 index 0000000..4b4bfee --- /dev/null +++ b/src/insights.js @@ -0,0 +1,72 @@ +import { compact, number } from './format.js'; + +const COMPONENT_FIELDS = ['inputTokens', 'outputTokens', 'cacheCreationTokens', 'cacheReadTokens']; +const LISTED_MODELS = 3; +const CACHE_READ_NOTE_THRESHOLD = 0.5; + +function componentSum(usage) { + return COMPONENT_FIELDS.reduce((sum, field) => sum + number(usage[field]), 0); +} + +// ccusage reports one cost per model, so a model priced at exactly zero while it +// still moved tokens means the pricing table had no entry for it. +export function unpricedModels(totals) { + return (totals?.modelBreakdowns ?? []) + .map((item) => ({ modelName: item.modelName ?? 'unknown', tokens: componentSum(item), cost: number(item.cost) })) + .filter((item) => item.tokens > 0 && item.cost === 0) + .sort((a, b) => b.tokens - a.tokens); +} + +// ccusage folds agent-specific reasoning tokens into totalTokens without giving +// them a column of their own, so the visible columns can add up to slightly less. +export function reasoningDrift(totals) { + return number(totals?.totalTokens) - componentSum(totals); +} + +export function fleetNotes(fleet, settings = {}) { + const totals = fleet?.totals ?? {}; + const totalTokens = number(totals.totalTokens); + const notes = []; + if (totalTokens <= 0) { + return notes; + } + + const cacheRead = number(totals.cacheReadTokens); + if (cacheRead / totalTokens > CACHE_READ_NOTE_THRESHOLD) { + notes.push( + `Cache reads are ${((cacheRead / totalTokens) * 100).toFixed(1)}% of Total Tokens ` + + `(${compact(cacheRead)} of ${compact(totalTokens)}); output is ${compact(totals.outputTokens)}. ` + + 'Total Tokens tracks context replay, not work performed.', + ); + } + + if (!settings.noCost) { + const unpriced = unpricedModels(totals); + if (unpriced.length > 0) { + const unpricedTokens = unpriced.reduce((sum, item) => sum + item.tokens, 0); + const listed = unpriced.slice(0, LISTED_MODELS).map((item) => `${item.modelName} ${compact(item.tokens)}`); + const remaining = unpriced.length - listed.length; + const suffix = remaining > 0 ? `, +${remaining} more` : ''; + notes.push( + `${unpriced.length} model(s) have no pricing data and count as $0 ` + + `(${((unpricedTokens / totalTokens) * 100).toFixed(1)}% of tokens): ${listed.join(', ')}${suffix}. ` + + 'Reported cost is a lower bound.' + + (settings.offline ? ' Re-run with --online to fetch current pricing.' : ''), + ); + } + } + + const drift = reasoningDrift(totals); + if (drift !== 0) { + notes.push( + `${compact(drift)} tokens (${((Math.abs(drift) / totalTokens) * 100).toFixed(4)}%) are reasoning tokens ` + + 'counted in Total Tokens but not in the Input/Output/Cache columns.', + ); + } + + return notes; +} + +export function renderNotes(notes) { + return notes.map((note) => `Note: ${note}`).join('\n'); +} diff --git a/test/args-config.test.js b/test/args-config.test.js index 79ad223..d9fec3f 100644 --- a/test/args-config.test.js +++ b/test/args-config.test.js @@ -56,10 +56,25 @@ test('configures token and cost graphs', () => { const cost = resolveSettings(parseArgs(['--graph', '--graph-metric', 'cost']).options, {}, defaults); assert.equal(cost.graphMetric, 'cost'); + + const outputTokens = resolveSettings(parseArgs(['--graph', '--graph-metric', 'output']).options, {}, defaults); + assert.equal(outputTokens.graphMetric, 'output'); + assert.throws( () => resolveSettings(parseArgs(['--graph-metric', 'cost', '--no-cost']).options, {}, defaults), /cannot be combined with --no-cost/, ); + assert.throws( + () => resolveSettings(parseArgs(['--graph-metric', 'reads']).options, {}, defaults), + /graph-metric must be tokens, output, or cost/, + ); +}); + +test('fetches pricing online by default so new models are not counted as $0', () => { + assert.equal(resolveSettings(parseArgs([]).options, {}, defaults).offline, false); + assert.equal(resolveSettings(parseArgs(['--offline']).options, {}, defaults).offline, true); + assert.equal(resolveSettings(parseArgs([]).options, { offline: true }, defaults).offline, true); + assert.equal(resolveSettings(parseArgs(['--online']).options, { offline: true }, defaults).offline, false); }); test('rejects SSH option and shell injection', () => { diff --git a/test/graph.test.js b/test/graph.test.js index ce0f2ef..352c47f 100644 --- a/test/graph.test.js +++ b/test/graph.test.js @@ -6,8 +6,8 @@ import { renderFleetGraph } from '../src/graph.js'; const fleet = { command: 'daily', daily: [ - { period: '2026-07-14', totalCost: 25, totalTokens: 500 }, - { period: '2026-07-15', totalCost: 50, totalTokens: 1000 }, + { outputTokens: 20, period: '2026-07-14', totalCost: 25, totalTokens: 500 }, + { outputTokens: 10, period: '2026-07-15', totalCost: 50, totalTokens: 1000 }, ], }; @@ -26,6 +26,16 @@ test('supports cost-scaled graphs', () => { assert.match(output, /total \$75/); }); +test('supports output-scaled graphs so cache reads cannot dominate', () => { + const output = renderFleetGraph(fleet, { graphMetric: 'output', noCost: false }, 100); + assert.match(output, /Output tokens over time · day buckets · total 30 · peak 20/); + assert.match(output, /OUTPUT/); + // 2026-07-14 has fewer total tokens but more output, so it must own the peak bar. + const blocks = (line) => (line.match(/█/g) ?? []).length; + const [first, second] = output.split('\n').filter((line) => line.includes('2026-07-1')); + assert.ok(blocks(first) > blocks(second), `${blocks(first)} should exceed ${blocks(second)}`); +}); + test('uses the active report bucket label', () => { for (const [command, bucket] of [['weekly', 'week'], ['monthly', 'month']]) { const report = { command, [command]: fleet.daily }; diff --git a/test/insights.test.js b/test/insights.test.js new file mode 100644 index 0000000..cb8c06e --- /dev/null +++ b/test/insights.test.js @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { fleetNotes, reasoningDrift, unpricedModels } from '../src/insights.js'; + +function fleetWith(totals) { + return { command: 'daily', daily: [], totals }; +} + +const totals = { + cacheCreationTokens: 1_000, + cacheReadTokens: 960_000, + inputTokens: 20_000, + modelBreakdowns: [ + { cacheCreationTokens: 500, cacheReadTokens: 460_000, cost: 12.5, inputTokens: 10_000, modelName: 'priced-model', outputTokens: 9_000 }, + { cacheCreationTokens: 500, cacheReadTokens: 500_000, cost: 0, inputTokens: 10_000, modelName: 'new-model', outputTokens: 10_000 }, + ], + outputTokens: 19_000, + totalCost: 12.5, + totalTokens: 1_000_000, +}; + +test('flags models that moved tokens but carry no price', () => { + const unpriced = unpricedModels(totals); + assert.equal(unpriced.length, 1); + assert.equal(unpriced[0].modelName, 'new-model'); + assert.equal(unpriced[0].tokens, 520_500); +}); + +test('reports the gap between total tokens and the visible columns', () => { + assert.equal(reasoningDrift(totals), 0); + assert.equal(reasoningDrift({ ...totals, totalTokens: 1_002_500 }), 2_500); +}); + +test('explains cache-read share, unpriced models, and reasoning tokens', () => { + const notes = fleetNotes(fleetWith({ ...totals, totalTokens: 1_002_500 }), { offline: true }); + assert.equal(notes.length, 3); + assert.match(notes[0], /Cache reads are 95\.8% of Total Tokens/); + assert.match(notes[1], /1 model\(s\) have no pricing data and count as \$0 \(51\.9% of tokens\): new-model 520\.5K/); + assert.match(notes[1], /Re-run with --online/); + assert.match(notes[2], /2\.5K tokens .* are reasoning tokens/); +}); + +test('omits the --online hint when pricing is already fetched online', () => { + const notes = fleetNotes(fleetWith(totals), { offline: false }); + assert.ok(notes.some((note) => note.includes('no pricing data'))); + assert.ok(notes.every((note) => !note.includes('--online'))); +}); + +test('suppresses pricing notes when costs are hidden', () => { + const notes = fleetNotes(fleetWith(totals), { noCost: true }); + assert.ok(notes.every((note) => !note.includes('no pricing data'))); +}); + +test('returns no notes for an empty fleet', () => { + assert.deepEqual(fleetNotes(fleetWith({ totalTokens: 0 }), {}), []); +});