From e05d57bfacfaeb13171de67417fac0df8773a82d Mon Sep 17 00:00:00 2001 From: anirudhkumar-nanonets Date: Thu, 15 Oct 2026 18:27:17 +0530 Subject: [PATCH] feat(trail): count a learning's cost from where its work began --- src/cli.ts | 10 +-- src/notes/session-cost.ts | 126 ++++++++++++++++++++++++++++++++++---- test/trail-notes.test.ts | 47 +++++++++++++- 3 files changed, 167 insertions(+), 16 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index f555f680..ef401583 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -56,7 +56,7 @@ import { normalizePathPrefix } from "./util/paths.js"; import { latestSession, formatSessionStats, sessionInputRate } from "./claude/session-metrics.js"; import { listSessionIds, readSession, sessionDir } from "./claude/state.js"; import { costLabel, listNotes, noteName, notePlace, noteTargets, readingTokens, savedByNote, commitLines, PERSONAL_SAVED_LINE, PR_SPEEDUP, SECTIONS, tokensWorthSaying, sectionLead, shortDate, tokensLabel, writeNote, type Note, mentionedFiles } from "./notes/notes.js"; -import { currentSessionCost } from "./notes/session-cost.js"; +import { aboutTerms, currentSessionCost } from "./notes/session-cost.js"; import { beginPastRead, finishPastRead, type Finding } from "./notes/cold-start.js"; import { beginReviewRead, finishReviewRead, type ReviewFinding } from "./notes/pr-reviews.js"; import { currentBranch, noteAuthor, teammatesOn } from "./notes/git-facts.js"; @@ -1338,9 +1338,11 @@ async function runNoteCommand(repo: string, opts: NoteOptions): Promise { }; const minutes = wholeNumber(opts.minutes, "--minutes"); const tokens = wholeNumber(opts.tokens, "--tokens"); - // What the session spent, from the agent's own transcript when there is one; - // the flags win, since only the agent knows where its task began. - const spent = currentSessionCost(repo); + // What the session spent, from the agent's own transcript when there is one, + // counted from where the work the note names first came up; the flags win, + // since only the agent knows for sure where its task began. + const named = [...(opts.touches?.split(",").map((t) => t.trim()).filter(Boolean) ?? []), ...mentionedFiles(repo, body)]; + const spent = currentSessionCost(repo, undefined, Date.now(), aboutTerms(body, named)); const cost = minutes !== undefined || tokens !== undefined || spent ? { minutes: minutes ?? spent?.minutes, tokens: tokens ?? spent?.tokens } diff --git a/src/notes/session-cost.ts b/src/notes/session-cost.ts index 04820982..7845ccc3 100644 --- a/src/notes/session-cost.ts +++ b/src/notes/session-cost.ts @@ -9,7 +9,12 @@ * the agent passes `--minutes` / `--tokens`. * * Counted from the session's last finished `trail note` (or its start), so two - * notes from one long session don't both claim the same work. Tokens are the + * notes from one long session don't both claim the same work. Within that, + * from the turn where the work the note names first came up: the first prompt + * or tool call that mentions one of its files, symbols or commands. A long + * session that did other things first (a Slack search, a merge) doesn't charge + * them to this learning. When nothing matches, only the last 30 active + * minutes count, never the whole stretch. Tokens are the * ones the model actually processed fresh: input, cache writes and output. * Cache reads are left out, since they re-read the same context every turn and * would make an hour's work look like millions. Minutes are active minutes: a @@ -23,13 +28,17 @@ import { dirname, join, resolve } from "node:path"; export interface SessionCost { minutes: number; tokens: number; - /** When the counted stretch began, epoch ms. */ + /** When the stretch since the last note began, epoch ms. */ startedAt: number; + /** When the counted work began, epoch ms: the first mention of what the note is about. */ + countedFrom: number; /** Files the agent edited or wrote in that stretch, absolute, in order. */ edited: string[]; } const IDLE_MS = 10 * 60 * 1000; +/** With nothing in the transcript naming what the note is about, at most this much active time counts. */ +export const FALLBACK_ACTIVE_MS = 30 * 60 * 1000; /** A transcript untouched this long belongs to some earlier session. */ const ACTIVE_MS = 30 * 60 * 1000; @@ -92,12 +101,84 @@ const EDIT_TOOLS = new Set(["Edit", "MultiEdit", "Write", "NotebookEdit"]); export const NOTE_CALL = /(^|[|&;]\s*|\s)(npx\s+(-y\s+)?(@trailhq\/|@nanonets\/)?)?(trail|graft)\s+note\b/; +/** Words too common to say what a note is about: they'd match the first turn of any session. */ +const TOO_COMMON = new Set(["trail", "graft", "claude", "true", "false", "null", "main", "master", "node", "git", "npm", "test", "tests"]); + +/** + * What a note is about, as strings a transcript would show when that work + * began: the files it names (path and file name) and anything it puts in + * backticks, such as a symbol, a command or a setting. + */ +export function aboutTerms(body: string, files: string[] = []): string[] { + const terms = new Set(); + for (const f of files) { + const path = f.split("#")[0]!.trim(); + if (!path) continue; + terms.add(path); + const name = path.split("/").pop()!; + if (name.length >= 4) terms.add(name); + } + for (const m of body.matchAll(/`([^`\n]{3,80})`/g)) { + const t = m[1]!.trim().replace(/\(\)$/, ""); + if (t.length >= 4 && !TOO_COMMON.has(t.toLowerCase()) && specific(t)) terms.add(t); + } + return [...terms]; +} + +/** + * Whether a backticked term names something specific enough to date the work: + * a path or file (`a/b`, `x.ts`), or an identifier (`dropHtmlComments`, + * `repo_id`, `check-ignore`). A phrase or a plain word (`trail init`, + * `publish`) comes up in any session about the product, so it would date + * the work to the session's first turn. + */ +function specific(t: string): boolean { + if (/\s/.test(t)) return false; + return /[\/._]|[a-z][A-Z]|\w-\w/.test(t); +} + +/** A prompt the person typed, as opposed to a tool's result coming back. */ +function promptText(e: Entry): string | null { + if (e.type !== "user") return null; + const c = e.message?.content; + if (typeof c === "string") return c; + if (!Array.isArray(c) || c.some((b) => (b as { type?: string }).type === "tool_result")) return null; + const text = c.filter((b) => (b as { type?: string }).type === "text").map((b) => String((b as { text?: unknown }).text ?? "")).join("\n"); + return text || null; +} + +/** + * Index into `entries` where the work on a note began: the prompt of the turn + * whose prompt or tool calls first mention one of `terms`, never earlier than + * `from`. A `trail note` call doesn't count: the one being saved names them all. + */ +function firstMention(entries: Entry[], from: number, terms: string[]): number | null { + if (!terms.length) return null; + let turn = from; + for (let i = from; i < entries.length; i++) { + const e = entries[i]!; + const prompt = promptText(e); + if (prompt !== null) turn = i; + const said = [prompt ?? ""]; + for (const b of blocks(e)) { + if (b.type !== "tool_use") continue; + const input = JSON.stringify(b.input ?? {}); + if (NOTE_CALL.test(input)) continue; + said.push(input); + } + if (said.some((s) => s && terms.some((t) => s.includes(t)))) return turn; + } + return null; +} + /** * The cost of one transcript since its last finished note. Pure, for tests: * `text` is the JSONL, `now` closes the last stretch when the transcript ends - * mid-turn (it always does: the note call itself is in flight). + * mid-turn (it always does: the note call itself is in flight). `terms` are + * what the note is about (see `aboutTerms`); without them the whole stretch + * since the last note counts, as before. */ -export function costFromTranscript(text: string, now = Date.now()): SessionCost | null { +export function costFromTranscript(text: string, now = Date.now(), terms?: string[]): SessionCost | null { const entries: Entry[] = []; for (const line of text.split("\n")) { if (!line.trim()) continue; @@ -123,10 +204,19 @@ export function costFromTranscript(text: string, now = Date.now()): SessionCost }); const after = entries.slice(boundary + 1); - const counted = after.filter((e) => e.timestamp); - if (counted.length === 0) return null; - const times = counted.map((e) => Date.parse(e.timestamp!)).filter((t) => Number.isFinite(t)); - if (times.length === 0) return null; + const since = after.filter((e) => e.timestamp && Number.isFinite(Date.parse(e.timestamp))); + if (since.length === 0) return null; + const startedAt = Date.parse(since[0]!.timestamp!); + + // Where this note's work began within the stretch. + let counted = since; + if (terms) { + const at = firstMention(entries, boundary + 1, terms); + const stretch = new Set(since); + counted = at !== null ? entries.slice(at).filter((e) => stretch.has(e)) : lastActive(since, now, FALLBACK_ACTIVE_MS); + if (counted.length === 0) counted = since.slice(-1); + } + const times = counted.map((e) => Date.parse(e.timestamp!)); times.push(now); let activeMs = 0; for (let i = 1; i < times.length; i++) { @@ -157,15 +247,29 @@ export function costFromTranscript(text: string, now = Date.now()): SessionCost if (typeof file === "string" && file && !edited.includes(file)) edited.push(file); } } - return { minutes: Math.max(1, Math.round(activeMs / 60_000)), tokens, startedAt: times[0]!, edited }; + return { minutes: Math.max(1, Math.round(activeMs / 60_000)), tokens, startedAt, countedFrom: times[0]!, edited }; +} + +/** The entries in the last `budgetMs` of active time before `now`, oldest first. */ +function lastActive(entries: Entry[], now: number, budgetMs: number): Entry[] { + let used = 0; + let prev = now; + for (let i = entries.length - 1; i >= 0; i--) { + const t = Date.parse(entries[i]!.timestamp!); + const gap = prev - t; + if (gap > 0 && gap <= IDLE_MS) used += gap; + if (used > budgetMs) return entries.slice(i + 1); + prev = t; + } + return entries; } /** The running session's cost in `repo`, or null when there is no transcript to read. */ -export function currentSessionCost(repo: string, home?: string, now = Date.now()): SessionCost | null { +export function currentSessionCost(repo: string, home?: string, now = Date.now(), terms?: string[]): SessionCost | null { const path = currentTranscript(repo, home, now); if (!path) return null; try { - return costFromTranscript(readFileSync(path, "utf8"), now); + return costFromTranscript(readFileSync(path, "utf8"), now, terms); } catch { return null; } diff --git a/test/trail-notes.test.ts b/test/trail-notes.test.ts index 059c716a..8f353509 100644 --- a/test/trail-notes.test.ts +++ b/test/trail-notes.test.ts @@ -23,7 +23,7 @@ import { writeNote, type Note, } from "../src/notes/notes.js"; -import { costFromTranscript } from "../src/notes/session-cost.js"; +import { aboutTerms, costFromTranscript, FALLBACK_ACTIVE_MS } from "../src/notes/session-cost.js"; import { ensureRepoHome, repoPlace } from "../src/notes/home.js"; import { homeEnv, tmpRepo } from "./helpers.js"; @@ -198,6 +198,51 @@ test("the cost carries the files the session edited since its last note, for the ]); }); +test("a note names its work by its files and specific backticked terms, not by phrases", () => { + const body = "## Decided\n- `dropHtmlComments` in `src/notes/pr-reviews.ts` repeats until stable.\n- Run `trail init` and `publish`, keep `repo_id`, see `git check-ignore`.\n- `main` is fine."; + assert.deepEqual(aboutTerms(body, ["internal/ocr/bbox.go#NormalizeBox"]), [ + "internal/ocr/bbox.go", + "bbox.go", + "dropHtmlComments", + "src/notes/pr-reviews.ts", + "repo_id", + ]); +}); + +test("a note's cost starts at the turn where its work first came up, not at the session's start", () => { + const t = [ + // An hour on something else first. + line({ type: "user", timestamp: at(0), message: { content: "search slack for the release" } }), + line({ type: "assistant", timestamp: at(5), message: { id: "s", usage: usage(0, 50_000, 1000), content: [{ type: "tool_use", id: "x", name: "Bash", input: { command: "slack search release" } }] } }), + line({ type: "assistant", timestamp: at(9), message: { id: "s2", usage: usage(0, 40_000, 1000), content: [] } }), + // The turn the learning is about: its prompt counts, then the tool call that names the file. + line({ type: "user", timestamp: at(10), message: { content: "why is the box rotated?" } }), + line({ type: "assistant", timestamp: at(11), message: { id: "w", usage: usage(0, 3000, 500), content: [{ type: "tool_use", id: "r", name: "Read", input: { file_path: "/repo/internal/ocr/bbox.go" } }] } }), + line({ type: "assistant", timestamp: at(14), message: { id: "w2", usage: usage(0, 2000, 500), content: [] } }), + // The note being saved names everything: it never counts as where the work began. + line({ type: "assistant", timestamp: at(15), message: { id: "n", usage: usage(0, 0, 100), content: [{ type: "tool_use", id: "n1", name: "Bash", input: { command: "trail note --title x <<'EOF'\nslack bbox.go\nEOF" } }] } }), + ].join("\n"); + const now = Date.parse(at(15)); + const whole = costFromTranscript(t, now)!; + const scoped = costFromTranscript(t, now, aboutTerms("see `internal/ocr/bbox.go`"))!; + assert.equal(whole.tokens, 51_000 + 41_000 + 3500 + 2500 + 100); + assert.equal(scoped.tokens, 3500 + 2500 + 100); + assert.equal(scoped.minutes, 5); + assert.equal(scoped.countedFrom, Date.parse(at(10))); + assert.equal(scoped.startedAt, Date.parse(at(0)), "the stretch since the last note still starts at the top"); +}); + +test("when nothing in the transcript names the work, only the last 30 active minutes count", () => { + const t = Array.from({ length: 13 }, (_, i) => + line({ type: "assistant", timestamp: at(i * 5), message: { id: `m${i}`, usage: usage(0, 1000, 0), content: [] } }), + ).join("\n"); + const now = Date.parse(at(60)); + assert.equal(costFromTranscript(t, now)!.minutes, 60); + const capped = costFromTranscript(t, now, ["nowhere/in/it.go"])!; + assert.equal(capped.minutes, FALLBACK_ACTIVE_MS / 60_000); + assert.equal(capped.tokens, 7000, "the entries at 30, 35, …, 60 minutes"); +}); + test("no transcript entries, no cost", () => { assert.equal(costFromTranscript("", Date.now()), null); assert.equal(costFromTranscript("not json\n", Date.now()), null);