From 3c6ea528ebb0b70422bc10a7ec66e402801d26f4 Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 05:01:15 -0500 Subject: [PATCH 01/92] test(guards): run the npm audit guard without the allow-scripts value a parent npm exports `npm run` and `npx` hand user config to their children as npm_config_* variables, and npm 11.17 refuses allow-scripts arriving that way with EALLOWSCRIPTS, so on a machine that sets allow-scripts in ~/.npmrc the nested `npm audit --json` returned an error object and the guard threw reading `metadata` off it. The audit now runs with that one inherited variable dropped, which the child reads back from ~/.npmrc itself, and an error reply skips the check the way an unreachable registry already did instead of crashing. --- tests/guards/dependency_advisory_disclosure.test.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/guards/dependency_advisory_disclosure.test.ts b/tests/guards/dependency_advisory_disclosure.test.ts index a70483983..278e53295 100644 --- a/tests/guards/dependency_advisory_disclosure.test.ts +++ b/tests/guards/dependency_advisory_disclosure.test.ts @@ -157,17 +157,24 @@ describe('dependency advisory disclosure', () => { /** The "this repository" row is the one claim in the table no other test here re-derives: the rest check package identity and lock-file counts, which is a proxy for what `npm audit` reports, not the audit result itself. A real advisory landing in a dev-only transitive dependency (exactly what happened with adm-zip and hono) can make the row's "clean" claim false without moving anything those other checks look at. `auditReport` is computed once at module load so every test in this file pays for at most one `npm audit` call, not one per assertion. */ interface AuditReport { metadata: { vulnerabilities: { total: number } } } +// An error reply (offline, a registry failure, a rejected config) is JSON on stdout too, shaped `{ "error": { ... } }`. It carries no audit result, so it skips the check the way an unreachable registry does instead of reading a total off a report that is not there. +const asAuditReport = (text: string): AuditReport | null => { + const parsed = JSON.parse(text) as { metadata?: { vulnerabilities?: { total?: unknown } } } + return typeof parsed.metadata?.vulnerabilities?.total === 'number' ? parsed as AuditReport : null +} +// A parent `npm run` or `npx` hands its user config to children as npm_config_* variables, and npm 11.17 refuses allow-scripts arriving that way (EALLOWSCRIPTS, "not allowed in project-scoped installs") although it accepts the same key from the ~/.npmrc the child reads for itself. Dropping the inherited copy is what lets the audit run under `npm test` on a machine that sets allow-scripts in ~/.npmrc. +const auditEnv = Object.fromEntries(Object.entries(process.env).filter(([key]) => key.toLowerCase().replace(/-/g, '_') !== 'npm_config_allow_scripts')) let auditReport: AuditReport | null = null try { // execSync, not execFileSync: `npm` is a `.cmd` shim on Windows, and execFileSync fails to spawn one directly (EINVAL) without a shell. The command is a fixed literal with no interpolated input, so a shell string carries no injection risk here. - const out = execSync('npm audit --json', { cwd: repoRoot, encoding: 'utf8', maxBuffer: 16e6 }) - auditReport = JSON.parse(out) as AuditReport + const out = execSync('npm audit --json', { cwd: repoRoot, encoding: 'utf8', maxBuffer: 16e6, env: auditEnv }) + auditReport = asAuditReport(out) } catch (err) { // `npm audit` exits non-zero the moment it finds anything, which execFileSync treats as a thrown error -- but the JSON report is still on stdout in that case, so try that before giving up. const stdout = (err as { stdout?: Buffer | string } | null)?.stdout if (stdout !== undefined) { try { - auditReport = JSON.parse(stdout.toString()) as AuditReport + auditReport = asAuditReport(stdout.toString()) } catch { auditReport = null } From c3105200627ff4312f62491f8bed18dbfd09f429 Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 05:06:16 -0500 Subject: [PATCH 02/92] test(guards): drop the wall-clock bounds from the markup guard's critical-path index tests The markup linearity guard failed under machine load and passed alone. What failed were the two critical-path stopwatches: the built CLI's index of the pathological component was held under 10 s and the worker drain under 5 s. With other worktrees running tests the CLI step took 11.9 s and 18.7 s, and here it took 3.5 s on a quiet machine and 7 to 52 s with the file running four to six times in parallel beside CPU burners. Those bounds timed process startup and the machine's load, not the scan. The critical-path tests keep their wiring assertions, that the default worker drain and the built CLI both index the file and resolve realAttr, and drop the stopwatches. The extractor both paths run on a .cmp file is held linear on that exact shape by the .cmp growth case. --- tests/guards/markup_attribute_scan_stays_linear.test.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/guards/markup_attribute_scan_stays_linear.test.ts b/tests/guards/markup_attribute_scan_stays_linear.test.ts index 581569e73..84768342f 100644 --- a/tests/guards/markup_attribute_scan_stays_linear.test.ts +++ b/tests/guards/markup_attribute_scan_stays_linear.test.ts @@ -156,6 +156,7 @@ function tempDir(prefix: string): string { describe('critical path: a component with the pathological shape still indexes', () => { // Provenance: HAND-DERIVED. A real attribute declaration followed by a tag that never closes, repeated; realAttr is the one name present in the file. + // These pin the wiring, not the speed. The extractor both paths run on a .cmp file, extractSalesforceMarkup, is held linear on this exact shape by the '.cmp: attribute tag starts, name quote open, no >' case above, while a stopwatch around a drain or a spawned CLI reads the machine's load as much as the scan: the CLI index of this file took 3.5 s on a quiet machine and 7 to 52 s with the guards running in parallel. const body = (): string => '\n \n\n' + ' Date: Tue, 13 Oct 2026 05:22:19 -0500 Subject: [PATCH 05/92] fix(indexer): keep Playwright snapshot directories out of the index The Playwright CLI and the Playwright MCP server save every page's accessibility tree as page-.yml under .playwright-cli/ or .playwright-mcp/. The non-git walk already skips hidden directories, but index lists a git repo through git ls-files, so a repo that committed its snapshots indexed and embedded them; in this repo 8 of the 20 hits for search "updatedToolOutput WebSearch" came from two such files. isToolOutputPath, in skip_scope.ts, is now checked beside isParseSkipEligible at every site that decides what reaches the index: cmdIndex, the worker's indexer and drain, and the read commands' indexFileSyncPinned, each of which purges a file it refuses. It is kept out of parser.ts on purpose: every byte of parser.ts is in PARSER_FINGERPRINT, and moving that stamp would make every user reparse every language on upgrade. Since no fingerprint moves, no upgrade reparse clears snapshot rows an existing index holds, so reconcile re-queues an indexed snapshot as drift and the worker's drain purges it; an unindexed one is not reported as added. tests/tool_output_dirs_not_indexed.test.ts covers cmdIndex over a real git repo, the worker's default drain, reconcile purging a row an older index holds, and the built bundle's index plus symbol. --- src/cli.ts | 4 +- src/read_commands.ts | 9 +- src/reconcile.ts | 7 + src/skip_scope.ts | 8 + src/worker.ts | 5 +- tests/tool_output_dirs_not_indexed.test.ts | 169 +++++++++++++++++++++ 6 files changed, 197 insertions(+), 5 deletions(-) create mode 100644 tests/tool_output_dirs_not_indexed.test.ts diff --git a/src/cli.ts b/src/cli.ts index 033d661f8..8f08fa769 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -16,7 +16,7 @@ import { getTrackedFiles } from './repomap.js' import { collectWalkIndexFiles, MAX_FILES_SCANNED_FORCED } from './walk_index.js' import { dataDir, ENV_KEYS, globalDbPath, VERSION } from './constants.js' import { assetEmbedSha, indexFileSync, indexFileEmbeddings, indexedPathSpellingIsStale, isEmbedFresh, isParseSkipEligible, loadRegexExtractors, maxChunksEmbedSha } from './parser.js' -import { withDeclaredSkipScopeRoot } from './skip_scope.js' +import { isToolOutputPath, withDeclaredSkipScopeRoot } from './skip_scope.js' import { deleteFileEmbeddings, embeddingsDepsAvailable, ensureEmbeddingProvenance } from './embeddings.js' import { pruneUnembeddableChunks } from './embed_backfill.js' import { foregroundDownloadDeferred, offlineEmbedNotice, WARM_COMMAND } from './embed_preflight.js' @@ -345,7 +345,7 @@ export async function cmdIndex( // PDF/DOCX/PPTX/XLSX have no Language entry (no code symbols) so they report 'unknown', but they must still reach indexFileEmbeddings below for extracted-text embedding. The language is read from the file's head, as the walk that listed it did: a path-only check skipped every `.p`, `.w`, `.m` and `.t` a content sniff admits, so it was never indexed or counted. if (detectLanguageOfFile(key) === 'unknown' && !isEmbeddableDocument(key)) continue // indexing.skip_dirs / large_file_skip_kb: filter here, before the sha/entry work below. Without this pre-filter, indexFileSync's internal purge would run and then the unconditional indexFileEmbeddings call below would immediately re-embed a file meant to be fully excluded (origin's indexFileEmbeddings has no skip_dirs/size-cap branch). - if (isParseSkipEligible(key, ixCfg, walkRootKey)) { + if (isParseSkipEligible(key, ixCfg, walkRootKey) || isToolOutputPath(key, walkRootKey)) { removeFileFromIndex(getDb(dbPath), key) continue } diff --git a/src/read_commands.ts b/src/read_commands.ts index d5d585d52..a37c68c0e 100644 --- a/src/read_commands.ts +++ b/src/read_commands.ts @@ -15,7 +15,9 @@ import { enqueueDirtyPathSafe } from './hooks_index.js' import { dataDir, globalDbPath } from './constants.js' import { recordKnownRootThrottled } from './known_roots.js' import { LARGE_SYMBOL_LINE_THRESHOLD } from './hints/file_type_handler.js' -import { isReadOnlyDb } from './db.js' +import { getDb, isReadOnlyDb } from './db.js' +import { removeFileFromIndex } from './index_prune.js' +import { isToolOutputPath } from './skip_scope.js' import { fileIsAbsent, fingerprintFile } from './fingerprint.js' import { decodeSource, runGit, PER_FILE_COUNTERFACTUAL_CEILING, foldCaseForContainment, countNoun, requirePositiveStrictInt } from './util.js' import { renderContextWindow } from './util_context.js' @@ -204,6 +206,11 @@ export function indexFileSyncPinned(resolvedPath: string, dbPath: string): void if (isReadOnlyDb(dbPath)) return // A heal triggered by a read can be the only thing that ever indexes a project, so register its root here too rather than relying on a later edit or bulk walk. See recordKnownRootThrottled. recordKnownRootThrottled(resolvedPath, dataDir(), dbPath) + // A tool's snapshot never enters the index; one an older index still holds is purged, as cmdIndex and the worker purge it. + if (isToolOutputPath(resolvedPath)) { + removeFileFromIndex(getDb(dbPath), resolvedPath) + return + } const pinned = activePins?.get(pinKey(path.resolve(resolvedPath))) if (pinned === undefined) { indexFileSync(resolvedPath, dbPath) diff --git a/src/reconcile.ts b/src/reconcile.ts index 5aed4d1d2..c4fdf9368 100644 --- a/src/reconcile.ts +++ b/src/reconcile.ts @@ -11,6 +11,7 @@ import { indexedPathSpellingIsStale } from './parser.js' import { parserFingerprintForLanguage } from './parser_stamp.js' import { getDisplayRoot } from './project.js' import { getTrackedFiles } from './repomap.js' +import { isToolOutputPath } from './skip_scope.js' import { projectScopeClause } from './sql_path.js' import { countNoun, foldPath } from './util.js' @@ -216,6 +217,12 @@ export function reconcileProject(opts: ReconcileOptions = {}): ReconcileResult { seenOnDisk.add(folded) const entry = indexed.get(folded) + // A tool's snapshot is never indexed, so a new one is no drift; one an index from before that rule still holds is, and the drain purges it. + if (isToolOutputPath(file, projectRoot)) { + if (entry !== undefined) changed.push(file) + continue + } + if (entry === undefined) { added.push(file) continue diff --git a/src/skip_scope.ts b/src/skip_scope.ts index 2d7c73992..7f133b5d1 100644 --- a/src/skip_scope.ts +++ b/src/skip_scope.ts @@ -40,6 +40,14 @@ function outermostProjectRoot(filePath: string): string | undefined { } } +/** Directories a tool fills with its own output snapshots: the Playwright CLI and the Playwright MCP server save every page's accessibility tree there as `page-.yml`. The walk already skips them as hidden directories, but `index` lists a git repo through `git ls-files`, so a repo that commits them indexed and embedded every snapshot, and in this repo 8 of the 20 `search "updatedToolOutput WebSearch"` hits came from two of those files. No source lives in them, so they are skipped always rather than through `indexing.skip_dirs`. */ +const TOOL_OUTPUT_DIRS: ReadonlySet = new Set(['.playwright-cli', '.playwright-mcp']) + +/** True when a directory segment of `filePath` below the project root is one of {@link TOOL_OUTPUT_DIRS}. Checked beside parser.ts's isParseSkipEligible at every site that decides what reaches the index (cmdIndex, the worker's indexer and drain, the read commands' indexFileSyncPinned) and by reconcile, which re-queues a snapshot an older index still holds so the drain purges it. It lives here rather than in parser.ts because every byte of parser.ts is in PARSER_FINGERPRINT, and moving that stamp makes every user reparse every language. */ +export function isToolOutputPath(filePath: string, projectRoot?: string): boolean { + return hasSkipSegmentBelowRoot(filePath, projectRoot, (seg) => TOOL_OUTPUT_DIRS.has(foldPath(seg))) +} + /** True when a directory segment of `filePath` (the file name excluded) that lies below the project root satisfies `isSkipSegment`. `projectRoot` is the root the caller already knows (a walk root, a resolved project root); when omitted it is derived from project markers, and only once some segment matches at all, so the marker probing is not paid on the common path. A path outside the given root, or with no derivable root, is tested whole: the conservative answer, identical to the old behavior. */ export function hasSkipSegmentBelowRoot(filePath: string, projectRoot: string | undefined, isSkipSegment: (segment: string) => boolean): boolean { const dirSegments = pathSegments(filePath).slice(0, -1) diff --git a/src/worker.ts b/src/worker.ts index c1038e6a3..fb95e18c7 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -17,6 +17,7 @@ import { configProjectRootFor, loadConfig, withConfigProjectRoot } from './confi import { getDb } from './db.js' import { pathEqClause } from './sql_path.js' import { removeFileFromIndex, pruneDeletedFiles, sweepKnownRoots } from './index_prune.js' +import { isToolOutputPath } from './skip_scope.js' import { recordKnownRootThrottled, sweepExpiredKnownRootMarkers } from './known_roots.js' import { reclaimIndex } from './index_reclaim.js' import { applyIndexingPriority } from './process_priority.js' @@ -330,7 +331,7 @@ export function makeIndexer(dbPath: string): (absPath: string, sha: string) => u recordKnownRootThrottled(absPath, dir, dbPath) // Skip-eligibility must be checked UNCONDITIONALLY, before the parseUnchanged sha-gate: a file that becomes skip-eligible purely from a config change (same sha) would otherwise never reach indexFileSync's purge at all, leaving symbols/refs/files rows stale forever. Guard on ixCfgForSkip !== undefined for tests that mock loadConfig with a partial { worker: {...} } shape. const ixCfgForSkip = loadConfig().indexing - if (ixCfgForSkip !== undefined && isParseSkipEligible(absPath, ixCfgForSkip)) { + if ((ixCfgForSkip !== undefined && isParseSkipEligible(absPath, ixCfgForSkip)) || isToolOutputPath(absPath)) { // Purging stale rows is real work: unlike the `false` sha-gate skip below, this must count as "indexed" in processDirtyBatch's tally. removeFileFromIndex(getDb(dbPath), absPath) return true @@ -439,7 +440,7 @@ export function processDirtyBatch( continue } // Decided from the path and a stat before fingerprintFile reads the whole file, as cmdIndex does: a log past large_file_skip_kb was otherwise loaded whole on every drain only to be skipped, and one over 2 GiB failed that read and was requeued as transient until its retries ran out, its stale rows never purged. Counted as indexed for the same reason as makeIndexer's identical branch, which still covers its direct callers. - if (ixCfg !== undefined && isParseSkipEligible(p, ixCfg)) { + if ((ixCfg !== undefined && isParseSkipEligible(p, ixCfg)) || isToolOutputPath(p)) { recordKnownRootThrottled(p, dir, path.join(dir, 'global.db')) if (pruneDirtyPath(p, remove, dir, requeue)) indexed += 1 continue diff --git a/tests/tool_output_dirs_not_indexed.test.ts b/tests/tool_output_dirs_not_indexed.test.ts new file mode 100644 index 000000000..360799beb --- /dev/null +++ b/tests/tool_output_dirs_not_indexed.test.ts @@ -0,0 +1,169 @@ +/** Regression: the Playwright CLI and the Playwright MCP server write every page snapshot to `.playwright-cli/` or `.playwright-mcp/` as `page-.yml`. The non-git walk skips hidden directories, but `index` lists a git repo through `git ls-files`, so a repo that committed its snapshots indexed them, and `search "updatedToolOutput WebSearch"` in this repo returned 8 of its 20 hits from two such files. Both tests drive the real default paths: `cmdIndex` over a real git repo, and the worker's own drain with no injected indexer. */ +import { execFileSync, spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { BUNDLE, tgIsolatedEnv } from './helpers/bundle.js' +import { indexableDir, tempConfigPath } from './helpers/temp-config.js' + +vi.mock('../src/constants.js', async (importOriginal) => { + const original = await importOriginal>() + return { ...original, configPath: () => _testConfigPath } +}) + +const _testConfigPath = tempConfigPath('tg-tool-output-dirs.toml') + +import { cmdIndex } from '../src/cli.js' +import { invalidateConfigCache } from '../src/config.js' +import { closeAllDbs, getDb } from '../src/db.js' +import { querySymbols } from '../src/index_reader.js' +import { dataDir } from '../src/constants.js' +import { indexFileSync } from '../src/parser.js' +import { reconcileProject } from '../src/reconcile.js' +import { isToolOutputPath } from '../src/skip_scope.js' +import { normalizePath } from '../src/paths.js' +import { drainOnce, pendingEmbeddings } from '../src/worker.js' + +// Provenance: CAPTURE. The head of a real `.playwright-cli/page-*.yml` tracked in this repository, cut to a few nodes. +const SNAPSHOT = [ + '- main [ref=e2]:', + ' - generic [ref=e3]:', + ' - link "Open the Token-Goat README" [ref=e4] [cursor=pointer]:', + ' - /url: ../README.md', + ' - paragraph [ref=e6]: Approval demo / VS Code Copilot Chat', + '', +].join('\n') + +let TMP: string + +function git(...args: string[]): void { + execFileSync('git', ['-c', 'core.hooksPath=/dev/null', ...args], { cwd: TMP, stdio: 'ignore' }) +} + +function seed(): { app: string; snapshots: string[] } { + const app = path.join(TMP, 'app.ts') + fs.writeFileSync(app, 'export function zqToolOutputApp(): number {\n return 1\n}\n') + const snapshots = ['.playwright-cli', '.playwright-mcp'].map((dir) => { + fs.mkdirSync(path.join(TMP, dir), { recursive: true }) + const file = path.join(TMP, dir, 'page-2026-08-06T20-03-09-007Z.yml') + fs.writeFileSync(file, SNAPSHOT) + return file + }) + return { app, snapshots } +} + +function snapshotRows(dbPath: string): number { + const db = getDb(dbPath) + const files = db.prepare("SELECT count(*) AS n FROM files WHERE path LIKE '%.playwright-%'").get() as { n: number } + const chunks = db.prepare("SELECT count(*) AS n FROM chunks WHERE file_path LIKE '%.playwright-%'").get() as { n: number } + return files.n + chunks.n +} + +beforeEach(() => { + TMP = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), 'tg-tool-output-dirs-'))) + fs.writeFileSync(_testConfigPath, '') + invalidateConfigCache() +}) + +afterEach(() => { + vi.restoreAllMocks() + closeAllDbs() + invalidateConfigCache() + fs.rmSync(TMP, { recursive: true, force: true }) +}) + +describe('isToolOutputPath', () => { + it('matches a snapshot below the project root and nothing outside the two directories', () => { + expect(isToolOutputPath('/repo/.playwright-cli/page-1.yml', '/repo')).toBe(true) + expect(isToolOutputPath('/repo/web/.playwright-mcp/page-1.yml', '/repo')).toBe(true) + expect(isToolOutputPath('/repo/playwright-cli/page-1.yml', '/repo')).toBe(false) + expect(isToolOutputPath('/repo/.playwright-cli.yml', '/repo')).toBe(false) + expect(isToolOutputPath('/home/.playwright-cli/repo/app.ts', '/home/.playwright-cli/repo')).toBe(false) + }) +}) + +describe('tool snapshot directories on the real indexing paths', () => { + it('index leaves committed snapshots out of a git repo and still indexes its source', async () => { + git('init', '-q', '.') + git('config', 'user.email', 'test@example.com') + git('config', 'user.name', 'test') + seed() + git('add', '-A') + git('commit', '-qm', 'seed') + const dbPath = path.join(TMP, 'index.db') + + vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + await cmdIndex(TMP, { dbPath }) + + expect(snapshotRows(dbPath)).toBe(0) + expect(querySymbols({ name: 'zqToolOutputApp', limit: 10 }, dbPath).length).toBe(1) + }) + + it('the worker drain leaves a snapshot out when a hook enqueues it', async () => { + const { app, snapshots } = seed() + const dataDir = path.join(TMP, 'data') + const queue = path.join(dataDir, 'queue', 'dirty.txt') + fs.mkdirSync(path.dirname(queue), { recursive: true }) + fs.writeFileSync(queue, `${[app, ...snapshots].map((p) => normalizePath(p)).join('\n')}\n`) + + drainOnce(dataDir) + await pendingEmbeddings() + + const dbPath = path.join(dataDir, 'global.db') + expect(snapshotRows(dbPath)).toBe(0) + expect(querySymbols({ name: 'zqToolOutputApp', limit: 10 }, dbPath).length).toBe(1) + }) + + it('the built bundle indexes the source and leaves a committed snapshot out', () => { + git('init', '-q', '.') + seed() + // A top-level YAML key is a symbol, so a snapshot that reached the index would answer `symbol` for it. + fs.writeFileSync(path.join(TMP, '.playwright-cli', 'page-2026-08-06T20-03-43-226Z.yml'), 'zqSnapshotKey: page\n') + git('add', '-A') + git('-c', 'user.email=t@t.t', '-c', 'user.name=t', 'commit', '-qm', 'seed') + const env = tgIsolatedEnv(path.join(TMP, 'home'), { TOKEN_GOAT_HOME: path.join(TMP, 'data'), TOKEN_GOAT_EMBEDDINGS_ENABLED: 'false' }) + const run = (args: string[]): { status: number | null; stdout: string; stderr: string } => { + const r = spawnSync(process.execPath, [BUNDLE, ...args], { cwd: TMP, env, encoding: 'utf8', timeout: 120000 }) + return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' } + } + + const idx = run(['index', '.']) + expect(idx.status, idx.stderr).toBe(0) + expect(run(['symbol', 'zqToolOutputApp']).stdout).toContain('app.ts') + const snap = run(['symbol', 'zqSnapshotKey']) + expect(snap.status).toBe(1) + expect(snap.stdout).not.toContain('.playwright-cli') + }, 180000) + + // The rule is kept out of parser.ts so upgrading moves no parser fingerprint, which also means no upgrade reparse sweeps the old snapshot rows away: reconcile has to find them itself. + it('reconcile re-queues a snapshot an older index still holds, and the drain purges it', async () => { + const repo = indexableDir() + fs.mkdirSync(path.join(repo, '.playwright-cli')) + const snapshot = path.join(repo, '.playwright-cli', 'page-2026-08-06T20-03-09-007Z.yml') + const fresh = path.join(repo, '.playwright-cli', 'page-2026-08-06T20-03-43-226Z.yml') + fs.writeFileSync(snapshot, 'zqStaleSnapshotKey: page\n') + fs.writeFileSync(fresh, 'zqFreshSnapshotKey: page\n') + fs.writeFileSync(path.join(repo, 'app.ts'), 'export function zqReconcileApp(): number {\n return 1\n}\n') + execFileSync('git', ['-c', 'core.hooksPath=/dev/null', 'init', '-q', '.'], { cwd: repo, stdio: 'ignore' }) + execFileSync('git', ['-c', 'core.hooksPath=/dev/null', 'add', '-A'], { cwd: repo, stdio: 'ignore' }) + const dbPath = path.join(dataDir(), 'global.db') + // An index written before the rule existed: indexFileSync itself has no tool-output check, so this is exactly the row such an index holds. + indexFileSync(normalizePath(snapshot), dbPath) + indexFileSync(normalizePath(path.join(repo, 'app.ts')), dbPath) + expect(querySymbols({ name: 'zqStaleSnapshotKey', limit: 10 }, dbPath).length).toBe(1) + + const result = reconcileProject({ cwd: repo, dbPath }) + expect(result.changed.map((f) => normalizePath(f))).toContain(normalizePath(snapshot)) + // A snapshot that never reached the index is no drift. + expect(result.added.map((f) => normalizePath(f))).not.toContain(normalizePath(fresh)) + + drainOnce(dataDir()) + await pendingEmbeddings() + expect(querySymbols({ name: 'zqStaleSnapshotKey', limit: 10 }, dbPath).length).toBe(0) + expect(querySymbols({ name: 'zqFreshSnapshotKey', limit: 10 }, dbPath).length).toBe(0) + expect(querySymbols({ name: 'zqReconcileApp', limit: 10 }, dbPath).length).toBe(1) + }) +}) From 53520e3e8941b3e870bd806e5086c43bdf6a2fe0 Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 05:27:25 -0500 Subject: [PATCH 06/92] fix(pdf): say a scanned PDF has no text layer instead of printing an empty fence pdf-extract on a scanned or image-only PDF printed an empty untrusted-content fence and exited 0, with no hint; pdf-meta already reported "Text layer: no" for the same file, but pdf-extract never looked. An agent spent three calls doubting the command before it rasterized the pages itself. extractPdfText trims every page before joining them, so an empty result means no requested page holds a visible character, the same test pdf-meta applies. Then pdf-extract prints no fence, writes a note on stdout naming the pages with a pdftoppm and a Ghostscript command that rasterize them for Read or image-text, and exits 1 with the reason on stderr, so a || fallback fires and 2>/dev/null still shows the route. The pdf-extract matrix case runs the built bundle on a generated image-only page; docs/cli.md describes the new exit. --- docs/cli.md | 2 +- src/cli_office.ts | 26 +++++++++++++++++++++++++- tests/helpers/matrix_cases.ts | 19 +++++++++++++++++++ tests/pdf_extract.test.ts | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index a71baa8e8..408145596 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -125,7 +125,7 @@ token-goat pdf-extract manual.pdf --pages 12-15 --layout --head 120 | `token-goat resume ` | Emit a single post-compact recovery packet — top skills, edited and most-read files, last two Bash outputs, recent web fetches, and `git diff --stat`, capped at ~2000 tokens. After a compaction, files the compaction summary did not mention are marked `(not in the compaction summary)`, and any not already listed get their own section. Replaces 5-10 round-trips. | | `token-goat config list / get / set / validate` | Inspect or edit `config.toml` from the CLI. `validate` reports unknown keys with did-you-mean suggestions, plus any value in `config.toml`, a project file or the environment that validation rejected or clamped, with the reason and the value in effect instead (`overflow_guard.max_tokens = 500` is reported, since the floor is 1000). A project-root `.token-goat.toml` layers on top of the global config, overriding hint thresholds, indexing settings, etc. for that project only. It may not set the security sections `injection`, `webfetch`, `gdrive`, or `mcp`, nor `indexing.cross_project_symbols`: that file arrives with the repository, so a cloned project could otherwise switch off prompt-injection fencing or empty the fetch allow list for anyone who opened it. Those settings come from the global config or the environment only, and a project file that tries to set one is ignored with a message naming what was dropped. `config get`/`list`/`set` report which layer a value actually resolved from — the project file, an environment variable, or the global config — and where that layer's value was clamped or rejected they say so, naming what was asked for and what is in effect instead. | | `token-goat config-get ` | Look up one key from a config-shaped file (TOML/INI `key = value`, or YAML) without reading the whole thing. On a `.md` file, a leading `---`-fenced YAML frontmatter block (Jekyll/Hugo/SKILL.md style) is checked first and takes precedence over the TOML/INI fallback; a `.md` file with no frontmatter, or an unclosed fence, falls through to the normal lookup unchanged. | -| `token-goat pdf-extract ` | Extract plain text from a PDF instead of a raw Read. `--pages ` narrows to a page range (e.g. `1-5` or `3`); a range that starts past the last page is refused, and one that ends past it stops at the last page with a note on stderr; `--head`/`--tail`/`--grep`/`--max-matches`/`--section` slice the extracted text the same way `bash-output`/`web-output` do. `--layout` heuristically reconstructs column-aware reading order from text-item coordinates instead of raw content-stream order (imperfect on rotated/overlapping text). | +| `token-goat pdf-extract ` | Extract plain text from a PDF instead of a raw Read. `--pages ` narrows to a page range (e.g. `1-5` or `3`); a range that starts past the last page is refused, and one that ends past it stops at the last page with a note on stderr; `--head`/`--tail`/`--grep`/`--max-matches`/`--section` slice the extracted text the same way `bash-output`/`web-output` do. `--layout` heuristically reconstructs column-aware reading order from text-item coordinates instead of raw content-stream order (imperfect on rotated/overlapping text). When none of the requested pages has a text layer (a scanned or image-only PDF), it prints no empty fence: it exits 1 with a note on stdout naming the pages and a `pdftoppm`/Ghostscript command to rasterize them, so the page images can be Read or passed to `image-text`. | | `token-goat pdf-locate ` | Find which pages of a PDF match a regex, with a snippet per match, so you can `pdf-extract --pages` only those pages instead of pulling the whole document. `-i`/`--ignore-case` for case-insensitive matching; `--max-matches ` caps how many page matches to collect (default 50); `--context ` sets the snippet length around each match (default 80); `--pages ` narrows the scan to a page range, with the same past-the-end note on stderr as `pdf-extract`, which names only the pages searched when `--max-matches` ended the scan first; `-j`/`--json` emits `{ file, pattern, matchCount, pages, matches }`. | | `token-goat pdf-outline ` | List a PDF's bookmark/outline tree with page numbers instead of a raw Read. | | `token-goat pdf-meta [--json]` | Page count, title/author, and whether a PDF has an extractable text layer (so you know before extracting whether it's scanned/image-only). `--json` emits `{ pageCount, title, author, hasTextLayer }` — `hasTextLayer` as a real boolean rather than a prose sentence, and an absent title/author as `null` rather than the literal `(none)`. | diff --git a/src/cli_office.ts b/src/cli_office.ts index 78fc63831..23f0bb9cd 100644 --- a/src/cli_office.ts +++ b/src/cli_office.ts @@ -5,7 +5,9 @@ import { _applyFiltersAndPrint } from './cli_cached_output.js' import { formatCsvTable, parseWhereSpecs } from './csv_query.js' import { docxOutline, docxTables, docxText, formatDocxTables } from './docx_extract.js' import { emitErr } from './emit.js' +import { parsePageRange } from './pdf_extract.js' import { UNTRUSTED_FILE_TAG } from './injection_scan.js' +import { canQuoteArg, echoedValue, quotedArg } from './hint_suggestion_guard.js' import { displaySafeJson, displaySafePath, displaySafeText } from './paths.js' import { pptxNotesText, pptxOutline, pptxSlideText, pptxTextGrep } from './pptx_extract.js' import { @@ -28,7 +30,7 @@ import { sliceTranscript, } from './transcript_extract.js' import { fenceFileFieldIfMatched } from './untrusted_fence.js' -import { cappedSourceBytesSaved, countNoun, redactUrlQuery } from './util.js' +import { cappedSourceBytesSaved, countNoun, isWindows, redactUrlQuery } from './util.js' import { extractVideoChapters } from './video_chapters.js' import { formatXlsxColumns, @@ -60,11 +62,33 @@ export function recordXlsxStat(kind: string, file: string, emitted: string): voi recordStat(kind, bytesSaved, savedTokensFromBytes(bytesSaved)) } +/** What pdf-extract prints instead of an empty fence when no requested page has a text layer: which pages, why, and two rasterizers whose page images the model can Read or `image-text` can OCR. Ghostscript's console binary is `gswin64c` on Windows and `gs` elsewhere. A path no quote mark holds in both shells gets the explanation without the commands. */ +function noTextLayerNote(file: string, first: number, last: number, pageCount: number): string { + const pages = first === last ? `page ${first}` : `pages ${first}-${last}` + const lines = [`No text layer on ${pages} of ${countNoun(pageCount, 'page')} in ${echoedValue(file)}: it looks scanned or image-only, and pdf-extract reads only a PDF's text layer, it does not OCR.`] + if (!canQuoteArg(file)) return lines[0]! + const pdf = quotedArg(file) + lines.push( + 'Rasterize the pages, then Read each page image, or run token-goat image-text on it for its text:', + ` pdftoppm -r 150 -png -f ${first} -l ${last} ${pdf} page`, + ` ${isWindows() ? 'gswin64c' : 'gs'} -q -dSAFER -dBATCH -dNOPAUSE -sDEVICE=png16m -r150 -dFirstPage=${first} -dLastPage=${last} -sOutputFile=page-%d.png ${pdf}`, + ) + return lines.join('\n') +} + export async function cmdPdfExtract( file: string, opts: { pages?: string; head?: string; tail?: string; grep?: string; section?: string; maxMatches?: string; layout?: boolean; full?: boolean; lines?: string }, ): Promise { const result = await runPdfExtractText(file, opts.pages, opts.layout === true) + // Every page's text is trimmed before the join, so an empty result means no requested page holds a visible character: the test pdf-meta's hasTextLayer applies, so one real glyph still counts as text. + if (result.pagesExtracted > 0 && result.text.trim() === '') { + if (result.pagesNote !== undefined) emitErr(result.pagesNote) + const range = parsePageRange(opts.pages, result.pageCount) ?? { start: 1, end: result.pageCount } + // On stdout as well as the exit code: an empty fence at exit 0 read as a broken command (an agent spent three calls doubting it), and an agent's `2>/dev/null` or `| head` would hide a note written only to stderr. + out(noTextLayerNote(file, range.start, range.end, result.pageCount)) + throw new CliError(`no text layer on the requested pages of ${echoedValue(file)}; it looks scanned or image-only`) + } const text = redactSecrets(result.text).text const printed = _applyFiltersAndPrint(text, opts, true, UNTRUSTED_FILE_TAG) // On stderr after the body, as the recall filters' own notes are: stdout is fenced document text, where a token-goat line would read as part of the document. diff --git a/tests/helpers/matrix_cases.ts b/tests/helpers/matrix_cases.ts index ff001daec..5007eb1c9 100644 --- a/tests/helpers/matrix_cases.ts +++ b/tests/helpers/matrix_cases.ts @@ -215,6 +215,15 @@ export const MINIMAL_PDF = '%PDF-1.4\n' + '5 0 obj\n<< /Length 44 >>\nstream\nBT /F1 24 Tf 20 100 Td (Hello PDF) Tj ET\nendstream\nendobj\n' + 'trailer\n<< /Size 6 /Root 1 0 R >>\n%%EOF\n' +// Minimal hand-authored single-page PDF that only paints a 2x2 grayscale image, so its page has no text layer: the shape of a scanned document, for the pdf-extract case below. +export const IMAGE_ONLY_PDF = '%PDF-1.4\n' + + '1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n' + + '2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n' + + '3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources << /XObject << /Im1 4 0 R >> >> /MediaBox [0 0 200 200] /Contents 5 0 R >>\nendobj\n' + + '4 0 obj\n<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /ASCIIHexDecode /Length 9 >>\nstream\n00FFFF00>\nendstream\nendobj\n' + + '5 0 obj\n<< /Length 30 >>\nstream\nq 200 0 0 200 0 0 cm /Im1 Do Q\nendstream\nendobj\n' + + 'trailer\n<< /Size 6 /Root 1 0 R >>\n%%EOF\n' + // Two-font single-line PDF for the pdf-locate case below: pdfjs 6.3.289 only emits separate text items with no space item between them when adjacent Tj operators switch fonts (same-font runs get merged into one item), so a MINIMAL_PDF-style single-Tj fixture never exercises the item join this smoke test is here to cover. export const MIXED_FONTS_PDF = '%PDF-1.4\n' + '1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n' + @@ -1024,6 +1033,16 @@ export const cases: Record void | Promise> = { const rLayout = run(['pdf-extract', pdfPath, '--layout']) expect(rLayout.status, rLayout.stderr).toBe(0) expect(rLayout.stdout).toContain('Hello PDF') + // A scanned page used to come back as an empty fence at exit 0. Now it exits 1 so `|| fallback` fires, and stdout alone (as under `2>/dev/null`) names the pages and a rasterize route instead of a fence. + const scanPath = path.join(dir, 'scan.pdf') + fs.writeFileSync(scanPath, Buffer.from(IMAGE_ONLY_PDF, 'latin1')) + const rScan = run(['pdf-extract', scanPath]) + expect(rScan.status).toBe(1) + expect(rScan.stdout).toContain('No text layer on page 1 of 1 page') + expect(rScan.stdout).toContain('-dFirstPage=1 -dLastPage=1') + expect(rScan.stdout).toContain('token-goat image-text') + expect(rScan.stdout).not.toContain('') + expect(rScan.stderr).toContain('no text layer on the requested pages') }, 'pdf-locate': () => { const dir = mkIsolated('tg-matrix-pdfl-') diff --git a/tests/pdf_extract.test.ts b/tests/pdf_extract.test.ts index 906b83f03..755c85614 100644 --- a/tests/pdf_extract.test.ts +++ b/tests/pdf_extract.test.ts @@ -30,6 +30,34 @@ function pdfBytes(): Uint8Array { return new Uint8Array(Buffer.from(MINIMAL_PDF, 'latin1')); } +// Same shape, but the page only paints a 2x2 grayscale image, so it has no text layer: what a scanned page looks like to pdfjs. +const IMAGE_ONLY_PDF = `%PDF-1.4 +1 0 obj +<< /Type /Catalog /Pages 2 0 R >> +endobj +2 0 obj +<< /Type /Pages /Kids [3 0 R] /Count 1 >> +endobj +3 0 obj +<< /Type /Page /Parent 2 0 R /Resources << /XObject << /Im1 4 0 R >> >> /MediaBox [0 0 200 200] /Contents 5 0 R >> +endobj +4 0 obj +<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /ASCIIHexDecode /Length 9 >> +stream +00FFFF00> +endstream +endobj +5 0 obj +<< /Length 30 >> +stream +q 200 0 0 200 0 0 cm /Im1 Do Q +endstream +endobj +trailer +<< /Size 6 /Root 1 0 R >> +%%EOF +`; + // Same shape as MINIMAL_PDF, but with three text objects: two on the same row at different x (a simulated two-column line) and one on a row below, to exercise --layout's row-grouping + left-to-right reconstruction. const LAYOUT_PDF = `%PDF-1.4 1 0 obj @@ -205,6 +233,13 @@ describe('extractPdfText', () => { expect(result.text).toContain('Hello PDF'); }); + // pdf-extract's no-text-layer note keys on this: a page that only paints an image extracts to nothing at all, not to whitespace. + it('extracts nothing from a page that only paints an image, the shape of a scan', async () => { + const result = await extractPdfText(new Uint8Array(Buffer.from(IMAGE_ONLY_PDF, 'latin1'))); + expect(result.pagesExtracted).toBe(1); + expect(result.text).toBe(''); + }); + it('respects a --pages range beyond the document, clamped to what exists', async () => { const result = await extractPdfText(pdfBytes(), '1-5'); expect(result.pagesExtracted).toBe(1); From 1441c5322654511c83999372c1a9ba16b6315026 Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 05:32:08 -0500 Subject: [PATCH 07/92] fix(read): repeat a miss's suggestions on stdout so 2>/dev/null cannot hide them A read command's miss went to stderr only, Did you mean list and Try: line included. 14 of about 50 recorded agent calls ran these commands with 2>/dev/null; piped into head, two misses printed nothing at all, and in a third the || fallback then put a raw file holding a live Shopify token into context. section, read, symbol, brief, outline and skeleton now repeat the lines after a failure's first one on stdout as well, escaped as stderr escapes them, and still exit 1. That is where every miss puts its guidance: the near names, the Try: command, the file a symbol is really defined in, or the qualified commands an ambiguous name needs. A one-line miss and a --json body print nothing extra. tests/read_miss_suggestion_on_stdout.test.ts runs each command on the built bundle with stderr discarded. --- docs/cli.md | 2 +- src/cli.ts | 4 + src/cli_dispatch.ts | 8 +- src/command_error.ts | 7 ++ src/read_brief.ts | 11 ++- tests/read_miss_suggestion_on_stdout.test.ts | 94 ++++++++++++++++++++ 6 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 tests/read_miss_suggestion_on_stdout.test.ts diff --git a/docs/cli.md b/docs/cli.md index 408145596..490e2fe3c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -187,7 +187,7 @@ token-goat pdf-extract manual.pdf --pages 12-15 --layout --head 120 | `token-goat baseline` | Emit a project map: file count, per-language file counts, the top indexed symbols (by name/kind/location), and the most recently modified files. `--subagent` emits a terser variant (fewer symbols, and the recent files only with `--json`) for context handed to a freshly spawned subagent; `--json` for the machine-readable form. | | `token-goat compact-doc ` | Build an extractive compact sidecar for a large reference doc (`.md`/`.markdown`). The compact is stored in the token-goat data dir as a SHA-keyed sidecar; `pre_read` serves it in place of the full file when it exists and is fresh, typically saving 60–95% of the tokens the full read would cost (measured across this repo's own 44 docs: median 67%, range 5–99%, depending on how much of the doc is prose under headings). Use `--force` to rebuild, `--sentences N` to control lines per section (default 2), `--show` to print the result. The sidecar is automatically marked stale when you edit the source file. Config: `[hints] stable_doc_compacts = true` (default on). | -Missed lookups recover surgically: `read` and `section` print a "Did you mean…?" list on a miss, and `section` auto-redirects on an unambiguous heading-prefix match — a typo costs at most one extra glance, not a re-read. +Missed lookups recover surgically: `read` and `section` print a "Did you mean…?" list on a miss, and `section` auto-redirects on an unambiguous heading-prefix match — a typo costs at most one extra glance, not a re-read. The miss exits 1 with its message on stderr, and `read`, `section`, `symbol`, `brief`, `outline` and `skeleton` repeat its suggestion lines ("Did you mean", "Try: token-goat ...") on stdout as well, so a call run with `2>/dev/null` or piped into `head` still shows the correction. ### Skill efficiency — the `` marker diff --git a/src/cli.ts b/src/cli.ts index 8f08fa769..d2159aef6 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -968,6 +968,7 @@ export function buildProgram(): Command { // `symbol` searches one name; there is no comma form that would search several. { noun: 'spec', mergeable: false }, ), + { echoSuggestion: true }, ) }) @@ -995,6 +996,7 @@ export function buildProgram(): Command { // Whether the comma form the note names would actually run depends on the specs themselves, not on the command: `read` merges `::` specs and nothing else. { noun: 'spec', mergeable: readSpecsMergeable([spec, ...more]) }, ), + { echoSuggestion: true }, ), ) @@ -1055,6 +1057,7 @@ export function buildProgram(): Command { }), { noun: 'spec', mergeable: namedSpecsMergeable([spec, ...more]) }, ), + { echoSuggestion: true }, ) }) @@ -1108,6 +1111,7 @@ export function buildProgram(): Command { ...(opts.stats === true ? { stats: true } : {}), }), ), + { echoSuggestion: true }, ), ) } diff --git a/src/cli_dispatch.ts b/src/cli_dispatch.ts index ad8bf15b8..ea104c0fc 100644 --- a/src/cli_dispatch.ts +++ b/src/cli_dispatch.ts @@ -4,7 +4,7 @@ import * as fs from 'node:fs' import { extractErrorMessage } from './util.js' import { extraFileArgsNote } from './read_spec.js' import { out, err, CliError } from './cli.js' -import { formatCommandError, formatFailedResultText } from './command_error.js' +import { failureSuggestion, formatCommandError, formatFailedResultText } from './command_error.js' import { echoedValue } from './hint_suggestion_guard.js' export function readStdinPaths(): string[] { @@ -29,12 +29,14 @@ export function runExit(fn: () => number): void { } } -/** Same adapter as `runExit`, but for the `run*` handlers that return `{ text, code }` instead of printing directly. Writes `text` to stdout on success (code 0), or to stderr as a command error otherwise, then maps `code` onto `process.exitCode` — preserving which stream each handler's message goes to (these handlers only ever write to one stream per call). */ -export function runExitText(fn: () => { text: string; code: number }): void { +/** Same adapter as `runExit`, but for the `run*` handlers that return `{ text, code }` instead of printing directly. Writes `text` to stdout on success (code 0), or to stderr as a command error otherwise, then maps `code` onto `process.exitCode` — preserving which stream each handler's message goes to (these handlers only ever write to one stream per call). `echoSuggestion` is the read family's exception: the lines after a failure's first one, its Did you mean / Try: guidance ({@link failureSuggestion}), go to stdout as well, so `2>/dev/null` cannot hide them. */ +export function runExitText(fn: () => { text: string; code: number }, opts: { echoSuggestion?: boolean } = {}): void { try { const { text, code } = fn() if (code === 0) out(text) else err(formatFailedResultText(text)) + const suggestion = code !== 0 && opts.echoSuggestion === true ? failureSuggestion(text) : null + if (suggestion !== null) out(suggestion) process.exitCode = code } catch (e) { err(formatCommandError(e)) diff --git a/src/command_error.ts b/src/command_error.ts index 757895a7a..14728888e 100644 --- a/src/command_error.ts +++ b/src/command_error.ts @@ -43,6 +43,13 @@ export function formatFailedResultText(text: string): string { return isJsonBody(text) ? text : formatCommandError(new CliError(text.split('\n'))) } +/** The part of a read command's failure text that says what to do next: every line after the first, which states the miss, escaped line by line as {@link displaySafeFailureText} does. That is where each miss puts its `Did you mean:` list, its `Try: token-goat ...` line, the file a symbol is really defined in, or the qualified commands an ambiguous name needs. Null for a one-line failure and for a JSON body. The read commands repeat it on stdout because their misses went only to stderr: 14 of about 50 recorded agent calls ran them with `2>/dev/null`, so a miss piped into `head` printed nothing at all, and in one the `||` fallback then put a raw file holding a live API token into context. */ +export function failureSuggestion(text: string): string | null { + if (isJsonBody(text)) return null + const rest = text.split('\n').slice(1) + return rest.some((line) => line.trim() !== '') ? rest.map(displaySafeText).join('\n') : null +} + /** The same failure text with each line escaped on its own and no `token-goat:` label, for a surface that is not stderr: the MCP server hands it back as a tool result. A JSON body passes through untouched. */ export function displaySafeFailureText(text: string): string { return isJsonBody(text) ? text : text.split('\n').map(displaySafeText).join('\n') diff --git a/src/read_brief.ts b/src/read_brief.ts index 49c0da90a..26e7985d4 100644 --- a/src/read_brief.ts +++ b/src/read_brief.ts @@ -22,7 +22,7 @@ import { fileIsGone, findSpecSeparator, guardText, indexFreshness, readFileText, import { emit, emitErr } from './emit.js' import { formatAmbiguity, parseCrossFileMultiSpec, parseReadSpec, resolveSymbolSpec, specScopeRoot } from './read_spec.js' import { formatBareNameSpecError, trimBlankLines } from './read_suggest.js' -import { formatCommandError, formatFailedResultText } from './command_error.js' +import { failureSuggestion, formatCommandError, formatFailedResultText } from './command_error.js' import { echoedValue } from './hint_suggestion_guard.js' export interface BriefOptions { @@ -225,6 +225,12 @@ export function runBriefCrossFile(pairs: { file: string; symbol: string }[], opt return { text, code: anyFound ? 0 : 1 } } +/** A failed result's Did you mean / Try: lines repeated on stdout below its stderr message, as `runExitText`'s `echoSuggestion` does for the other read commands, so `2>/dev/null` cannot hide them. */ +function echoSuggestion(text: string): void { + const suggestion = failureSuggestion(text) + if (suggestion !== null) emit(suggestion) +} + /** Handle ``token-goat brief "file::symbol"``: dispatches to {@link runBriefCrossFile} for a cross-file `a.ts::x,b.ts::y` spec, to {@link runBriefMulti} for a comma-separated same-file `file::a,b` spec, otherwise runs the single-symbol {@link runBriefCore} path, then emits the result -- `emitErr` on a nonzero code, `emit` on success. */ export function runBrief(opts: BriefOptions): number { // Same reasoning as runRefs/runFind/runTypes: a limit of 0 (or negative) would silently slice the caller list down to zero entries instead of surfacing a clear "you asked for nothing" error, consistent with every other --limit flag in this codebase. Validated once here rather than inside runBriefCore because --limit applies to the whole invocation, so a multi-symbol spec must report it once, not once per symbol. @@ -239,6 +245,7 @@ export function runBrief(opts: BriefOptions): number { const { text, code } = runBriefCrossFile(crossFilePairs, opts) if (code === 0) emit(text) else emitErr(formatFailedResultText(text)) + if (code !== 0) echoSuggestion(text) return code } @@ -249,6 +256,7 @@ export function runBrief(opts: BriefOptions): number { const { text, code } = runBriefMulti(file, multiSymbols, opts) if (code === 0) emit(text) else emitErr(formatFailedResultText(text)) + if (code !== 0) echoSuggestion(text) return code } } @@ -256,5 +264,6 @@ export function runBrief(opts: BriefOptions): number { const { text, code } = runBriefCore(opts) if (code === 0) emit(text) else emitErr(formatFailedResultText(text)) + if (code !== 0) echoSuggestion(text) return code } diff --git a/tests/read_miss_suggestion_on_stdout.test.ts b/tests/read_miss_suggestion_on_stdout.test.ts new file mode 100644 index 000000000..b70341a2c --- /dev/null +++ b/tests/read_miss_suggestion_on_stdout.test.ts @@ -0,0 +1,94 @@ +/** Regression: a read command's miss went only to stderr, so `token-goat section "f.md::Development setup" 2>/dev/null | head -60` printed nothing at all; 14 of about 50 recorded agent calls redirected stderr that way, and in one the `||` fallback then put a raw file holding a live API token into context. Each of `section`, `read`, `symbol`, `brief`, `outline` and `skeleton` now repeats the lines after a miss's first one (its Did you mean / Try: guidance) on stdout, keeping exit 1. Every case runs the built bundle with stderr discarded, as the agent did. */ +import { execFileSync, spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' + +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { BUNDLE, tgIsolatedEnv } from './helpers/bundle.js' + +let root: string +let repo: string +let env: NodeJS.ProcessEnv + +function runNoStderr(args: string[]): { status: number | null; stdout: string } { + const r = spawnSync(process.execPath, [BUNDLE, ...args], { cwd: repo, env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 60000 }) + return { status: r.status, stdout: r.stdout ?? '' } +} + +beforeAll(() => { + root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), 'tg-miss-stdout-'))) + repo = path.join(root, 'repo') + fs.mkdirSync(repo) + fs.writeFileSync(path.join(repo, 'mod.ts'), 'export function alphaSym(): number {\n return 1\n}\nexport function betaSym(): number {\n return 2\n}\n') + fs.writeFileSync(path.join(repo, 'dupfile.ts'), 'export class DupHolder {\n dup(): number {\n return 10\n }\n}\nexport function dup(): number {\n return 20\n}\n') + fs.writeFileSync(path.join(repo, 'doc.md'), '# Doc\n\n## Install\n\nRun npm install.\n\n## Usage\n\nRun it.\n') + fs.writeFileSync(path.join(repo, 'note.xyz'), 'plain\n') + const git = (...args: string[]): void => { + execFileSync('git', ['-c', 'core.hooksPath=/dev/null', ...args], { cwd: repo, stdio: 'ignore' }) + } + git('init', '-q', '.') + git('add', '-A') + git('-c', 'user.email=t@t.t', '-c', 'user.name=t', 'commit', '-qm', 'seed') + env = tgIsolatedEnv(path.join(root, 'home'), { TOKEN_GOAT_HOME: path.join(root, 'data'), TOKEN_GOAT_EMBEDDINGS_ENABLED: 'false' }) + const idx = spawnSync(process.execPath, [BUNDLE, 'index', '.'], { cwd: repo, env, encoding: 'utf8', timeout: 120000 }) + expect(idx.status, idx.stderr).toBe(0) +}, 180000) + +afterAll(() => { + fs.rmSync(root, { recursive: true, force: true }) +}) + +describe('a read-command miss with stderr discarded', () => { + it('section names the near heading on stdout', () => { + const r = runNoStderr(['section', 'doc.md::Install steps']) + expect(r.status).toBe(1) + expect(r.stdout).toContain('Did you mean:') + expect(r.stdout).toContain('- Install') + }) + + it('section points at outline on stdout when nothing is close', () => { + const r = runNoStderr(['section', 'doc.md::Zzqx']) + expect(r.status).toBe(1) + expect(r.stdout).toContain('Try: token-goat outline "doc.md"') + }) + + it('read names the near symbol on stdout', () => { + const r = runNoStderr(['read', 'mod.ts::alphaSm']) + expect(r.status).toBe(1) + expect(r.stdout).toContain('Did you mean:') + expect(r.stdout).toContain('alphaSym') + }) + + it('symbol names the near symbol on stdout', () => { + const r = runNoStderr(['symbol', 'alphaSm']) + expect(r.status).toBe(1) + expect(r.stdout).toContain('alphaSym') + }) + + it('brief lists the qualified commands for an ambiguous name on stdout', () => { + const r = runNoStderr(['brief', 'dupfile.ts::dup']) + expect(r.status).toBe(1) + expect(r.stdout).toContain('token-goat brief "dupfile.ts::DupHolder.dup"') + }) + + it.each(['outline', 'skeleton'])('%s says what still works on a file type it cannot list', (command) => { + const r = runNoStderr([command, 'note.xyz']) + expect(r.status).toBe(1) + expect(r.stdout).toContain('To ask for .xyz file support') + }) + + it('keeps stdout empty for a one-line miss that offers nothing to run', () => { + const r = runNoStderr(['outline', 'missing.ts']) + expect(r.status).toBe(1) + expect(r.stdout).toBe('') + }) + + it('leaves a hit unchanged', () => { + const r = runNoStderr(['section', 'doc.md::Install']) + expect(r.status).toBe(0) + expect(r.stdout).toContain('Run npm install.') + expect(r.stdout).not.toContain('Did you mean') + }) +}) From 05221b07ac1b4ff3dffb91c30759fb604f53bab4 Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 05:36:50 -0500 Subject: [PATCH 08/92] feat(section): read every spec passed, not only the first section a b read a and printed "Note: 1 extra spec argument ignored", exit 0, so agents chained one section per heading: 14 spawns across three recorded commands, about 1-2 s each. The comma form that serves them in one call was in --help but not in the skill agents load. Specs that each name a heading are now served the way their comma form is, through runSectionCrossFile, but paired from the specs themselves rather than a comma-joined string, so a heading that holds a comma stays whole. Specs that cannot merge run one after another, one block each, and the call succeeds when any does. Words a shell split out of one unquoted heading keep the note that says how to quote it. The token-goat skill (src/canonical_skill.md, regenerated) gains one failure-shape line naming both forms; section --help and docs/cli.md say several specs read like the comma form. tests/cli_extra_positional_args.test.ts drops section from the commands that note extra specs and pins the new reads, and the section matrix case covers two specs on the built bundle. --- docs/cli.md | 2 +- src/canonical_skill.md | 1 + src/canonical_skill.ts | 146 ++++++++++++------------ src/cli.ts | 32 +++--- src/read_section.ts | 24 +++- src/read_spec.ts | 9 +- tests/cli_extra_positional_args.test.ts | 28 +++-- tests/helpers/matrix_cases.ts | 6 + tests/section_multiple_specs.test.ts | 67 +++++++++++ 9 files changed, 212 insertions(+), 103 deletions(-) create mode 100644 tests/section_multiple_specs.test.ts diff --git a/docs/cli.md b/docs/cli.md index 490e2fe3c..d3dadf820 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -37,7 +37,7 @@ token-goat pdf-extract manual.pdf --pages 12-15 --layout --head 120 | `token-goat note-get [--symbol NAME]` | Read back the note attached to a file or one indexed symbol within it. Flags whether the note has gone stale (the underlying code changed since it was written) via a `stale` field under `--json`. | | `token-goat note-list [--stale-only]` | List every recorded architecture note. `--stale-only` shows just the notes whose fingerprint no longer matches the current index — i.e. the file/symbol they describe changed since the note was written. Staleness is purely advisory: nothing here auto-rewrites or deletes a note. | | `token-goat write-file ` | Write exact bytes to a file, sidestepping shell-escaping trouble with backticks, quotes, `$vars`, and CRLF. `--from ` copies bytes from a source file; `--b64 ` decodes a base64 payload; with neither, reads from stdin. | -| `token-goat section "doc.md::Heading"` | Pull one Markdown section by heading. Fully supports ATX (`#`/`##`) and Setext-style (`Heading\n===` level 1 and `Heading\n---` level 2) headings. A miss that is an unambiguous prefix of exactly one heading, or a distinctive suffix/word-subset of exactly one heading (e.g. `Setup` → "Installation and Setup", `Config Options` → "Configuration Options"), auto-redirects with a `(redirected from: …)` marker (and a `redirectedFrom` field under `--json`); a query matching 2+ headings is never guessed and reports a miss instead. A genuine miss lists only headings similar to the query as "Did you mean" suggestions, not every heading in the file. Disambiguate duplicates with `"doc.md::Heading#2"`. Comma-separated `"doc.md::A,B"` fetches several sections from one file in a single call, mirroring `read`'s `file::a,b` multi-symbol grammar, and automatically subsumes nested child sections (`(already included in section 'Parent', lines X-Y)`) to prevent duplicate tokens. Cross-file `"a.md::Heading1,b.md::Heading2"` fetches sections from several files in one call, mirroring `read`'s `a.ts::x,b.ts::y` cross-file grammar — a bare heading after a `file::Heading` segment inherits the previous file, and each section is keyed by its full `file::Heading` pair so two files sharing a heading name cannot overwrite each other. `token-goat section doc.md --list` lists every heading in the file instead of reading one; `--grep ` narrows that list to headings matching a regex (falls back to a literal substring match if the pattern doesn't compile), same convention as `outline`/`types`/`exports`'s own `--grep`. Without `--list`, `--grep` filters the section body instead: only the matching lines print, each under the nearest sub-heading above it, followed by `N of M lines matched`. A pattern that matches nothing says so and exits 0. YAML front matter at the top of a file is skipped, so its closing `---` is never read as a heading underline. | +| `token-goat section "doc.md::Heading"` | Pull one Markdown section by heading. Fully supports ATX (`#`/`##`) and Setext-style (`Heading\n===` level 1 and `Heading\n---` level 2) headings. A miss that is an unambiguous prefix of exactly one heading, or a distinctive suffix/word-subset of exactly one heading (e.g. `Setup` → "Installation and Setup", `Config Options` → "Configuration Options"), auto-redirects with a `(redirected from: …)` marker (and a `redirectedFrom` field under `--json`); a query matching 2+ headings is never guessed and reports a miss instead. A genuine miss lists only headings similar to the query as "Did you mean" suggestions, not every heading in the file. Disambiguate duplicates with `"doc.md::Heading#2"`. Comma-separated `"doc.md::A,B"` fetches several sections from one file in a single call (several specs, `section "a.md::A" "b.md::B"`, read the same way), mirroring `read`'s `file::a,b` multi-symbol grammar, and automatically subsumes nested child sections (`(already included in section 'Parent', lines X-Y)`) to prevent duplicate tokens. Cross-file `"a.md::Heading1,b.md::Heading2"` fetches sections from several files in one call, mirroring `read`'s `a.ts::x,b.ts::y` cross-file grammar — a bare heading after a `file::Heading` segment inherits the previous file, and each section is keyed by its full `file::Heading` pair so two files sharing a heading name cannot overwrite each other. `token-goat section doc.md --list` lists every heading in the file instead of reading one; `--grep ` narrows that list to headings matching a regex (falls back to a literal substring match if the pattern doesn't compile), same convention as `outline`/`types`/`exports`'s own `--grep`. Without `--list`, `--grep` filters the section body instead: only the matching lines print, each under the nearest sub-heading above it, followed by `N of M lines matched`. A pattern that matches nothing says so and exits 0. YAML front matter at the top of a file is skipped, so its closing `---` is never read as a heading underline. | | `token-goat skill-section "::"` | Extract a named section from an installed skill without reading the full skill file. | | `token-goat skeleton "file"` | Show all signatures in a file without bodies — typically 70–90% fewer tokens than a full read. `--force-refresh` reparses from disk first, bypassing a stale index. `--stats` adds a per-symbol reference count and doc-coverage flag, computed live from the index. `--grep ` narrows to symbols whose name matches a regex (a literal substring when the pattern is not valid regex), which is how you skim one area of a large file without dumping its whole symbol list; `--min-lines ` drops symbols shorter than N lines. Both compose, and if a filter removes everything the output says so and names the filter, rather than looking like a file with no symbols. Accepts a comma-separated file list (`"a,b,c"`) to cover several files in one call, one clearly-headed block per file; extra space-separated file arguments are reported in a note naming that comma form instead of being silently dropped. With `--json`, a comma-separated list returns one merged document (rows carry their own `filePath`), not one document per file. | | `token-goat outline "file"` | List top-level symbols with line ranges and docstring hints — one-glance file map. Doc hints are clipped to about a sentence, with a visible ellipsis whenever the doc goes on past the hint; the full doc comment is one `read "file::symbol"` away (`--json` carries it whole). `--force-refresh` reparses from disk first, bypassing a stale index. `--stats` adds a per-symbol reference count and doc-coverage flag, computed live from the index. `--grep ` narrows to symbols whose name matches a regex (a literal substring when the pattern is not valid regex), which is how you skim one area of a large file without dumping its whole symbol list; `--min-lines ` drops symbols shorter than N lines. Both compose, and if a filter removes everything the output says so and names the filter, rather than looking like a file with no symbols. Accepts a comma-separated file list (`"a,b,c"`) to cover several files in one call, one clearly-headed block per file; extra space-separated file arguments are reported in a note naming that comma form instead of being silently dropped. With `--json`, a comma-separated list returns one merged document (rows carry their own `filePath`), not one document per file. | diff --git a/src/canonical_skill.md b/src/canonical_skill.md index da89938e7..fc8762a2e 100644 --- a/src/canonical_skill.md +++ b/src/canonical_skill.md @@ -40,6 +40,7 @@ Failure shapes to catch yourself in, and the command that replaces each: - paging one function with view/view_range → `read "file::symbol"` - reading a symbol plus chasing its callers and containing doc section as separate reads → `brief "file::symbol"` - reading one heading of a large doc → `section "file::Heading"` +- one `section` call per heading, chained → a single call: `section "file::A,B"` or `section "a.md::A" "b.md::B"` - reading a procedural shell script check or step → `section "script.sh::# Banner"` or `read "script.sh::# Banner"` - searching for a symbol's callers → `refs file::symbol --callers` - searching for a *concept* rather than a literal string → `semantic "description"` diff --git a/src/canonical_skill.ts b/src/canonical_skill.ts index a07856690..d0302db90 100644 --- a/src/canonical_skill.ts +++ b/src/canonical_skill.ts @@ -3,78 +3,78 @@ import { gunzipSync } from 'node:zlib' // Keep the canonical document out of the eagerly loaded startup bundle. const CANONICAL_SKILL_GZIP_BASE64 = - "H4sIAAAAAAACCp1b25IbOXJ951ekeyIsNYPFlnZ9bU3Mhua68mpmtJI2xhsbG0uwKsnCdBVQAlBN0aFw7JM/wOEv8Kftl9gnEygWW93jWT+Mhk1WoYBE4uTJ" + - "k1lVVS2c6fmakr9hV+29SYuGYx3skKx313Txu8i05Z0PTIFNY92eDq3vmHa240g+0D7wMMj3tuH1bCSqfd8b10R6HI/91ncrGWJFkWuMjg+9ccnWK/Jj6qzj" + - "FfVmwGW7uKKtiW3lxzSM6ZICpzE4ciYEf6DY2ZojmUSdP+gjqfYxrWlzNZuAGRubNhT41vIhkqHIMVrvaOcD9TZGbiiaW+v2cX2xMGE/9uxS1VqXruniD3L7" + - "H+kDbU1gsu7W1wYTJzMMneVIqVWr0N4kfkaP5IZHFEY3f1jgFHwcsOhbJuMa4t4mXPGtsS4Z6zjQ18zN1tQ39IUJzcXilkPUDXi6/uf1Ly8Wpuv8gZsqed/F" + - "6wVRRZ+b2MqH12wa+fBN4EE/dH67qKpq8enfVBV9w46DSdzQ9jjfn13wPcVQX9XGeWdr0/0p3tiuW/fNM+LGJrIJaww66zC6+d3WxWS6bkWObzlQam0Up1hT" + - "VX22WHzyCf3QMu6g30VeLJ53B3PErqozGblWrfeXP/8XpRZDqkkHb11ayTOtfHlojzq+TA/fGDE5BYPp4WZHhhLXrbPvRl7TW5m1jeQ8pTZwbH3XYC6BTd3K" + - "0PjFxBuK9t9YH8/vuR9SJJuuZR7vRo6y3xc6MZl2dmq9Q70y0o9jTHTANy/IMTcX4o14empxNJInPErvgzXjgQOmN8bRdN2RjhzXi8XblvMkrIcLBS4Ojzs7" + - "G2UPGT4/X3nHu4RH/Dg2e4YHw8q1GSMXOx1saqmxsQ6MoU04zp8jZkr52jU9160ZXcOBgh/3bXekXzx5QjihURd+9KMsVMFgVbZTfjuYmH3Cuobfc6NbaWjP" + - "brSOuyP5wbwbmbZWpoJ1wiy1iYzNVkM3NnCd1EOyYwS7bxPVpuvWi8UPusWJBnOM1PsormITXB0H7eDDzTUZ6rwfMEFFARpMjCvgFn5x+9kprfzATh8UTV8Q" + - "LvDAODvdUZch/hO4KvOqWxP23JDZG+uASLux62gItmb9LsKyW2yNHCZ1OpsidzsxITb+O7b7duvHkM3rD/Bn2APIKMu+XiwqWi43Zg+Iwt1V7fvBCJRulks5" + - "ReKCcQy39hbwSKcrVmceI9exSxwi1d4lfo/zLNPb2RATDZ2p9RiRl9ta0+3I707W0RWtdVJx3Oq8mmBv2VUN33LnB/himZqhOJiDE0uxSxSTCSnStjNOD0bg" + - "SnYS2AzvOerRAVJug+Ud+d6mBG86tLZuFQXkF1w2BL/tuC/worhhBHst35qOvOMy2Rx1NvT4BCqPohg9n+7LyZ7sxNfhopFNqGVOO3GfZe1dzUNa3oGhTnyw" + - "o5iCdfvVBG0n7IDnrvKZC2yiF5jcma4jiQGACwmnElzXAqazCL1YLr/BqcYM83rFYCdIXRWI8W4GY7K11zQDs4+D9U/j2q+WS3qxA1qtJBzYBLiQo4DAJDHy" + - "BqhgU+vHlGchCNiyQpEY4tb6zqhbihUceYl4+zap05VLBw6yqmvamlS33ODkDiaYruNOHhyp8TKIAhqmBCC1UcdouLYNR1oeWsaql6cgoLt4mjwIQ9et6YvO" + - "jA3TF77h7BWvxaSIrooBCK80BN5xYFczhVG4kOuONFjsX8u0xHZiN5cEN5mbujWRtsxO7mtwwmUu4oyzgMjHDKNmTK0PCFPxxirXKuZcLxZf5+dQ3QHyI/Xm" + - "SGB1cFwc3OA4xrwSp9hYeBgcOsMawr2QCwFHnLrYctdRy93AIa7pjfwpkA3uU27CfgS/D6bXgDWRvuILqZUYosghY+AxE20Qt2nYJbuzHKLOpoff5cUPAxuE" + - "EniJQMejqBMFf3GpNynB0YP8PmdJEjDhTvAjE8w+mKGlhju7FTrUqZmEI4Ce2Z2tdT4+5NB0LSwnjAKhlRyvzpsGsNmacMsRaFu3eRzr5OCqM2EmmEAJ+o4O" + - "JrhMNOS04mwqovZCBdyN8we3Xiy+OoXmx+LEiFly5OAcGhe74+V1xuuO9XaE638/xWk8dwrTgBa/I5ueARo/itD02PFhRaNLwdQ3wFgQ+4k2imcCES6fTcj6" + - "UDB/nA+07c2eL5/lkBcQ/eD3sLUkA9Qa13RMj3m9X89Qs7dNtRudWPxS3Nt2I9C3NQPYh6caSCDOLTHUutVErO6AmESxKCH7elEhBKn/Idyp8yszKiFw15m9" + - "PGJnhbCUedDWN0f6y3/8J21kCy4Ekq51IRebRUWD2eNYYkunm2RocI4r/POnYNyef3KQkmCZYqKhG4VgxBIIQX3gfFgjJm2sw0+Nr0tSRSZSZLh74oyP8kgN" + - "oQ8/EzNvy+cd9gN7JiPL/WX4PMKv9dI70x6Cr7kZJfiJpTWTpLrl+gYuFRMPd8bTS9axvb7+hD43znG42ODabKV7f19UH0VjXdOjk42ypXealZRFU1XlC+4b" + - "5OeF9LICpRF0McuYZWq3HOzuaIVc5mu433KjVro1tjNb29l0LNH73cjh+PHAVYUY0yEqqkVmvxxM6NX41bSFkiorXenw5ZFqM6Qx/G/SoD5wSqg3V5sDb09/" + - "9PVQ/kCS+OLLs2UgtQq9daabTpipk2Yuh+AT0w4pVpnB3YdRpdD56WBS+xlVlY5cdSam6hBsYqwE6QlYgNitbrk3simFn8cES/kgvH6HFKgxyWxBkQvb2I+4" + - "9mRGubHKY/0hmW3Hf1RD6o5tmT5VaPps+llgQiHWh6OEELFtoMD1GKJkFrZrahMaOphOyJoPFpRWY9Todqa3nTW4ZfA6md4M8Dyl4wIYY9eVc3drulGWFsdt" + - "CiycXw/hv7z5/rur3z//9uXVv377UqH+cW+c3XFMK+p8fYPvVhK+VkCEnd1f6hN/jN5V4lh63yOz3q7rRxu6os3R9N1Dv72//6dFRUiPxBsyuAsWy8kGNDW2" + - "ZweDxxX2rTdJgohk1zIhuafqOWmueDYigILZxdanFTUWobrX22vjMmNThC+pik3zURW9Pxr11ZdfY5DvdztkY42vRd+RG60TRUamrtNVerzSWC3PM2dy07UM" + - "txmanSxicyUfs3C10dvkqw4yEb7JcUQkDbMvqTOmIEAth0qgXG7j9wi+OCxy8Ub5ZGp95Gf0gw8NbRpfv7/7RPlOnDdurvJf/D5tntErf+DwCloKbYYhfXSn" + - "fBc72zDWgj+cTxw3z+ir9zV3tHnfxfdVbJlTLPfIV7Xvxt7hcfInwkb5LEGu/CFutFmchQgNK29+8+Lly3XfXH3x6+ffffPVy++/WfcNPZZTVs3CoMg9BcyE" + - "eTXZveWnqkSRT0HAPqNHn+a49Jn6sl4kwVuvEMQ0h+r19IgZ2Z+gBqfuJaVgXI5fZ3hyx4oTzMBHMiKfwkptJGlRVqlgaOrgo8oMqqSWx+aAhfBEn4rtZL5g" + - "3xmFDMUev8oIEsdoB5exSFwdMijVJwufkRGxqRQ5Xe05XY0On4Bqm8UXmalf0ybHxu+ef/vVZnU/P1k9wCFWD3OD1YNBckWbbEY9tVebeMMdJ0lSO8bv54i5" + - "+ulQvqJN3cL1Gqoq/VW+E0CswD0FzH7z1e/x9UPQeDUDRvjw9Lncc2fOcnVxB72+/LX6udF2RZu9iCbFlyN9iuEr23yGH++i5tVHiLd6EJOu5nB0NQcZ3HQG" + - "JlcP4sjqDnhcPYgb+Y9y1xw9rv4/wLG6c8rFS8qBVuc6P5FXdw7jqhwnfHroGKwXizfjtlJRSjw8avpZmxCy7CxJ2C2HrUm2h6YYpzusazlYkXpLOmdPmppK" + - "2DET8ThwDgeioGnZouhoi81MIYjJAHOhA6jShzC7pq87EWpGh3g7ClOqAVygRNSzcWcyy5aFSkEy4EZFpDfjtuTni8UHenEqaXyg53qIPyw+VFUl/y0+nJVT" + - "NiqbumOWBEu9hD6cJBuz9ahxaH3kXIafspKSJQiBCKNb090nlcLNB3rzYA3lodpJVsU6f8C4i08++URHy8a8Z7jF4nUuEMnW5Es0zy0ihtWa0WyKfki2txlw" + - "/TD4kEZnU5FGQP2gbyYvo/bTbFeaFzoPOwzBckLWDC4L67568WKd9T2YmG8hjdQMTp43ThWz7RF81HedHo31cil7MItZWTI/BC9yUa5CwBGv74llwkfO9XAU" + - "sTB7aOJ+Rz3vzfaYIENEf5Y2Jk9C4kVMw6rhBaOoDIl3Y6dhrprEyUgHP3aNlEio9yL+G5dFEZXVE9XBJlvbCC96PYpjR934dyNOsoiSopGpVrmGP/+2KJ0f" + - "KAe3O+78A+5qbFY0ptQC53IlWqNqN66ZhMdf0QfayEqoqpIf6OmTDelQEKG1RnDgXB5VqU+qSyKvjLFUBjAO9mPzulx2UsBG4Qhy2Ya2YPVnSnvmDXjol7Yh" + - "VCcRCHNVDsHN7p0P3Mhc8XOV8WNaMpSXObyY21KV8bkYsDXRRhlA710VeoO/qEJ4aX3jO78/6rgv1L9yFRDaQZfao4zQeCRPetnntoPGCW4yRsL29CopYTZx" + - "ToZqH4YxygyydxYUWHyg5yeFrvcCOgcfUivOAD/8cI9PD3aQB0GYEJHQFdFH1VYmLSmP/RbH8h46R5XGJfqu+lbn8X3mcHfz6zN/+vDXpdqrrHacET9g13fi" + - "H9DZZ7uKOXHkrAioqeW7wdiwKWEBqgWst6bPPQQuIUcSVjhX4/IFBagEdV6ZkKwcoCGwml9ji1SK9W9+z/U4x5wh2B4QFsftj8iosuueI7YUcfXETmaCCtl4" + - "zHapOshSi6P6WJOVnAZVGsZJChB+TbwpwjWgSMKfrg615dF2TZaQE8RZgNjlmtSQndBAB9yCWH96WNbG5ezjmRFricmmEcpG9pU860dRLKLFWjdVch350HC4" + - "Xiyermm5/Db3Foxhj7J6hr1cYTvZNo8JS/6QKzKn5grsr2ZLhV/DJioBj1NNRaqJklduTrwXF2m0Ltx8kzG3Nbc4abccCgBouTXbhaLHQddg4aFteI0KUgg7" + - "FbBmg+38qHX6HUAyrDNabmgvNUhZMAdFf0H8otVikpXgJXjYMy1ZFEU7rlFnutda1JqG/o2DLx0cYtyVOncp5A+drW3qjuvFL7Afv8ZidsFmidRlFIXhfy1w" + - "KrcBU8+MW54oSxYmo0CrlmvYWdPle6Xc3pye4YPIMaPs7zkqF2rAWs88VYnrxLCIxB5w9r0Px2clIkKm8ElY7KQaKvYhdkLhMDXymf2I2mKp/8sqkglYWPAH" + - "LdHEGouQmO/IbCOXoJUracIkp+M6ujKAR1kBP64Xv4RVX4p3Fp9QTBObvvT7CB1nt9Nq0qQe5lD20EHIjoykL3CMG3Dd4mexDaO7UVFIPtJuDCBJWjBMfqj4" + - "PQr4yKX0FE+htLhkMO5GXLJfL/4OK3jFQcSfYoDtuJcFlNQ4+4WxHTa9HVHO7W0cTIi5SKLsgxvqWOKYcvDjbOZr+q0wFo2Wpx6inBqoSdaLv8d8np9D7Amu" + - "HtuIiik3l2stwYbRueKmasPcUsBOEgQc8MkbZdKijEqFQyY+OpirTpM6ccuhlbIgiGWQIy3QUHh7d9QwcP/OKSj+gyyiESUiWNNJ5ar4K7T1bsR6sIavhO7V" + - "nbG9tkHccSTacmvFRs+oCX6Q5/qpASW7qrgGothMro9+l7gYp9eqXMvNfgYq8xBUd2zQi2aOILZDZyx0N2QLF7P2HdsPAamNJjRuPspFaTUQoI9jl3uXBqOK" + - "f+5h8iFmphvZ5R/Wi38UeBLypGJq3t+JRGnRTEIZdoZdQieKFt/zY+IpnGUJQeI67MkO/SX/hIe80Ua3iYPlRET6K0D+FEAbPP+VlLbJ4LBH4TeKK2KVTBAl" + - "MCpInIilug6ORjlyU2H7FD81DZaeEPBX0APJVKxrcqeTkk0yCD0RbRD4XXna9fT8hndm7DTF+uUTatA7kumkphJD8EJI5r4xn4ru0v0MN2I0kZmVGxx8KGiQ" + - "M5GAUjBie7FnzJmZVWEfGd2avhBZTrqzZj0xlCx4PfzM7qRgA5l7CoCSJeNbhBTxWZNXj+vkIUzRj6GGpbPjIoRIaoGK6C00Zk3PhNgJV0G8yX1WoVkuV7lk" + - "/PTvn+jy0CTVIoOuzVAcSsoccwMWR3OmO0YRGfIpzf1Jb8FH3ow9KGFuokGOGNFsBJT17mMiiIQsY8Q5AczZt/btfF1C699SplfZo/Nj0sGL5I6mh+3YdZxK" + - "n6T+Ia74MXCdIBZ9RxNxuKaLv/z5v3ESzq+/pu8m4nGH3vlwCv8NJ8HWZ1kWaVoBZFRMADfd8eJyrf2RO4v6+7dfvFIYbtDOpBy8ILLAiyMb48grejdCUN0d" + - "xVT8HvKvpqdgDRgmcgDbfZz/D69aZW6LwLD6OAmTxHcM+UDUYz92pU2kiuKuZ8naFKzRInS5FhtnXP0/I5GwgbxOrlOcRxuUz6Zos6KGG1tr16psIHVmy5IQ" + - "yM5oqPyqhMprWq/XF+oprwv0cUNf2/elAS1RxyYm8UfEosCp5A6yd2yScCeBRbVuzWFIJI5h+qFjiVQ5qUSq47Td6JVJbaQ9DiEFzsbLyg8Co+BBqSk+m7Bx" + - "CMI9zTb6DqkGopuoKynYXmkK/YZ5KBOZ1KQdH3KTh4iUhQfM+mZjHcYttLrcRKcNNrhShcDnLtnqlTYtvA1mkHZKd9oyLV6/G3FbdnCK436v+or4eqG/j6UD" + - "xkTavDndgNnl3Dm3RpQTsblcleCNpBwtW8lLfWzqFMrNpKMwFtP8aBD15kPm9m9tHdk8rZ4+eYK05+mTp9Uv9OMvnjytfvnkyeZyTXmVcvv2WImAv5XuNnXr" + - "lVSeDfBdW32kfyEruDmXDHa/l6Yf5BH7EaCYl79ePJ+o6T74g8sM1e7guSL26JzNLpUu6YbdUQ51F72MA5yIB4u2FuzwFE8nSyGZ1hP/urSrrReLFwr5K7J9" + - "z41VjpbHMQeT+ZosOG9C8vT8zdurXD9x5tbuzXQspRIOWMvZ9bUcpudv3tIVvckFoumO6+VSdLkz1VovyqU2ST/PuMHHRSXZevTLnsCstM+swA1zq27mAyds" + - "1HP+5rRd399ygIZ777RKaUkKKx9PrFR1ys/J4y0E9UGdaFypFUPWGSRVQN6rrXDn70escxzM1kTXIF3Rq2Nq1Wp4o2KydbADWjzp8SbspT8CMtVnZSqX2jMz" + - "rRJ7AX0ID328QfcEbsaJmu/fII/KvTeCGVoC0phbZDRZkBzjKeCUPopth5Ce+RgKPdOqviwl9De5WFUWdGbvB8qB+taFCTeNtDdLvqnDqQ/OaUb216ltL57B" + - "llgpKuF4/nGLqQD4BTqCzRZNfj5dzOnRJGMUBevUlzq1ysoQRhk5amJZ6ClNu42Ng4/qitIQqxQOPwqL0w7le9QSRAOQNtwgX2i3HMZmCYJVpaz042LQAbXw" + - "2OJ9F4SAWSlIaKmmJzZ3ZL8NbHLJWBpngc+Ie9IzulzS16ULNCqvkAoHLug5RmCxEVHoVu/I1BZskj7HcBJTopTzQeuOZ9UQ6RFsPGveMiDai1J1ektHpvg7" + - "gfeiWmGCRtudl8vzgjSSKt9pQ/sMApQ+wZAH40pP5XKMvDy1XP9k0fjZqb1bRjCUu79+Zj/3qZF8DVg+9ZGd6i6w6NT/iYZgXHKnxTtr0pIVFSU8ioy69anN" + - "3DffKrls7icRlw9G7JVj3J0AehbyxKPnIV2iUnEBUYJwuDIEZeuiZVV7rjVM5Y6saRCdxqoEnXkcejDUPP44ZCBi3wPYd74uwv7lPWh3pY3yEhXA4MXzf+KV" + - "FBgtv+dSB3m/Y3XqdxU4lZdqpobYUwts6WKVgfAORTgKZzPKSqZDX1roi4ag38HaJoqyoE34F2fOJt/nKV5A2BJlMXLeoZKenF7eAV6UkllpDNYUGmgAfnrC" + - "PSERk7SJyRa6XBU1afZ6Ewq2fF9PoWha0MhK7aOIepFM00C3E8CU0gbdWvPzOyG0LpEpa+mcmRdWtD++buUB4tClwI8TOK0ilz7llaBJEyQQIqyqDr7raMze" + - "8gPqjCp3z/q+zxq6T23nG+kr31xtzvrMN/NGdFjmZ3bKl9ryMh/b5Z0W+WeqJMpbYD591Cm/phd/dW+6+7l96ad+dHjY9MKNvoiXI6Z2OEjB/96m+vLCzZfB" + - "n7+0kMtJsy4K6bvQM/nXtFbMMHP2TlI+jtqtVDjN9OzSxqHL4aw55LdA6d1oGUeoY3OjQO0hzoJ7vGYRYOkNelAy+YjjwJCPl8trzb6mFC22Bmfji5cvEL/r" + - "G3q8Ybe3joWrVFI1ETyMV9GhQyPFq7qzf5KL132zuSzyE4Z4i70Hxw0aMU51jLmv4+yi+0vim7b7Y/fmPSGivdwwD/LyFCbBDc0mpvUk2Z/SRpmpkMQqIIBu" + - "6r2vocEM6d5XxhTG81twUdPg2Rude2gdLt73XlqOt0Je5ldNb7mdHr/Oe/Lgu2hlfnccjwK/G22QVyezQ8ubhFEJQQCK0oC+Ka0J53PyV7zPNqEann56my3q" + - "jLO2jNmdXl/Udz5EBCqhDEigLjvrksTysY9RC3BR3X/ac0WlqH0WRejJ739JyFHeV6H1Nov4mFLwI2BRrVT13HvIedfT61SKCNqTbbQJET03OEIWIR51Kuni" + - "436zojdvn7/9StZwebbt/pDft9TrqWfIMbaOZ0VCWcjifwAIzyTLrD4AAA==" + "H4sIAAAAAAACCp17244cSXLle36FqRoQWYmMLHJmpdUWGy2wr0MNu5tDctA7EIRJzwjLDO+KcA+6e1QyB8RCT/qAxX7Bftp8iXDM3CMj69Lq0UOzs+LiF3Oz" + + "Y2bHLKqqWjjT8zUlf8Ou2nuTFg3HOtghWe+u6eKPkWnLOx+YApvGuj0dWt8x7WzHkXygfeBhkOu24fVsJKp93xvXRHoaj/3WdysZYkWRa4yOH71xydYr8mPq" + + "rOMV9WbAY7u4oq2JbeXHNIzpkgKnMThyJgR/oNjZmiOZRJ0/6JRU+5jWtLmaLcCMjU0bCnxr+RDJUOQYrXe084F6GyM3FM2tdfu4vliYsB97dqlqrUvXdPGv" + + "8vq/0SfamsBk3a2vDRZOZhg6y5FSq1KhvUn8gp7IC08ojG4+WeAUfByw6Vsm4xri3iY88b2xLhnrONC3zM3W1Df0lQnNxeKWQ9QDeL7+X+vfXixM1/kDN1Xy" + + "vovXC6KKvjSxlR9v2TTy47vAg/7o/HZRVdXi87+rKvqOHQeTuKHtcX4+u+B7iqG+qo3zztam+3O8sV237psXxI1NZBP2GHTVYXTzt62LyXTdihzfcqDU2ihK" + + "saaq+mKx+Owz+qllvEF/jLxYvOwO5ohTVWUy8qxK76///v8otRhSRTp469JK5rRy8dAedXxZHq4YETkFg+XhZUeGEtetsx9GXtN7WbWN5DylNnBsfddgLYFN" + + "3crQuGPiDUX7F9bp+SP3Q4pk07Ws48PIUc77Qhcmy85KrW+oVkb6eYyJDrjyihxzcyHaiNlTC9NInjCVvgdpxgMHLG+Mo+m6Ix05rheL9y3nRVgPFQpcFB5v" + + "djbKGTJ0fr7zjncJU/w8NnuGBkPKtRkjFzkdbGqpsbEOjKFNOM7nETGl/OyaXurRjK7hQMGP+7Y70m+ePSNYaNSNH/0oG1UwWJXjlHsHE7NOWNfwR270KA3t" + + "2Y3WcXckP5gPI9PWylKwT4ilNpFx2Croxgauk2pIVoxg922i2nTderH4SY840WCOkXofRVVsgqrD0A4+3FyToc77AQtUFKDBxLgCbuGO28+stPIDO50omr4g" + + "XOCBYTvdUbch+hO4KuuqWxP23JDZG+uASLux62gItma9FiHZLY5GjEmVzqbI3U5EiIP/ge2+3foxZPH6A/QZ8gAyyravF4uKlsuN2QOi8HZV+34wAqWb5VKs" + + "SFQwjuHW3gIe6fTE6kxj5Dl2iUOk2rvEH2HPsrydDTHR0JlazYi8vNaabkd+d5KO7miti4rjVtfVBHvLrmr4ljs/QBfL0gzFwRycSIpdophMSJG2nXFqGIEr" + + "OUlgM7TnqKYDpNwGyzvyvU0J2nRobd0qCsgdPDYEv+24L/CiuGEEey3fmo6847LY7HU29PQEKk+iCD1b9+UkT3ai61DRyCbUsqadqM+y9q7mIS3vwFAnOthR" + + "TMG6/WqCthN2QHNX2eYCm+gFJnem60h8AOBC3Kk417WA6cxDL5bL72DVWGHerwjsBKmrAjHezWBMjvaaZmB231n/Mq7983JJr3ZAq5W4A5sAF2IKcEziI2+A" + + "Cja1fkx5FYKALSsUiSBure+MqqVIwZEXj7dvkypdeXTgILu6pq1JdcsNLHcwwXQddzJxpMbLIApoWBKA1EYdo+HaNhxpeWgZu16enICe4mnxCBi6bk1fdWZs" + + "mL7yDWeteCsihXdVDIB7pSHwjgO7mimMEgu57kiDxfm1TEscJ05zSVCTuahbE2nL7OS9BhYuaxFlnDlEPmYYNWNqfYCbijdWY60izvVi8W2eh+oOkB+pN0dC" + + "VAfFheEGxzHmnTjFxhKHQaEzrMHdS3Ah4Airiy13HbXcDRzimt7JnwLZiH3KSziP4PfB9OqwpqCv6EJqxYcocsgYmGYKG0RtGnbJ7iyHqKvpoXd588PABq4E" + + "WiLQ8STqQhG/uNSblKDoQe7PoyRxmFAn6JEJZh/M0FLDnd1KONSpmCRGQHhmd7bW9fiQXdO1RDlhFAitxLw6bxrAZmvCLUegbd3mcawTw1VlwkqwgOL0HR1M" + + "cDnQEGuFbSqi9hIKuBvnD269WHxzcs1PRYnhs8TkoBzqF7vj5XXG6471dbjr/3Py05h3ctOAFr8jm14AGu95aHrq+LCi0aVg6htgLAL7KWwUzQQiXL6YkPUx" + + "Z/40G7TtzZ4vX2SXF+D9oPeQtSQD1BrXdExPeb1fz1Czt021G51I/FLU23Yj0Lc1A6IPTzWQQJRbfKh1qymwugNi4sWiuOzrRQUXpPoHd6fKr5FRcYG7zuxl" + + "ip2VgKWsg7a+OdJf/+P/0kaO4EIg6Vo3crFZVDSYPcwSRzq9JEMj5rjCP38Oxu35FwcpCZYpIhq6UQKMWBwhQh8oH/aIRRvrcKvxdUmqyESKDHVPnPFRplQX" + + "+vicWHlbfu9wHjgzGVneL8PnEX6nj8oQeLXc38gSBbbzaCtswMLvYxxD2EvH8tj1vWFfrr682ED3TjfMum+ur19e0MVWfn15R1RD8DU3ozhcOV3NXqluub7B" + + "UDHxcGcP+sg6ttfXn9GXxjkOeVo9mQfvL6p7EYDK8cnpXPLp7jQTKoKmqsoPPDTIrwsjyg40dKGLWZYuS7vlYHdHKwFtfob7LTcqpVtjO7O1nU3HEjF8GDkc" + + "7w9cVfBrHTxxOYjpzsGEXoVfTWoj6bmGSB0uHqk2QxpDPu/NLInfXG0OvD390ddD+QOJ6auvz7aBdC701plusmpTJ82WDsEnph3SurKCu5NRpXD9+WBS+wVV" + + "lY5cdSam6hBsYuwEKREiD5Fb3XJv5FBKThATJOWD5BI7pF2NSWaLsLxEOPsRz57EKC9Weax/TWbb8b+pIPXEtkyfKxx+Md0WaFJY9+EobktkGyhwPYYo2Yzt" + + "mtqEhg6mkwDRB4swWv3i6Hamt501eGXwupjeDNA8TQEEpMauK7Z+a7pRthbHbQoseYYa/r+8+/GHqz+9/P711f/+/rW6l6e9cXbHMa2o8/UNrq3EZa6AQju7" + + "v9QZf47eVaJY+t4Ts96u6ycbuqLN0fTdY/c+PnwL2DKwIFxxKIL/YtmAw8b27CDwuMK59SaJ45KMXhYk71Q9J81Pz0YEUDC72Pq0osYiPOj19dq4HCWqVynp" + + "kU3zUdVj3Bv1zdffYpAfdztkgI2vhVOSF60TFkiWrsvVkHyl8YHMZ84ormsZbjM0O9nE5kp+ZrJso6/JpQ7UFK5k3yU0itmXdB1LEOcgRiXuQ17jj3D4MBZ5" + + "eKMxbGp95Bf0kw8NbRpff7w7o1wT5Y2bq/wXf0ybF/TGHzi8AX9Dm2FI996Ua7GzDWMv+MP5xHHzgr75WHNHm49d/FjFljnF8o5cqn039g7TyZ9wLuW3ONby" + + "h6jRZnHmItSVvfv9q9ev131z9dXvXv7w3Tevf/xu3Tf0VKysmrleoZgKmEm012T1lltV8SKfI+j7gp58nn3hF6rL+pAEDPqEIKY5VG+nKWYJxgQ1sLrXlIJx" + + "2X+d4ckdKU4wAx3JiHxyK7WRREkjWQVDUwcfldpQ9rZMmx2WOO3PRXayXkT8GYUMxR53ZQTxY7SDylgkyw5Zm3KiJYaSEXGoFDld7TldjQ6/gGqbxVc5O4Dr" + + "V9/4w8vvv9msHo6JVo/ELavH45HVo05yRZssRrXaq0284Y6TJMYd4/45Yq5+2ZWvaFO3UL2GqkrvyjUBxArxroDZ77/5Ey4/Bo1XM2CEDk+/yzt31ixPF3XQ" + + "58tfq1/rbVe02QtRU3Q50ucYvrLNF7h5FzWv7iHe6lFMuprD0dUcZPDSGZhcPYojqzvgcfUobuQ/yltz9Lj67wDH6o6Vi5YUg1blOrfIqzvGuCrmhF+PmcF6" + + "sXg3bislwkTDo6a8tQkhU92S+N1y2Jpke/CYcXrDupaDFXq5pJD2xOMpbR5z8B8Hzu5AWDstlRTubrGZsRIxGWAuuAdlF+Fm1/RtJ+TQ6OBvR4mUagAXQiLq" + + "2bgzamfLEkqBpuBGiat347ZwAovFJ3p1KqN8opdqxJ8Wn6qqkv8Wn85KOBulat0x05ClRkOfTjSR2XrUVbQmc079T5lQyRIkgAijW9PdmUqx6BO9e7Ru81i9" + + "JjNxnT9g3MVnn32mo2VhPjDcYvE2F6XkaPIjmlsX4sRqnWq2RD8k29sMuH4YfEijs6nQMQj9wKkmL6P202pXmos6DzkMwXJCpo5YFtJ98+rVOnOKEDHfgo6p" + + "GTF5Pjhl6bZHxKO+69Q01sulnMHMZ2Wa/hC8UFS58gFFvH7Al0k8cs7Bo3CG1YOH9zvqeW+2xwTqI/qzVDV5kiBeCDzsGlowCrOReDd26uaqiRCNdPBj10hZ" + + "hnovBQfjMhGjVH6iOthkaxuhRW9HUeyoB/9hhCULESq8nPKja+jzHwq7+omyc7ujzj/hrcZmFmVKLWCXK0mUlS9yzUR2/jN9oo3shKoq+YGeP9uQDgXiW+sS" + + "B84lWaUXpaIllM4YSzUC4+A8Nm/LYyfWbZQYQR7b0BZR/Rm7n+MGTPq1bQgVUTjCXAmEc7N75wM3slbcrjJ+TFsG2zOHF3NbKkE+FyC2JtooA+i7qxLe4C+q" + + "4F5a3/jO74867ivVr1x5BMPQpfYoIzQeyZM+9qXtwKsiNhkj4Xh6pbGwmjgPhmofhjHKCrJ2FhRYfKKXJ1aw9wI6Bx9SK8oAPfz0gE4PdpCJQEwIMekK0aQM" + + "L5OWscd+C7N8IJyjSv0S/VB9r+v4Mcdwd/PrM3369Lel2qvMdpwFfsCuH0Q/wO3PThVr4siZEVBRy7XB2LApbgGsBaS3pi89SDUJjsStcK4A5gcKUAnqvDEh" + + "WTGgIbCKX32LVKf1b/7I9TjHnCHYHhAWx+3PyKiy6p4jthSO1WInMYH5bDxWu1QeZKkFWZ3WZCanQWWIYUkBZLOJN4UsBxSJ+9PdoZ492q7JtHUCIQwQu1yT" + + "CrKTMNABt1AgOE2W+XixfcwZsZeYbBrBbGRdyat+EkUiWiB2U/XYkQ8Nh+vF4vmalsvvcz/DGPYo5WfYy1W9k2zzmJDkT7kKdGrowPlqtlTia8hEaedxquNI" + + "BVPyys0p7sVD6q1PXKBibmtuYWm3HAoAaIk3y4Wih6Grs/DgNrx6BSm+nYpms8F2ftTegB1AMqwzWm5oL3VP2TAHRX9B/MIPY5GV4CXisBdaJikselyjtvWg" + + "tKg1Df2Fgy9dIyLclSp3aR4YOlvb1B3Xi9/gPH6HzeyCzbSsyygKwf9O4FReA6aeCbfMKFuWSEaBViXXsLOmy+9Kib85zeGD0DGjnO85KpfQgLWGeqpM14kh" + + "EfE9iNn3PhxfFI8ImsIniWIn1lCxD74TDIepkc/sR9QzS8+B7CKZgI0Ff9CyUKyxCfH5jsw2cnFauXonkeRkrqMrA3iUMnBzvfgtpPpatLPohGKayPS130fw" + + "OLudVrAm9jC7sscMISsykr7AMW4Q6xY9i20Y3Y2SQvKTdmNAkKRFyuSHij+iaQC5lFrx5EqLSgbjbkQl+/Xif2AHbzgI+VMEsB33soGSGme9MLbDobcjKPTe" + + "xsGEmAszGn1wQx2LH9MY/Dhb+Zr+IBGLestT31JODVQk68U/YD0vzyH2BFdPbUSVlpvLtZZ9w+hcUVOVYW5jYCcJAgx80kZZtDCjUlWRhY8O4qrTxE7ccmil" + + "FInAMohJCzSUuL07qht4+OQUFP9RNtEIExGs6aRaVvQV3Ho3Yj/YwzcS7tWdsb22XtxRJNpya0VGL6gJfpB5/dT0klVVVANebEbXR79LXITTayWw5WY/A5W5" + + "C6o7Nuh/M0cEtkNnLHg3ZAsXs5Yh2w8BqY0mNG4+ykVpbxCgj2OX+6UGo4x/7pvyIeZIN7LLN9aL/ynwJMGTkqn5fKcgSgt14spwMuwSul+04J+niSd3likE" + + "8euQJzv0tPwTJnmnzXVTDJYTEenpQPCnANpg/jdSTicDY48S3yiuiFRygCiOUUHiFFiq6sA0islNxfST/9Q0WPpQEL8iPJBMxbomd1dpsEkGriei9QL3NU67" + + "nuZveGfGTlOs3z6jBv0qOZzUVGIIXgKSuW7Ml6Kn9HCEGzGa0MwaGxx8KGiQM5GA8jN8e5FnzJmZVWIfGd2avhJaTjrCZn04lCzieuiZ3UnBBjT35AAlS8ZV" + + "uBTRWZN3j+dkEqbox1BD0llx4UIktUAV9hYcs6ZnEthJrAJ/k3u7QrNcrnKZ+vk/PNPtoTGrRQZdm6EolJQ55gIsiuZMd4xCMmQrzT1R7xGPvBt7hIS5cQc5" + + "YkSDE1DWu/uBIBKyjBHnAWDOvrVX6NviWv+ecniVNTpPkw5eKHc0WmzHruNUejP1D1HF+8B1glj0Ok2BwzVd/PXf/z8s4fz5a/phCjzuhHc+nNx/w0mw9UWm" + + "RZpWABkVE8BNd7y4XGtP5s6i5v/9V28Uhhu0UGkMXhBZ4MWRjXHkFX0YQajujiIq/gj6V9NTRA0YJnJAtPs0/x9atcqxLRzD6n4SJonvGLJB1GM/dqU1pYqi" + + "rmfJ2uSs0ZZ0uRYZZ1z9Lz2RRAN5n1ynOPc2KJ9N3mZFDTe21k5ZOUDqzJYlIZCTUVf5TXGV17Rery9UU94W6OOGvrUfS9Nboo5NTKKP8EWBU8kd5OzYJImd" + + "BBZVujWHIZEohumHjsVT5aQSqY7TFqc3JrWR9jBCCpyFl5kfOEbBg1JTfDFh4xAk9jTb6DukGvBuwq6kYHsNU+j3zENZyMQm7fiQG0uEpCxxwKxXN9Zh3IKr" + + "y4172tSDJ5UIfOmSrd5oo8T7YAZp4XSnI9Pi9YcRr2UFpzju98qviK6X8PepdN2YSJt3pxewupw753aMYhGby1Vx3kjK0SaWvNTHpu6k3MA6SsRimp8NvN58" + + "yNxyru0qm+fV82fPkPY8f/a8+o3+/M2z59Vvnz3bXK4p71Je3x4rIfC30lGnar2SyrMBvmt7kfQvZAY355LB7vfSaIQ8Yj8CFPP214uXU2i6D/7gcoRqd9Bc" + + "IXt0zWaXSmd2w+4oRt1FL+MAJ+LBopUGJzz500lSSKbV4t+WFrn1YvFKIX9Ftu+5sRqj5XHMweR4TTacDyF5evnu/VWunzhza/dmMkuphAPWcnZ9Lcb08t17" + + "uqJ3uUA0vXG9XAovd8Za60O51Cbp51lscL+oJEePHt0TmJWWnRViw9wenOOBEzaqnb87HdePtxzA4T64rFJaksLK/YWVqk65nTy+fFAd1IXGlUoxZJ5BUgXk" + + "vdp+d/5Nxjr7wSxNdCrSFb05plalhq84JlkHO6CtlJ5uwl76I0BTfVGWcqk9M9MucRbghzDp0w26J/AyLGp+foNMlXtvBDO0BKQ+t9BosiEx48nhlD6KbQeX" + + "nuMxFHqmXX1dSujvcrGqbOhM3o+UA/VLDxNuGmmplnxTh1MdnIcZWV+nVsF4BlsipagBx8v7ba0C4BfoQjZbNBb6dDEPjyYaozBYp17YqT1XhjAakaMmlome" + + "0ijc2Dj4qKooTbgawuGmRHHaFf0AWwJvgKANL8gF7dDD2CxOsKo0Kr1fDDqgFh5bfGMDFzArBUlYqumJzV3g7wObXDKWZl3gM/ye9Kkul/Rt6TyNGldIhQMP" + + "9BwjsNgIKXSrb+TQFtEkfYnhxKdEKecjrDueVUOkL7HxrHnLAG8vTNXpyyBZ4h8F3gtrhQUabbFeLs8L0kiqfKdN9DMI0PAJgjwYV/o4l2Pk5anN+xeLxi9O" + + "LeUygqHc/fUre8hPzetrwPKpj+xUd4FEp55TNCHjkTtt5ZmTlqyoMOFRaNStT22OffOrksvmfhJR+WBEXtnH3XGgZy5PNHru0sUrFRUQJgjGlSEoSxdtstrn" + + "rW4qd2RNg+gyVsXpzP3Qo67m6X2XAY/9AGDfuVyI/csH0O5Km/PFKyCCF83/hc9gILT8bU0d5JuS1anHVuBUPuSZmnBPbbelc1YGwncb4Sgxm9GoZDL60rZf" + + "OAS9BmmbKMyCNv5fnCmbXM9LvACxJcxi5HxCJT05fTAEvCgls9KMrCk00ADx6Qn3JIiYqE0stoTLVWGTZp9UoWDLD/UUCqcFjqzUPgqpF8k0DXg7AUwpbdCt" + + "Nb++E0LrEjlkLZ0z88KK9uTXrUwgCl0K/LDAaRe59CmfIU2cICEgwq7q4LuOxqwtP6HOqHT3rNf8rIn81Oq+kV72zdXmrLd9M29+h2R+ZXd+qS0vs9ku77Tl" + + "v1AmUb488+led/6aXv3N/fDu1/bCn3rgoWHTRz768V/2mNrhIAX/Bxv5y0c+Xwd//qFELifNuiik70Jt8m9prZhh5uw7qGyO2q1UYppp7tLGodvhzDnkL0/p" + + "w2gZJtSxuVGg9iBnEXu8ZSFg6R16UHLwEceBQR8vl9eafU0pWmwNbOOr16/gv+sberpht7eOJVappGoieBivokOHRopXdWf/LA+v+2ZzWegnDPEeZ48YN6jH" + + "ONUx5roO20X3l/g3/cQApzfvCRHu5YZ5kA+2sAhuaLYwrSfJ+ZQ2yhwKia8CAuihPvjpG8SQHvxMTWE8f3kXNQ2efUW6B9fh4kPfwmV/K8HL/Knpy7rT9Ot8" + + "Jo9+/1bWd0fxKPCH0Qb5XDMrtHy9GDUgCEBRGtA3pTXhbCd/wzd0E6ph9tMXdFFXnLllrO70yaR+ZyIkUHFlQAJV2VmXJLaPc4xagIuq/tOZKypF7bMoRE/+" + + "5kxcjsZ9FVpvM4mPJQU/AhZVSlXPvQeddz19wqWIoD3ZRpsQ0XMDE7Jw8ahTSRcf95sVvXv/8v03sofLs2P3h/yNpz5PPYOOsXU8KxLKRhb/CdwbWWggPwAA" export const CANONICAL_SKILL_MD = gunzipSync(Buffer.from(CANONICAL_SKILL_GZIP_BASE64, 'base64')).toString('utf8') diff --git a/src/cli.ts b/src/cli.ts index d2159aef6..ec1f54c84 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -41,9 +41,9 @@ import { enqueueDirtyPathsSafe } from './hooks_index.js' // Loaded on demand inside cmdCompress, not at module scope: bash_runner pulls in the whole bash tool-filter registry (every language, linter, cloud and package-manager filter), which only the compress command ever uses. See the same reasoning for relay in cmdHook. import { runRead, runPrSlice } from './read_commands.js' import { runSymbol } from './read_symbol.js' -import { namedSpecsMergeable, readSpecsMergeable } from './read_spec.js' +import { extraArgsRejoinSpacedPath, namedSpecsMergeable, readSpecsMergeable } from './read_spec.js' import { runBrief } from './read_brief.js' -import { runSection } from './read_section.js' +import { runSection, runSectionSpecs } from './read_section.js' import { runListSections } from './read_inspect.js' import { runSkeleton, runOutline, type SkeletonOptions } from './read_outline.js' import { runSemantic, runSemanticMulti } from './read_semantic.js' @@ -1027,7 +1027,7 @@ export function buildProgram(): Command { program .command('section [more...]') .description( - 'read one section from a file (spec: file::heading, or file:: — e.g. "Lesson 16" resolves a longer unique heading; comma-separated file::A,B for a merged multi-heading view), or list all sections with --list', + 'read one section from a file (spec: file::heading, or file:: — e.g. "Lesson 16" resolves a longer unique heading; comma-separated file::A,B for a merged multi-heading view, as several specs a.md::A b.md::B also give), or list all sections with --list', ) .option('-j, --json', 'output as JSON') .option('--list', 'list all section headings in the file instead of reading one') @@ -1043,22 +1043,16 @@ export function buildProgram(): Command { emitExtraFileArgsNote('section --list', spec, more, { mergeable: false }) return runListSections({ file: spec, ...(opts.json === true ? { json: true } : {}), ...(opts.grep !== undefined ? { grep: opts.grep } : {}) }) }) - : runExitText(() => - noteExtraFileArgs( - 'section', - spec, - more, - () => - runSection({ - spec, - ...(opts.json === true ? { json: true } : {}), - ...(maxLines !== undefined ? { maxLines } : {}), - ...(opts.grep !== undefined ? { grep: opts.grep } : {}), - }), - { noun: 'spec', mergeable: namedSpecsMergeable([spec, ...more]) }, - ), - { echoSuggestion: true }, - ) + : runExitText(() => { + const sectionOpts = { + ...(opts.json === true ? { json: true } : {}), + ...(maxLines !== undefined ? { maxLines } : {}), + ...(opts.grep !== undefined ? { grep: opts.grep } : {}), + } + // Words a shell split out of one unquoted heading are not several specs: that shape keeps the note that says how to quote it. + if (more.length > 0 && !extraArgsRejoinSpacedPath(spec, more, 'spec')) return runSectionSpecs([spec, ...more], sectionOpts) + return noteExtraFileArgs('section', spec, more, () => runSection({ spec, ...sectionOpts }), { noun: 'spec', mergeable: namedSpecsMergeable([spec, ...more]) }) + }, { echoSuggestion: true }) }) program diff --git a/src/read_section.ts b/src/read_section.ts index 7db9cf3f0..83cdaf0a6 100644 --- a/src/read_section.ts +++ b/src/read_section.ts @@ -6,7 +6,7 @@ import { eachUnfencedLine } from './markdown_lines.js' import { detectLanguage } from './parser_types.js' import { displaySafeJson } from './paths.js' import { guardText, healStaleIndex, readFileText, recordReadStat, resolveAgainstProjectRoot, sumFileSizes } from './read_commands.js' -import { stripHtmlIdSpelling, parseCrossFileMultiSpec } from './read_spec.js' +import { namedSpecsMergeable, stripHtmlIdSpelling, parseCrossFileMultiSpec } from './read_spec.js' import { didYouMean, filterSimilarHeadings } from './read_suggest.js' import { listSections, readSection, type SectionResult } from './section_reader.js' import { compileGrepMatcher, countNoun } from './util.js' @@ -323,3 +323,25 @@ export function runSectionCrossFile(pairs: { file: string; symbol: string }[], o } return { text, code: anyFound ? 0 : 1 } } + +/** `section a b ...`, several positional specs in one call. Only the first used to be read, behind a note that the rest were ignored, so agents spawned one `section` per heading (14 spawns across three recorded commands, about 1-2 s each). Specs that each name a heading are served exactly as their comma form `a,b` is, by {@link runSectionCrossFile}, but paired from the specs themselves rather than from a comma-joined string, so a heading holding a comma stays one heading. Otherwise each spec runs in turn, one block per spec, and the call succeeds when any of them does, as the comma form's does. */ +export function runSectionSpecs(specs: string[], opts: Omit): { text: string; code: number } { + const joined = specs.join(',') + if (namedSpecsMergeable(specs)) { + const pairs = specs.map((spec) => { + const idx = findSectionSeparator(spec, opts.projectRoot) + return { file: spec.slice(0, idx), symbol: spec.slice(idx + 2) } + }) + return runSectionCrossFile(pairs, { ...opts, spec: joined }) + } + let anyFound = false + const jsonOut: Record = {} + const textBlocks: string[] = [] + for (const spec of specs) { + const sub = runSection({ ...opts, spec }) + if (sub.code === 0) anyFound = true + if (opts.json === true) jsonOut[spec] = sub.code === 0 ? (JSON.parse(sub.text) as unknown) : { error: sub.text } + else textBlocks.push(`${spec}:\n${sub.text}`) + } + return { text: opts.json === true ? displaySafeJson(jsonOut) : textBlocks.join('\n\n'), code: anyFound ? 0 : 1 } +} diff --git a/src/read_spec.ts b/src/read_spec.ts index 1aed3c03b..403b1328d 100644 --- a/src/read_spec.ts +++ b/src/read_spec.ts @@ -77,6 +77,12 @@ export function specFilePart(spec: string): string { return colonIdx === -1 ? spec : spec.slice(0, colonIdx) } +/** Whether `first` and `extras`, rejoined with the spaces a shell split them at, name one real file, or for a spec one whose file part is real. An unquoted path or heading holding spaces reaches the CLI split into words, and when the rejoined words name a file, that one argument is what was meant rather than a list of its fragments. */ +export function extraArgsRejoinSpacedPath(first: string, extras: readonly string[], noun: 'file' | 'spec'): boolean { + const spaced = [first, ...extras].join(' ') + return fileExists(expandSpecPath(noun === 'spec' ? specFilePart(spaced) : spaced)) +} + export function extraFileArgsNote( command: string, first: string, @@ -85,9 +91,8 @@ export function extraFileArgsNote( ): string { const noun = opts.noun ?? 'file' const head = `Note: ${countNoun(extras.length, `extra ${noun} argument`)} ignored (${extras.join(', ')}).` - // An unquoted path holding spaces reaches the CLI split into words; when the words rejoined with spaces name a real file, that path is what was meant, so suggest it quoted rather than a comma list of its fragments. const spaced = [first, ...extras].join(' ') - if (fileExists(expandSpecPath(noun === 'spec' ? specFilePart(spaced) : spaced))) return `${head} Together they make one ${noun === 'spec' ? 'spec' : 'path'} holding spaces, which a shell splits unless it is quoted: token-goat ${command} ${quotedArg(spaced)}` + if (extraArgsRejoinSpacedPath(first, extras, noun)) return `${head} Together they make one ${noun === 'spec' ? 'spec' : 'path'} holding spaces, which a shell splits unless it is quoted: token-goat ${command} ${quotedArg(spaced)}` if (opts.mergeable === false) return `${head} Run ${command} once per ${noun}.` return `${head} ${command} reads one ${noun}, or a comma-separated list: token-goat ${command} ${quotedArg([first, ...extras].join(','))}` } diff --git a/tests/cli_extra_positional_args.test.ts b/tests/cli_extra_positional_args.test.ts index 221d25031..f0f21f99d 100644 --- a/tests/cli_extra_positional_args.test.ts +++ b/tests/cli_extra_positional_args.test.ts @@ -88,7 +88,6 @@ describe('extra positional arguments are reported, never dropped in silence', () { name: 'imports', argv: () => ['imports', fileA, fileB], noun: 'file', dropped: () => fileB, mergedSuggestion: true }, { name: 'read', argv: () => ['read', `${fileA}::alpha`, `${fileB}::beta`], noun: 'spec', dropped: () => `${fileB}::beta`, mergedSuggestion: true }, { name: 'brief', argv: () => ['brief', `${fileA}::alpha`, `${fileB}::beta`], noun: 'spec', dropped: () => `${fileB}::beta`, mergedSuggestion: true }, - { name: 'section', argv: () => ['section', `${docA}::Heading A`, `${docB}::Heading B`], noun: 'spec', dropped: () => `${docB}::Heading B`, mergedSuggestion: true }, { name: 'refs', argv: () => ['refs', `${fileA}::alpha`, `${fileB}::beta`], noun: 'spec', dropped: () => `${fileB}::beta`, mergedSuggestion: true }, { name: 'symbol', argv: () => ['symbol', 'alpha', 'beta'], noun: 'spec', dropped: () => 'beta', mergedSuggestion: false }, { name: 'section --list', argv: () => ['section', '--list', docA, docB], noun: 'file', dropped: () => docB, mergedSuggestion: false }, @@ -138,12 +137,11 @@ describe('extra positional arguments are reported, never dropped in silence', () expect(named).toEqual([`${fileA}::alpha,${fileB}::beta`]) }) - it('runs every comma form brief, refs and section name', async () => { + it('runs every comma form brief and refs name', async () => { const named: string[] = [] for (const [command, first, second] of [ ['brief', `${fileA}::alpha`, `${fileB}::beta`], ['refs', `${refA}::gamma`, `${refB}::delta`], - ['section', `${docA}::Heading A`, `${docB}::Heading B`], ] as const) { // Positive control: without this, an unindexed fixture would exit 1 for every shape and the assertion below would hold for a reason that has nothing to do with the note. await runCli([command, first]) @@ -156,24 +154,40 @@ describe('extra positional arguments are reported, never dropped in silence', () await runCli([command, suggested![1]!]) expect(lastExitCode, `${command} note named a spec that does not run: ${suggested![1]!}`).toBe(0) } - expect(named).toHaveLength(3) + expect(named).toHaveLength(2) }) - it('names no comma form for the shapes brief, refs and section do not merge', async () => { + it('names no comma form for the shapes brief and refs do not merge', async () => { for (const [command, first, second] of [ ['brief', `${fileA}:1`, `${fileB}:1`], ['refs', `${refA}:1`, `${refB}:1`], - ['section', docA, docB], ] as const) { const output = await runCli([command, first, second]) expect(output).toContain(`Run ${command} once per spec.`) expect(output).not.toContain(`token-goat ${command} "`) - // Why it cannot be named: none of the three reaches a multi-spec path for these shapes. + // Why it cannot be named: neither reaches a multi-spec path for these shapes. await runCli([command, `${first},${second}`]) expect(lastExitCode, `${command} unexpectedly served ${first},${second}`).toBe(1) } }) + // section is the one spec command that no longer notes its extra specs: it reads them all. Agents spawned one `section` per heading because only the first was read. + it('section reads every spec it is given, as its comma form does, with no note', async () => { + const output = await runCli(['section', `${docA}::Heading A`, `${docB}::Heading B`]) + expect(lastExitCode).toBe(0) + expect(output).toContain('body a') + expect(output).toContain('body b') + expect(output).not.toContain(' ignored (') + }) + + it('section runs specs that do not merge one after another instead of noting them', async () => { + const output = await runCli(['section', docA, `${docB}::Heading B`]) + expect(lastExitCode).toBe(0) + expect(output).toContain('Invalid section spec') + expect(output).toContain('body b') + expect(output).not.toContain('Run section once per spec.') + }) + it('names no comma form for line specs, which read does not merge', async () => { const output = await runCli(['read', `${fileA}:1`, `${fileB}:1`]) expect(output).toContain('Run read once per spec.') diff --git a/tests/helpers/matrix_cases.ts b/tests/helpers/matrix_cases.ts index 5007eb1c9..349c32444 100644 --- a/tests/helpers/matrix_cases.ts +++ b/tests/helpers/matrix_cases.ts @@ -325,6 +325,12 @@ export const cases: Record void | Promise> = { expect(grepMiss.status, grepMiss.stderr).toBe(0) expect(grepMiss.stdout).toContain('filtered out by --grep zzznomatch') expect(grepMiss.stdout).not.toContain('Run npm install') + // Several specs are all read, as their comma form is, rather than all but the first being noted as ignored. + const twoSpecs = run(['section', 'README.md::Fixture', 'README.md::Install']) + expect(twoSpecs.status, twoSpecs.stderr).toBe(0) + expect(twoSpecs.stdout).toContain('# Fixture — ') + expect(twoSpecs.stdout).toContain('already included in section') + expect(twoSpecs.stdout + twoSpecs.stderr).not.toContain('ignored') }, // Deliberately a keyword smoke test, not a proof of real embedding-vector search: this shared fixture is indexed with embeddings disabled (isolate-home.ts sets TOKEN_GOAT_EMBEDDINGS_ENABLED=false for the whole suite, and tgEnv inherits it), so this case only exercises the FTS keyword fallback and would pass identically whether or not real semantic search is wired up - it is not a substitute for that proof. The dedicated proof - a meaning-only natural-language query finding a symbol whose name/body never uses the query's words, with a control run showing the same query genuinely misses under FTS alone - lives in tests/semantic_embeddings_e2e.test.ts. semantic: () => { diff --git a/tests/section_multiple_specs.test.ts b/tests/section_multiple_specs.test.ts new file mode 100644 index 000000000..2f57787ac --- /dev/null +++ b/tests/section_multiple_specs.test.ts @@ -0,0 +1,67 @@ +/** Regression: `section a b` read only the first spec and printed "Note: 1 extra spec argument ignored ...", so agents spawned one `section` per heading (14 spawns across three recorded commands, about 1-2 s each). Several positional specs are now served in one call the way their comma form is. Runs the built bundle, as the agents did. */ +import { spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' + +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { BUNDLE, tgIsolatedEnv } from './helpers/bundle.js' + +let root: string +let env: NodeJS.ProcessEnv + +function run(args: string[]): { status: number | null; stdout: string; stderr: string } { + const r = spawnSync(process.execPath, [BUNDLE, ...args], { cwd: root, env, encoding: 'utf8', timeout: 60000 }) + return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' } +} + +beforeAll(() => { + root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), 'tg-section-specs-'))) + fs.writeFileSync(path.join(root, 'f.md'), '# Guide\n\n## Alpha\n\nalpha body\n\n## Beta\n\nbeta body\n\n## Gamma, Delta\n\ncomma body\n') + fs.writeFileSync(path.join(root, 'g.md'), '# Other\n\n## Omega\n\nomega body\n') + env = tgIsolatedEnv(path.join(root, 'home'), { TOKEN_GOAT_HOME: path.join(root, 'data'), TOKEN_GOAT_EMBEDDINGS_ENABLED: 'false' }) +}) + +afterAll(() => { + fs.rmSync(root, { recursive: true, force: true }) +}) + +describe('section with several positional specs', () => { + it('reads every heading of one file, as the comma form does, with no ignored-argument note', () => { + const r = run(['section', 'f.md::Alpha', 'f.md::Beta']) + expect(r.status, r.stderr).toBe(0) + expect(r.stdout).toContain('# Alpha — ') + expect(r.stdout).toContain('alpha body') + expect(r.stdout).toContain('# Beta — ') + expect(r.stdout).toContain('beta body') + expect(r.stdout + r.stderr).not.toContain('ignored') + expect(r.stdout).toBe(run(['section', 'f.md::Alpha,f.md::Beta']).stdout) + }) + + it('reads headings across files', () => { + const r = run(['section', 'f.md::Alpha', 'g.md::Omega']) + expect(r.status, r.stderr).toBe(0) + expect(r.stdout).toContain('alpha body') + expect(r.stdout).toContain('omega body') + }) + + it('keeps a heading that holds a comma whole, which a comma-joined spec would split', () => { + const r = run(['section', 'f.md::Gamma, Delta', 'f.md::Beta']) + expect(r.status, r.stderr).toBe(0) + expect(r.stdout).toContain('comma body') + expect(r.stdout).toContain('beta body') + }) + + it('runs specs that cannot merge one after another and still serves the valid one', () => { + const r = run(['section', 'nonsense', 'f.md::Beta']) + expect(r.status, r.stderr).toBe(0) + expect(r.stdout).toContain('Invalid section spec') + expect(r.stdout).toContain('beta body') + }) + + it('keeps the quoting note for a heading a shell split into words', () => { + const r = run(['section', 'f.md::Gamma,', 'Delta']) + expect(r.stdout + r.stderr).toContain('which a shell splits unless it is quoted') + }) +}) From 709f7e6e5e38d542c4b43d84d553752e8195bbca Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 05:41:41 -0500 Subject: [PATCH 09/92] fix(skills): print a heading tree from skill-body --compact when there is no usable slice skill-body --compact printed the full body without a word whenever a skill had no compact slice: humanizer's marker cuts 97.6% of its body, and marketing and token-goat have no marker, so they came back byte-identical. The Skill reload deny advertised --compact for every skill regardless, and one session spent six calls loading one skill. usableCompactSlice, in skill_cache.ts, returns the marker's slice only when it is at most half the body, the ratio the hook's cold-load inline gate already required. Without one, skill-body --compact prints the heading tree the hook inlines in the same case (planHeadingTree), names a skill-section command for a real heading, and says on stderr that no slice was served; a skill with too few headings to map gets its body back, said as such. The full body is unchanged without --compact. The reload deny, the heading-tree deny and the bare pointer deny now name skill-body --compact only for a usable slice, so no deny points at a command that would not serve what it promises. tests/hooks_skill.test.ts pins those texts; fixtures meant to be size-gated slices are lengthened so they stay under half their bodies. tests/skill_body_compact_no_slice.test.ts and the skill-body matrix case run the built bundle; the hint-template guard lists the heading argument the fallback's skill-section command carries as a reviewed safe value; docs/cli.md describes the fallback. --- docs/cli.md | 4 +- src/cli_skills.ts | 40 ++++++++-- src/hooks_skill.ts | 25 +++--- src/skill_cache.ts | 9 ++- ...templates_survive_suggestion_guard.test.ts | 1 + tests/helpers/matrix_cases.ts | 10 +++ tests/hooks_skill.test.ts | 42 ++++++++-- tests/skill_body_compact_no_slice.test.ts | 78 +++++++++++++++++++ 8 files changed, 184 insertions(+), 25 deletions(-) create mode 100644 tests/skill_body_compact_no_slice.test.ts diff --git a/docs/cli.md b/docs/cli.md index d3dadf820..09351d04c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -112,7 +112,7 @@ token-goat pdf-extract manual.pdf --pages 12-15 --layout --head 120 | `token-goat web-history` | List cached WebFetch responses (newest first) with their IDs, byte sizes, status codes, and URL previews. | | `token-goat mcp-output [id]` | Retrieve a cached MCP tool result by ID (or an on-disk tool spill file via `--file `). Supports `--json-query ''` for narrow dot-path and bracket slicing on large Jira, Confluence, or JSON responses (e.g. `issues[*].key`, `total`), avoiding whole-file re-reads; a query that matches nothing exits 1 with the reason, as `json-query` does. Also supports `--head `, `--tail `, `--grep `, `--section `, `--json`, and `--full`. | | `token-goat mcp-history` | List cached MCP tool result entries (newest first) with their IDs and byte sizes — same role as `bash-history`/`web-history` for the `mcp-output` cache. | -| `token-goat skill-body ` | Retrieve a cached Skill body by name without re-invoking the skill (which would replay side effects). Prints the full body; `-c`/`--compact` prints the compact slice instead. No head/tail/grep slicers. | +| `token-goat skill-body ` | Retrieve a cached Skill body by name without re-invoking the skill (which would replay side effects). Prints the full body; `-c`/`--compact` prints the compact slice instead. A skill with no `` marker, or one whose marker cuts more than half the body, has no compact slice: `--compact` then prints its heading tree and the `skill-section` command that loads one section, with a note on stderr (a skill with too few headings to map gets its body, said as such). No head/tail/grep slicers. | | `token-goat skill-history` | List cached Skill bodies (newest first) with their IDs, byte sizes, truncation status, and skill names. | | `token-goat skill-compact [name]` | Cache the compact slice for a skill so later `skill-body --compact` calls are instant, and print a confirmation. Resolves an installed skill by name (falling back to `~/.claude/skills//SKILL.md` when it was never loaded this session), or pass `--path ` to compact a skill straight from a file without name resolution. | | `token-goat skill-compact --all` | Batch-regenerate stale or missing compacts for every skill cached in the current session. Skips skills whose compact is already fresh (source SHA matches). The summary also counts skills that have no `COMPACT_END` marker (with a pointer to `token-goat skill-size` for per-skill recommendations) and skills whose source file no longer resolves, so the pass reports every skill it visited. Run after updating any skill file on disk. | @@ -193,7 +193,7 @@ Missed lookups recover surgically: `read` and `section` print a "Did you mean… When Claude Code invokes a skill, it re-injects the full skill body on every subsequent turn. A large skill file (e.g. a 10k-token `/improve` or `/ralph`) can cost 40–65k tokens per session across 6 active skills. The `` marker solves this: place it in any skill file to split it into a compact form (above the marker, ~400 tokens) and a reference section (below). Token-goat detects the marker the first time the skill fires, caches only the compact slice, and injects that from then on — labeled `--- compact form (N tokens) ---` so the model knows to request the full body only when it needs the detail. -To add the marker to a skill, open the file and insert `` on its own line where the "quick reference ends and the detail begins" — typically after the quick-start table and before step-by-step instructions. The full reference section is still reachable via `token-goat skill-section "::"` or `token-goat skill-body ` when needed. +To add the marker to a skill, open the file and insert `` on its own line where the "quick reference ends and the detail begins" — typically after the quick-start table and before step-by-step instructions. A marker that leaves more than half the body above it is not treated as a compact slice: `skill-body --compact` and the Skill hook's denies then point at the heading tree and `skill-section` instead. The full reference section is still reachable via `token-goat skill-section "::"` or `token-goat skill-body ` when needed. **Re-load and direct-read protection.** Even without the marker, token-goat protects against the two other ways large skills burn context in a long session: diff --git a/src/cli_skills.ts b/src/cli_skills.ts index 60acddd27..13dde0b24 100644 --- a/src/cli_skills.ts +++ b/src/cli_skills.ts @@ -2,6 +2,9 @@ import * as fs from 'node:fs' import * as path from 'node:path' import { CliError, out } from './cli.js' +import { formatCommandError } from './command_error.js' +import { emitErr } from './emit.js' +import { extractMarkdownHeadings } from './hints/markdown_hints.js' import { buildLineDiff } from './hooks_read_slice.js' import { displaySafeJson, displaySafeText } from './paths.js' import { findSpecSeparator } from './read_commands.js' @@ -9,6 +12,7 @@ import { didYouMeanLines, filterSimilarHeadings } from './read_suggest.js' import { listSections } from './section_reader.js' import { getSessionId } from './session.js' import { + COMPACT_END_MARKER, contentHash, extractCompactFromMarker, extractNamedSection, @@ -20,10 +24,11 @@ import { skillOutputsDir, storeCompact, storeOutput, + usableCompactSlice, } from './skill_cache.js' import { formatLocalTimestamp, recordStat, savedTokensFromBytes } from './stats.js' import { decodeSource, extractErrorMessage, stripLower } from './util.js' -import { echoedValue, quotedArg } from './hint_suggestion_guard.js' +import { canQuoteArg, echoedValue, nameSubject, quotedArg, sentenceStart } from './hint_suggestion_guard.js' export async function cmdSkillBody(name: string, opts: { compact?: boolean }): Promise { const filePath = await getSkillFilePath(name) @@ -33,16 +38,41 @@ export async function cmdSkillBody(name: string, opts: { compact?: boolean }): P const body = decodeSource(fs.readFileSync(filePath)) if (opts.compact === true) { - const emitted = extractCompactFromMarker(body) ?? body - out(emitted) - const bytesSaved = Buffer.byteLength(body, 'utf8') - Buffer.byteLength(emitted, 'utf8') - if (bytesSaved > 0) recordStat('skill_body:compact', bytesSaved, savedTokensFromBytes(bytesSaved), undefined, name) + const compact = usableCompactSlice(body) + if (compact !== null) { + out(compact) + const bytesSaved = Buffer.byteLength(body, 'utf8') - Buffer.byteLength(compact, 'utf8') + if (bytesSaved > 0) recordStat('skill_body:compact', bytesSaved, savedTokensFromBytes(bytesSaved), undefined, name) + } else { + out(await compactStandIn(name, filePath, body)) + } } else { out(body) } await incrementSkillHit(name) } +/** What `skill-body --compact` prints for a skill with no usable compact slice ({@link usableCompactSlice}). It used to print the full body without a word that the flag had done nothing: humanizer's marker cuts 97.6% of it, and marketing and token-goat have no marker at all, so one session spent six calls loading one skill. Prints the heading tree the Skill hook inlines in the same case, with how to load one section, and says on stderr that no slice was served; a skill too flat to map gets its body back, said as such. The hook module is imported only here, so `skill-body` does not load it on every run. */ +async function compactStandIn(name: string, filePath: string, body: string): Promise { + const bodyBytes = Buffer.byteLength(body, 'utf8') + const marked = extractCompactFromMarker(body) + const why = + marked === null + ? `no compact slice (no ${COMPACT_END_MARKER} marker; ${bodyBytes} bytes)` + : `no compact slice under half its body (the part before ${COMPACT_END_MARKER} is ${Buffer.byteLength(marked, 'utf8')} of its ${bodyBytes} bytes)` + const subject = sentenceStart(nameSubject('skill', name)) + const { planHeadingTree } = await import('./hooks_skill.js') + const tree = planHeadingTree(body, bodyBytes, name, filePath) + if (tree === null) { + emitErr(formatCommandError(`${subject} has ${why} and too few headings to map, so --compact printed the full body.`)) + return body + } + emitErr(formatCommandError(`${subject} has ${why}; --compact printed its heading tree instead of the body.`)) + const firstHeading = extractMarkdownHeadings(body).find((h) => h.level > 1)?.text + const headingArg = firstHeading !== undefined && displaySafeText(firstHeading) === firstHeading && canQuoteArg(firstHeading) ? quotedArg(firstHeading) : '""' + return `${subject} has ${why}; ${tree.phrase} Load one section with \`token-goat skill-section ${quotedArg(name)} ${headingArg}\`, or drop --compact for the full body.\n\n${tree.sectionsList}` +} + export async function cmdSkillCompact(name: string | undefined, opts: { path?: string; all?: boolean }): Promise { const sessionId = getSessionId() diff --git a/src/hooks_skill.ts b/src/hooks_skill.ts index 1162c5d22..e9c06d30e 100644 --- a/src/hooks_skill.ts +++ b/src/hooks_skill.ts @@ -14,6 +14,7 @@ import { hasSessionOutput, sessionOutputBodyBytes, extractCompactFromMarker, + usableCompactSlice, } from './skill_cache.js'; import { recordSkillVersionSnapshot } from './skill_version_drift.js'; import { PER_FILE_COUNTERFACTUAL_CEILING } from './util.js'; @@ -84,7 +85,7 @@ function resolveSkillContext(event: HookEvent): { skillName: string } | null { } /** Heading-tree stand-in for an oversized skill body, or null when the body is not worth mapping. Two callers, one floor pair. Both floors come verbatim from fold_structure.ts's large-markdown outline, which measured them against real session transcripts: a skill needs at least OUTLINE_MIN_HEADINGS of structure to be worth a map at all, and the map has to be genuinely small next to the body (OUTLINE_MAX_REPLACEMENT_RATIO) or the round trip costs more than it saves. */ -function planHeadingTree(body: string, bodyBytes: number, skillName: string, sourcePath: string): { sectionsList: string; treeBytes: number; phrase: string } | null { +export function planHeadingTree(body: string, bodyBytes: number, skillName: string, sourcePath: string): { sectionsList: string; treeBytes: number; phrase: string } | null { const headings = extractMarkdownHeadings(body); if (headings.length < OUTLINE_MIN_HEADINGS) return null; const { sectionsList } = formatHeadingTreeParts(headings, skillName); @@ -121,11 +122,14 @@ export async function preSkillHandler(event: HookEvent): Promise { const cachedBytes = await sessionOutputBodyBytes(event.sessionId, skillName); const denyCredit = cachedBytes !== null ? Math.min(cachedBytes, PER_FILE_COUNTERFACTUAL_CEILING) : 0; recordStat('session_hint', denyCredit, savedTokensFromBytes(denyCredit), undefined, 'skill-reload-deny'); + const reloadPath = await harnessSkillPath(event, skillName); + // Named only when the skill has a slice `skill-body --compact` will serve: it used to be offered for every skill, and for one with no marker or a marker near the end the command handed back the whole body or nearly all of it. + const compactUsable = reloadPath !== null && usableCompactSlice(await readFile(reloadPath, 'utf-8').catch(() => '')) !== null; return denyOutput( skillSubject + ' was already loaded this session and is cached. Use `token-goat skill-section ' + - nameArg + ' ' + skillSectionArg(await harnessSkillPath(event, skillName)) + '` to recall a section, `token-goat skill-body ' + - nameArg + ' --compact` to recall the compact slice, or `token-goat skill-body ' + nameArg + - '` for the full body instead of re-loading it.', + nameArg + ' ' + skillSectionArg(reloadPath) + '` to recall a section, ' + + (compactUsable ? '`token-goat skill-body ' + nameArg + ' --compact` to recall the compact slice, ' : '') + + 'or `token-goat skill-body ' + nameArg + '` for the full body instead of re-loading it.', ); } @@ -137,6 +141,7 @@ export async function preSkillHandler(event: HookEvent): Promise { const bodyBytes = Buffer.byteLength(body, 'utf-8'); if (bodyBytes > OVERSIZED_FIRST_LOAD_THRESHOLD_BYTES) { const compact = extractCompactFromMarker(body); + const usable = usableCompactSlice(body); if (compact !== null) { // The compact slice is already in hand here, so denying with a pointer to `skill-body --compact` costs an extra agent turn (reasoning + Bash call + that command's echo) to re-fetch content this handler just computed. Inline it instead whenever it is genuinely smaller than the body and under COMPACT_INLINE_MAX_BYTES. This used to reuse the oversize gate as the inline bound, which put every slice between 6 KB and the body's size on the deny path -- the agent paid the round trip and received exactly the same bytes anyway, which is why that branch's stat shows real traffic and zero savings. With no marker, a slice no smaller than the body, or a read failure, the original pointer deny stands verbatim. const compactBytes = Buffer.byteLength(compact, 'utf-8'); @@ -163,19 +168,19 @@ export async function preSkillHandler(event: HookEvent): Promise { skillSubject + ' is large (' + bodyBytes + ' bytes)' + (compact !== null ? ', and its compact slice is too large to inline; ' : ' with no compact slice; ') + tree.phrase + ' Use `token-goat skill-section ' + nameArg + ' ' + skillSectionArg(sourcePath, body) + '` to load a specific section' + - (compact !== null ? ', `token-goat skill-body ' + nameArg + ' --compact` to load the compact slice' : '') + + (usable !== null ? ', `token-goat skill-body ' + nameArg + ' --compact` to load the compact slice' : '') + ', or `token-goat skill-body ' + nameArg + '` for the full body.\n\n' + fenceUntrustedFileContent(tree.sectionsList), ); } if (compact !== null) { - // Too little structure to map and a slice too large to inline: the pointer is all that is left, and it is still better than the whole body. Without a marker this falls through to the normal load instead, exactly as before -- there is nothing to point at. + // Too little structure to map and a slice too large to inline: the pointer is all that is left, and it is still better than the whole body. Without a marker this falls through to the normal load instead, exactly as before -- there is nothing to point at. A marker cutting more than half the body keeps the pointer but loses the --compact half of it, since that command would not serve the slice. recordStat('skill_oversized_first_load'); return denyOutput( - skillSubject + ' is large (' + bodyBytes + - ' bytes) and has a compact slice available. Use `token-goat skill-section ' + nameArg + - ' ' + skillSectionArg(sourcePath, body) + '` to load a specific section, `token-goat skill-body ' + nameArg + - ' --compact` to load the compact slice, or `token-goat skill-body ' + nameArg + '` for the full body.', + skillSubject + ' is large (' + bodyBytes + ' bytes)' + (usable !== null ? ' and has a compact slice available' : '') + + '. Use `token-goat skill-section ' + nameArg + ' ' + skillSectionArg(sourcePath, body) + '` to load a specific section, ' + + (usable !== null ? '`token-goat skill-body ' + nameArg + ' --compact` to load the compact slice, ' : '') + + 'or `token-goat skill-body ' + nameArg + '` for the full body.', ); } } diff --git a/src/skill_cache.ts b/src/skill_cache.ts index 0402a349a..e9863f1e1 100644 --- a/src/skill_cache.ts +++ b/src/skill_cache.ts @@ -12,7 +12,7 @@ import { DEFAULT_MAX_COUNT, DEFAULT_MAX_AGE_MS } from './disk_cache.js' import { redactSecrets } from './secret_redact.js' import { recordStat } from './stats.js' -const COMPACT_END_MARKER = '' +export const COMPACT_END_MARKER = '' // The 'skills' subdir name, exported so prune-cache/clean-cache (cache_session_commands.ts) can include it in their managed-subdirs list. Unlike the other cache subdirs, entries here are plain .txt/.meta/.hits/@compact files rather than disk_cache.ts's JSON blob envelope, so eviction is handled by this module's own pruneSkillOutputs rather than the generic pruneBlobs. export const SKILLS_OUTPUT_SUBDIR = 'skills' @@ -197,6 +197,13 @@ function collectSectionBody(lines: string[], startIdx: number, matchedLevel: num return bodyLines } +/** {@link extractCompactFromMarker}'s slice, only when it is at most half the body. A marker placed near the end cuts a "slice" that is nearly the whole skill (humanizer's is 97.6% of it), so recalling it as compact hands over almost every byte while reading as a saving. The Skill hook's cold-load inline gate has always required this ratio; `skill-body --compact` and every deny that names it now require the same one, so no command is advertised for a slice it would not serve. */ +export function usableCompactSlice(body: string): string | null { + const compact = extractCompactFromMarker(body) + if (compact === null) return null + return Buffer.byteLength(compact, 'utf-8') * 2 <= Buffer.byteLength(body, 'utf-8') ? compact : null +} + export function extractCompactFromMarker(body: string): string | null { if (!body || !body.includes(COMPACT_END_MARKER)) { return null diff --git a/tests/guards/hint_templates_survive_suggestion_guard.test.ts b/tests/guards/hint_templates_survive_suggestion_guard.test.ts index 717e310f7..2611b8baf 100644 --- a/tests/guards/hint_templates_survive_suggestion_guard.test.ts +++ b/tests/guards/hint_templates_survive_suggestion_guard.test.ts @@ -377,6 +377,7 @@ const SAFE_BARE: ReadonlyArray<{ key: string; reason: string }> = [ { key: 'read_git.ts::suggestedRef', reason: '`HEAD~n` for a number n, or the fixed empty-tree hash' }, { key: 'hooks_bash.ts::monitoringHint', reason: 'a flag string from the fixed MONITORING_COMMAND_PATTERNS table (src/hints/lang_patterns.ts)' }, ...['answer_router.ts::command', 'answer_router.ts::cls.intent', 'bash_extractors.ts::fmt', 'hint_target.ts::format', 'hooks_read.ts::fmt', 'read_spec.ts::command', 'read_spec.ts::commandName', 'read_suggest.ts::command', 'read_symbol.ts::hit.command', 'cli_recall.ts::RECALL_COMMAND[hit.cacheType]', 'hooks_bash.ts::tgRead.sub'].map((key) => ({ key, reason: 'a command name (or `json`/`yaml` prefix) from a fixed set in the source, never user text' })), + { key: 'cli_skills.ts::headingArg', reason: 'a real heading of the skill already passed through quotedArg (kept only when displaySafeText leaves it unchanged and canQuoteArg holds), else the fixed `""` placeholder, as hooks_skill.ts skillSectionArg returns' }, { key: 'cli_install.ts::leftover.flag', reason: 'an install flag such as `--codex` from the fixed leftoverIntegrations table' }, ...['cli_install.ts::removal.label', 'cli_install.ts::leftover.label'].map((key) => ({ key, reason: 'a fixed integration label in a status line (`Removed token-goat Codex CLI integration.`), not a command' })), ...['hooks_bash.ts::filterName', 'hooks_bash.ts::cfg.timeout_seconds', 'hooks_bash.ts::capArgs', 'hooks_bash.ts::shellQuoteSingle(rawCmd)'].map((key) => ({ key, reason: 'the compress wrapper command line the hook executes, built from a filter name, numbers and a shellQuoteSingle-quoted command' })), diff --git a/tests/helpers/matrix_cases.ts b/tests/helpers/matrix_cases.ts index 349c32444..c7eeb958d 100644 --- a/tests/helpers/matrix_cases.ts +++ b/tests/helpers/matrix_cases.ts @@ -2163,6 +2163,16 @@ export const cases: Record void | Promise> = { expect(r.status).not.toBe(0) expect((r.stdout + r.stderr).length).toBeGreaterThan(0) expect(r.stdout + r.stderr).not.toMatch(/unknown command|is not a function/) + // A skill with no compact slice used to come back whole from --compact; it now gets its heading tree and a stderr note. + const skillDir = path.join(homeBase, '.claude', 'skills', 'tgmatrixskill') + fs.mkdirSync(skillDir, { recursive: true }) + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), `# Matrix Skill\n\n${Array.from({ length: 8 }, (_, i) => `## Part ${i}\n\n${'w'.repeat(900)}`).join('\n\n')}\n`) + const compact = run(['skill-body', 'tgmatrixskill', '--compact']) + expect(compact.status, compact.stderr).toBe(0) + expect(compact.stdout).toContain('## Part 7') + expect(compact.stdout).toContain('token-goat skill-section "tgmatrixskill" "Part 0"') + expect(compact.stdout).not.toContain('w'.repeat(900)) + expect(compact.stderr).toContain('no compact slice') }, 'skill-compact': () => { const r = run(['skill-compact', 'no-such-skill']) diff --git a/tests/hooks_skill.test.ts b/tests/hooks_skill.test.ts index a05ab9117..9b71f21c7 100644 --- a/tests/hooks_skill.test.ts +++ b/tests/hooks_skill.test.ts @@ -185,6 +185,8 @@ describe('preSkillHandler — duplicate-load advisory', () => { // Regression for F5: once a skill body is cached this session, a second Skill invocation must be denied with a compact-recall pointer instead of re-injecting the whole body. Drives the REAL registry: post stores via runHook, then a pre dispatch through runHook must come back deny. A no-op preSkillHandler (the pre-fix scaffold) returns pass here and fails this test. it('denies a second load through the real runHook dispatch and points at compact recall', async () => { + await fs.mkdir(path.join(sourceDir, 'ollama'), { recursive: true }); + await fs.writeFile(path.join(sourceDir, 'ollama', 'SKILL.md'), `Short rules.\n\n${'detail '.repeat(40)}\n`, 'utf-8'); const post = await runHook(skillPostEvent('ollama', 'Body for ollama.', 'sess-dup')); expect(post.hookType).toBe('pass'); @@ -196,6 +198,25 @@ describe('preSkillHandler — duplicate-load advisory', () => { } }); + // `skill-body --compact` serves no slice for a skill with no marker, or one whose marker cuts more than half the body (humanizer's cut 97.6% of it), so the reload deny must not name it there: it did for every skill, and the agent then pulled the whole body through it. + it.each([ + ['no marker', 'no-marker-reload', `${'rules '.repeat(40)}\n`], + ['a slice over half the body', 'late-marker-reload', `${'rules '.repeat(40)}\n\ntrailing\n`], + ])('names no --compact recall for a reloaded skill with %s', async (_label, skill, installed) => { + await fs.mkdir(path.join(sourceDir, skill), { recursive: true }); + await fs.writeFile(path.join(sourceDir, skill, 'SKILL.md'), installed, 'utf-8'); + await runHook(skillPostEvent(skill, installed, 'sess-reload-noslice')); + + const pre = await runHook(skillPreEvent(skill, 'sess-reload-noslice')); + expect(pre.hookType).toBe('deny'); + if (pre.hookType === 'deny') { + expect(pre.message).toContain('already loaded this session'); + expect(pre.message).toContain(`token-goat skill-section "${skill}"`); + expect(pre.message).toContain(`token-goat skill-body "${skill}"`); + expect(pre.message).not.toContain('--compact'); + } + }); + // The deny names the skill through nameSubject, quoted as its commands quote it, rather than as 'Skill `' + skillName + '`', where a backtick in the name would close the fence. safeSkillName (src/skill_cache.ts) keeps every name that reaches a deny to [A-Za-z0-9_:-] today, so this pins the form; tests/guards/hook_prose_names_are_quoted.test.ts pins the rule. HAND-DERIVED. it('names the skill in its own words the way its commands quote it', async () => { await runHook(skillPostEvent('my-skill:x', 'Body.', 'sess-name')); @@ -297,7 +318,8 @@ describe('preSkillHandler — oversized first-load gate', () => { await fs.mkdir(skillDir, { recursive: true }); // Original intent unchanged -- a slice too large to inline still gets the pointer. Only the bound moved: the inline decision no longer reuses the 6000-byte oversize gate, so the fixture has to clear COMPACT_INLINE_MAX_BYTES (24_000) to still be "oversized" for this purpose. const compact = 'z'.repeat(24_001); - const body = `${compact}\n\n${'x'.repeat(7000)}`; + // Long enough that the slice stays under half the body, the share `skill-body --compact` needs before it serves one. + const body = `${compact}\n\n${'x'.repeat(25_000)}`; await fs.writeFile(path.join(skillDir, 'SKILL.md'), body, 'utf-8'); const out = await preSkillHandler(skillPreEvent('huge-compact-skill', 'sess-huge')); @@ -335,7 +357,10 @@ describe('preSkillHandler — oversized first-load gate', () => { const out = await preSkillHandler(skillPreEvent('degenerate-compact', 'sess-degen')); expect(out.hookType).toBe('deny'); if (out.hookType === 'deny') { - expect(out.message).toContain('token-goat skill-body "degenerate-compact" --compact'); + expect(out.message).not.toContain(compact); + // The slice is the whole body, so `skill-body --compact` would not serve it: the pointer names the section and full-body commands only. + expect(out.message).toContain('token-goat skill-section "degenerate-compact"'); + expect(out.message).not.toContain('--compact'); } }); @@ -477,9 +502,9 @@ describe('hints.pre_skill_advisory wiring', () => { const skillName = 'oversized-skill-section-test'; const skillDir = path.join(sourceDir, skillName); await fs.mkdir(skillDir, { recursive: true }); - // Create a skill that is oversized but the compact won't inline (compact > COMPACT_INLINE_MAX_BYTES) + // Create a skill that is oversized but the compact won't inline (compact > COMPACT_INLINE_MAX_BYTES), while staying under half the body so it is still a slice `skill-body --compact` serves const compact = 'x'.repeat(25000); - const detail = 'y'.repeat(7000); + const detail = 'y'.repeat(26000); const body = `${compact}\n\n${detail}`; await fs.writeFile(path.join(skillDir, 'SKILL.md'), body, 'utf-8'); @@ -688,8 +713,8 @@ describe('preSkillHandler -- an oversized skill whose compact slice is too large expect(out.message).toContain('Section 7'); // Anti-vacuity: a branch that regressed to handing back the body would still contain every heading. The replacement has to be materially smaller than what it replaced. expect(out.message.length).toBeLessThan(bodyBytes * 0.5); - // The slice is still reachable -- it exists, it is just too big to inline, so the pointer to it stays. - expect(out.message).toContain('--compact'); + // The slice is ~97% of the body, which `skill-body --compact` does not serve, so the deny no longer names it. + expect(out.message).not.toContain('--compact'); expect(out.message).toContain('too large to inline'); }); @@ -750,7 +775,10 @@ describe('preSkillHandler -- an oversized skill whose compact slice is too large const out = await preSkillHandler(skillPreEvent('late-marker-flat', 'sess-late-flat')); expect(out.hookType).toBe('deny'); if (out.hookType !== 'deny' || !out.message) return; - expect(out.message).toContain('has a compact slice available'); + // The slice is 78% of the body, so the pointer stands without claiming a compact slice it would not serve. + expect(out.message).toContain('token-goat skill-section "late-marker-flat"'); + expect(out.message).not.toContain('has a compact slice available'); + expect(out.message).not.toContain('--compact'); expect(out.message).not.toContain('heading tree'); }); }); diff --git a/tests/skill_body_compact_no_slice.test.ts b/tests/skill_body_compact_no_slice.test.ts new file mode 100644 index 000000000..261ea8f60 --- /dev/null +++ b/tests/skill_body_compact_no_slice.test.ts @@ -0,0 +1,78 @@ +/** Regression: `skill-body --compact` printed the full body, without a word, for a skill with no usable compact slice. humanizer's marker cut 97.6% of its body, and marketing and token-goat have no marker, so they came back byte-identical; one session spent six calls loading one skill. Such a skill now gets the heading tree the Skill hook inlines in the same case, with how to load one section, and a stderr note. Runs the built bundle against isolated skill directories. */ +import { spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { BUNDLE } from './helpers/bundle.js' + +// HAND-DERIVED: eight H2 sections of 1200 bytes clear the hook's map floors (at least six headings, a tree well under 40% of the body). +const SECTIONS = Array.from({ length: 8 }, (_, i) => `## Section ${i}\n\n${'p'.repeat(1200)}`).join('\n\n') +const NO_MARKER = `# Mapped Skill\n\n${SECTIONS}\n` +const LATE_MARKER = `# Mapped Skill\n\n${SECTIONS}\n\ntrailing note\n` +const FLAT = `${'flat line of a skill with no headings\n'.repeat(200)}` + +let scratch: string +let env: NodeJS.ProcessEnv + +function install(name: string, body: string): void { + fs.mkdirSync(path.join(scratch, '.claude', 'skills', name), { recursive: true }) + fs.writeFileSync(path.join(scratch, '.claude', 'skills', name, 'SKILL.md'), body, 'utf-8') +} + +function cli(args: string[]): { status: number | null; stdout: string; stderr: string } { + const r = spawnSync(process.execPath, [BUNDLE, ...args], { encoding: 'utf8', env, timeout: 60000 }) + return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' } +} + +beforeEach(() => { + scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'tg-skillbody-noslice-')) + env = { + ...process.env, + TOKEN_GOAT_HOME: path.join(scratch, 'home'), + LOCALAPPDATA: path.join(scratch, 'local'), + XDG_DATA_HOME: path.join(scratch, 'local'), + APPDATA: path.join(scratch, 'roaming'), + USERPROFILE: scratch, + HOME: scratch, + } +}) + +afterEach(() => { + fs.rmSync(scratch, { recursive: true, force: true }) +}) + +describe('skill-body --compact for a skill with no usable compact slice', () => { + it.each([ + ['no marker', 'tgnomarker', NO_MARKER, 'no COMPACT_END marker'], + ['a marker cutting more than half the body', 'tglatemarker', LATE_MARKER, 'no compact slice under half its body'], + ])('prints the heading tree and how to load a section, not the body, for %s', (_label, name, body, why) => { + install(name, body) + const r = cli(['skill-body', name, '--compact']) + expect(r.status, r.stderr).toBe(0) + expect(r.stderr).toContain('no compact slice') + expect(r.stdout).toContain(why.replace('COMPACT_END', '')) + expect(r.stdout).toContain('## Section 0') + expect(r.stdout).toContain('## Section 7') + expect(r.stdout).toContain(`token-goat skill-section "${name}" "Section 0"`) + expect(r.stdout).not.toContain('p'.repeat(1200)) + expect(r.stdout.length).toBeLessThan(Buffer.byteLength(body, 'utf-8') / 2) + + // The command it names loads that section, and the full body stays one flag away. + const section = cli(['skill-section', name, 'Section 0']) + expect(section.status, section.stderr).toBe(0) + expect(section.stdout).toContain('p'.repeat(1200)) + expect(cli(['skill-body', name]).stdout).toContain('p'.repeat(1200)) + }) + + it('says it printed the full body when the skill has too few headings to map', () => { + install('tgflat', FLAT) + const r = cli(['skill-body', 'tgflat', '--compact']) + expect(r.status, r.stderr).toBe(0) + expect(r.stderr).toContain('no compact slice') + expect(r.stderr).toContain('printed the full body') + expect(r.stdout).toBe(FLAT) + }) +}) From 8753334e2d8cde98b374dace96a2eb8c62654c8a Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 05:47:56 -0500 Subject: [PATCH 10/92] fix(output): keep section, skill and recall answers under the harness inline limit, paged with a continuation Claude Code shows a Bash result inline only up to about 30,000 characters; past that it persists the result to a file and shows the model a 2 KB preview (measured on 2.1.29x: 26,588 chars came back inline, a 29,926-char stdout was persisted). token-goat's own answers had no ceiling: section "CLAUDE.arch.md::Security Boundaries" returned 52.5 KB, skill-body humanizer 49 KB and bash-output --file --grep 48 KB, and each cost further calls reading the persisted file back. HARNESS_INLINE_CEILING_CHARS (24,000, in overflow_guard.ts) caps them, lowered by overflow_guard.max_tokens and switched off with overflow_guard.enabled like guardText. section, skill-body and skill-section cut at a line boundary and name the token-goat read "file@A-B" that returns the next lines, its range sized to fit the ceiling too; a multi-section answer splits the ceiling across its sections, and the MCP section tool passes none. bash-output, web-output and mcp-output cut whole rows inside the fence, close it, and name below it the same recall with --lines A-B and its narrowing flags kept; a single line longer than the ceiling keeps its head and points at --grep, or --json-query for an MCP result. --full still prints everything. bash-output --grep with no --max-matches, --head, --tail or --full now stops at 100 matches with the --lines continuation for the rest, instead of the first 30 and last 80 behind a marker that pointed at --full; tests/cli.test.ts pins the new bound. tests/output_ceiling_continuation.test.ts runs each of the six commands on the built bundle, then runs the continuation it printed and checks it returns the next lines; the bash-output matrix case checks the cut. The truncator, display-sink and hint-template guards classify the new cut and its continuation command. --- docs/cli.md | 4 +- src/cli.ts | 4 +- src/cli_cached_output.ts | 137 +++++++++++-- src/cli_skills.ts | 24 ++- src/overflow_guard.ts | 33 ++++ src/read_commands.ts | 9 +- src/read_section.ts | 33 +++- src/skill_cache.ts | 10 +- tests/cli.test.ts | 11 +- .../guards/display_safe_sink_coverage.test.ts | 7 + ...templates_survive_suggestion_guard.test.ts | 2 +- .../guards/truncators_are_classified.test.ts | 17 ++ tests/helpers/matrix_cases.ts | 7 + tests/output_ceiling_continuation.test.ts | 181 ++++++++++++++++++ 14 files changed, 438 insertions(+), 41 deletions(-) create mode 100644 tests/output_ceiling_continuation.test.ts diff --git a/docs/cli.md b/docs/cli.md index 09351d04c..326b64e5e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -38,7 +38,7 @@ token-goat pdf-extract manual.pdf --pages 12-15 --layout --head 120 | `token-goat note-list [--stale-only]` | List every recorded architecture note. `--stale-only` shows just the notes whose fingerprint no longer matches the current index — i.e. the file/symbol they describe changed since the note was written. Staleness is purely advisory: nothing here auto-rewrites or deletes a note. | | `token-goat write-file ` | Write exact bytes to a file, sidestepping shell-escaping trouble with backticks, quotes, `$vars`, and CRLF. `--from ` copies bytes from a source file; `--b64 ` decodes a base64 payload; with neither, reads from stdin. | | `token-goat section "doc.md::Heading"` | Pull one Markdown section by heading. Fully supports ATX (`#`/`##`) and Setext-style (`Heading\n===` level 1 and `Heading\n---` level 2) headings. A miss that is an unambiguous prefix of exactly one heading, or a distinctive suffix/word-subset of exactly one heading (e.g. `Setup` → "Installation and Setup", `Config Options` → "Configuration Options"), auto-redirects with a `(redirected from: …)` marker (and a `redirectedFrom` field under `--json`); a query matching 2+ headings is never guessed and reports a miss instead. A genuine miss lists only headings similar to the query as "Did you mean" suggestions, not every heading in the file. Disambiguate duplicates with `"doc.md::Heading#2"`. Comma-separated `"doc.md::A,B"` fetches several sections from one file in a single call (several specs, `section "a.md::A" "b.md::B"`, read the same way), mirroring `read`'s `file::a,b` multi-symbol grammar, and automatically subsumes nested child sections (`(already included in section 'Parent', lines X-Y)`) to prevent duplicate tokens. Cross-file `"a.md::Heading1,b.md::Heading2"` fetches sections from several files in one call, mirroring `read`'s `a.ts::x,b.ts::y` cross-file grammar — a bare heading after a `file::Heading` segment inherits the previous file, and each section is keyed by its full `file::Heading` pair so two files sharing a heading name cannot overwrite each other. `token-goat section doc.md --list` lists every heading in the file instead of reading one; `--grep ` narrows that list to headings matching a regex (falls back to a literal substring match if the pattern doesn't compile), same convention as `outline`/`types`/`exports`'s own `--grep`. Without `--list`, `--grep` filters the section body instead: only the matching lines print, each under the nearest sub-heading above it, followed by `N of M lines matched`. A pattern that matches nothing says so and exits 0. YAML front matter at the top of a file is skipped, so its closing `---` is never read as a heading underline. | -| `token-goat skill-section "::"` | Extract a named section from an installed skill without reading the full skill file. | +| `token-goat skill-section "::"` | Extract a named section from an installed skill without reading the full skill file. `section`, `skill-section` and `skill-body` hold an answer to 24,000 characters, the size Claude Code still shows inline, cutting at a line boundary with a note naming the `token-goat read "file@A-B"` that returns the next lines. | | `token-goat skeleton "file"` | Show all signatures in a file without bodies — typically 70–90% fewer tokens than a full read. `--force-refresh` reparses from disk first, bypassing a stale index. `--stats` adds a per-symbol reference count and doc-coverage flag, computed live from the index. `--grep ` narrows to symbols whose name matches a regex (a literal substring when the pattern is not valid regex), which is how you skim one area of a large file without dumping its whole symbol list; `--min-lines ` drops symbols shorter than N lines. Both compose, and if a filter removes everything the output says so and names the filter, rather than looking like a file with no symbols. Accepts a comma-separated file list (`"a,b,c"`) to cover several files in one call, one clearly-headed block per file; extra space-separated file arguments are reported in a note naming that comma form instead of being silently dropped. With `--json`, a comma-separated list returns one merged document (rows carry their own `filePath`), not one document per file. | | `token-goat outline "file"` | List top-level symbols with line ranges and docstring hints — one-glance file map. Doc hints are clipped to about a sentence, with a visible ellipsis whenever the doc goes on past the hint; the full doc comment is one `read "file::symbol"` away (`--json` carries it whole). `--force-refresh` reparses from disk first, bypassing a stale index. `--stats` adds a per-symbol reference count and doc-coverage flag, computed live from the index. `--grep ` narrows to symbols whose name matches a regex (a literal substring when the pattern is not valid regex), which is how you skim one area of a large file without dumping its whole symbol list; `--min-lines ` drops symbols shorter than N lines. Both compose, and if a filter removes everything the output says so and names the filter, rather than looking like a file with no symbols. Accepts a comma-separated file list (`"a,b,c"`) to cover several files in one call, one clearly-headed block per file; extra space-separated file arguments are reported in a note naming that comma form instead of being silently dropped. With `--json`, a comma-separated list returns one merged document (rows carry their own `filePath`), not one document per file. | | `token-goat yaml-outline ` | Structural summary of a YAML document (array shape / object key types) instead of a raw Read. Multi-document streams (`---`-separated) outline as an array of documents. `--filter TEXT` keeps only the top-level keys containing the text and reports how many matched. | @@ -105,7 +105,7 @@ token-goat pdf-extract manual.pdf --pages 12-15 --layout --head 120 | `token-goat session-outline` | Turn-by-turn structure (role, preview, tool calls, approx size) of a Claude Code session JSONL transcript, instead of a raw Read; defaults to the current project's most recent session. | | `token-goat session-slice ` | Full content of one turn range from a Claude Code session JSONL transcript (see `session-outline` for turn numbers), instead of a raw Read. | | `token-goat session-audit [--dir ] [--json] [--tool-errors]` | Corpus-wide token attribution across every local Claude Code session transcript, including nested subagent transcripts (default corpus: `~/.claude/projects`): measured billed usage from each API response's own usage record, estimated content size by source and by tool, a per-attachment-kind census ranked by modeled billed cost (cache write plus compaction-capped cache re-reads over the model-visible fields only), a hook-output census split by origin, a subagent-lane rollup (spawn-prefix size and its modeled billed carriage, plus a per-agent-type breakdown from each lane's meta file), a Read-interception census (diverted reads versus full serves, with each large full serve split into first read versus repeat and repeats classified as deliberate paging or divert-miss candidates), a Bash filter fire-rate census (results carrying a token-goat marker versus the untouched remainder, bucketed by bare command head: the binary name only), and billed cost by session position. `--tool-errors` replaces this with a census of failed tool calls instead: calls, errors and unknown errors per tool and per model, with the most frequent unknown error prefixes, so a call that fails without a recognized reason still points at what to classify next. A deny that delivers the content it withholds, such as a skill's compact slice, is counted separately and named on its own line rather than folded into the error counts. Output is aggregate counts only, never transcript content or command lines. | -| `token-goat bash-output ` | Retrieve a cached Bash output by ID instead of re-running the command. Large outputs return a head(30)+tail(80) view by default; pass `--full` for the entire stored entry with no elision, `--head N`/`--tail N` for a specific slice, or narrow with `--grep PATTERN` (cap `--grep` to the first N hits with `--max-matches N`). The cut in the middle of a default view is labelled with the lines it dropped and the exact `--lines` that fetches them, for example `...(elided lines 31-520 of 600: --lines 31-520)...`. `--lines A-B` prints exactly that inclusive, 1-based range with no elision (a range past the end is clamped, one that starts past the end is an error); `-n`/`--line-numbers` prefixes each printed line with its line number (`N:text`), so a `--grep` hit can be located; `--context N` with `--grep` also shows N lines around each hit, with `--` between separate groups. `web-output` and `mcp-output` take the same three options. Read a file directly with `--file ` (e.g. a background task's `tasks/.output`); add `--transcript` to parse that file as a subagent JSONL transcript, keeping only assistant text blocks in order before the slicers apply. Add `--verify-last-write [seconds]` (with optional `--strict`) when reading with `--file` to verify the output file was modified within the last N seconds (default 60s), alerting or rejecting stale/cached reads from silent no-op commands before repeating runs. | +| `token-goat bash-output ` | Retrieve a cached Bash output by ID instead of re-running the command. Large outputs return a head(30)+tail(80) view by default; pass `--full` for the entire stored entry with no elision, `--head N`/`--tail N` for a specific slice, or narrow with `--grep PATTERN` (cap `--grep` to the first N hits with `--max-matches N`; with none of `--max-matches`, `--head`, `--tail` or `--full`, `--grep` stops at 100 hits and names the `--lines` that continue it). Any answer longer than 24,000 characters, the size Claude Code still shows inline rather than persisting to a file, is cut at a line boundary below the closing fence tag with a note naming the `--lines A-B` (narrowing flags kept) that returns the next part; `web-output` and `mcp-output` do the same, `--full` opts out, and `overflow_guard.enabled = false` turns it off. The cut in the middle of a default view is labelled with the lines it dropped and the exact `--lines` that fetches them, for example `...(elided lines 31-520 of 600: --lines 31-520)...`. `--lines A-B` prints exactly that inclusive, 1-based range with no elision (a range past the end is clamped, one that starts past the end is an error); `-n`/`--line-numbers` prefixes each printed line with its line number (`N:text`), so a `--grep` hit can be located; `--context N` with `--grep` also shows N lines around each hit, with `--` between separate groups. `web-output` and `mcp-output` take the same three options. Read a file directly with `--file ` (e.g. a background task's `tasks/.output`); add `--transcript` to parse that file as a subagent JSONL transcript, keeping only assistant text blocks in order before the slicers apply. Add `--verify-last-write [seconds]` (with optional `--strict`) when reading with `--file` to verify the output file was modified within the last N seconds (default 60s), alerting or rejecting stale/cached reads from silent no-op commands before repeating runs. | | `token-goat bash-history` | List cached Bash outputs (newest first) with their IDs, byte sizes, and exit codes. | | `token-goat compress [--cmd ''] [--shell ]` | Run a command through token-goat's output compression filters. Agents and interactive users must pass ordinary plain shell commands directly, using positional arguments or `--cmd`. `--cmd-b64` is strictly an internal hook transport flag generated programmatically by token-goat hooks, not a flag to use manually: LLM-generated base64 can corrupt characters and file paths. `--shell` specifies runner (`bash`, `pwsh`, `powershell`, or `native`, defaulting to bash on POSIX and resolving PowerShell on Windows). `-f passthrough` applies no filter, only the size limits. `--max-tokens ` caps what is delivered, and output cut by the cap or a filter ends with a `bash-output --full` line to recall the rest. The Bash hook adds `--cap-hint-b64`, a base64 sentence printed after that line naming the narrower command to run next. The wrapped command gets no standard input on any path unless `--stdin` is given, which passes the caller's standard input through byte for byte. | | `token-goat web-output ` | Retrieve a cached WebFetch response body by ID — same head+tail default and `--full`/`--head`/`--tail`/`--grep`/`--max-matches` slicers as `bash-output`, plus `--lines A-B`, `-n` and `--context N`. `--raw` returns the body as actually fetched, before `webfetch.compress_bodies`'s HTML-cleaning pass, for recovering a selector/script tag/embedded JSON that the default cleaned text drops; falls back to the (already-raw) cleaned body when no separate raw copy was stored. | diff --git a/src/cli.ts b/src/cli.ts index ec1f54c84..514c09b80 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -39,7 +39,7 @@ import { embedPolicyResolver, runDetachedWorkerDaemon } from './worker.js' import { ensureWorkerAlive, isWorkerRunning, startDetachedWorker, stopWorker, WorkerAlreadyRunningError } from './worker_lifecycle.js' import { enqueueDirtyPathsSafe } from './hooks_index.js' // Loaded on demand inside cmdCompress, not at module scope: bash_runner pulls in the whole bash tool-filter registry (every language, linter, cloud and package-manager filter), which only the compress command ever uses. See the same reasoning for relay in cmdHook. -import { runRead, runPrSlice } from './read_commands.js' +import { outputCeilingChars, runRead, runPrSlice } from './read_commands.js' import { runSymbol } from './read_symbol.js' import { extraArgsRejoinSpacedPath, namedSpecsMergeable, readSpecsMergeable } from './read_spec.js' import { runBrief } from './read_brief.js' @@ -1044,10 +1044,12 @@ export function buildProgram(): Command { return runListSections({ file: spec, ...(opts.json === true ? { json: true } : {}), ...(opts.grep !== undefined ? { grep: opts.grep } : {}) }) }) : runExitText(() => { + const ceiling = opts.json === true ? null : outputCeilingChars() const sectionOpts = { ...(opts.json === true ? { json: true } : {}), ...(maxLines !== undefined ? { maxLines } : {}), ...(opts.grep !== undefined ? { grep: opts.grep } : {}), + ...(ceiling !== null ? { maxChars: ceiling } : {}), } // Words a shell split out of one unquoted heading are not several specs: that shape keeps the note that says how to quote it. if (more.length > 0 && !extraArgsRejoinSpacedPath(spec, more, 'spec')) return runSectionSpecs([spec, ...more], sectionOpts) diff --git a/src/cli_cached_output.ts b/src/cli_cached_output.ts index 0f9dc0282..612261018 100644 --- a/src/cli_cached_output.ts +++ b/src/cli_cached_output.ts @@ -7,10 +7,12 @@ import { CliError, err, formatCommandError, out } from './cli.js' import { requireNonNegativeInt } from './cli_dispatch.js' import { redactIfDotenv } from './dotenv_redact.js' import { UNTRUSTED_TOOL_TAG, UNTRUSTED_WEB_TAG } from './injection_scan.js' -import { echoedValue, quotedArg } from './hint_suggestion_guard.js' +import { canQuoteArg, echoedValue, fencedCommand, quotedArg } from './hint_suggestion_guard.js' +import { forClient } from './mcp_client_text.js' +import { linesWithinChars } from './overflow_guard.js' import { noMatchMessage, queryJson } from './json_query.js' import { displaySafeJson } from './paths.js' -import { guardJsonRows } from './read_commands.js' +import { guardJsonRows, outputCeilingChars } from './read_commands.js' import { headElidedNotice, traversalLimitNotice } from './query_notices.js' import { extractTranscriptText } from './read_inspect.js' import { compileGuardedRegex } from './regex_guard.js' @@ -20,7 +22,7 @@ import { AMBIGUOUS_HEADING_LIMIT } from './read_section.js' import { extractSection } from './section_reader.js' import { clipLongMatchLine } from './tool_filters/helpers.js' import { fenceIfScanMatched, fenceJsonStrings, fenceUntrusted } from './untrusted_fence.js' -import { countNoun, decodeSource, isWindows } from './util.js' +import { countNoun, decodeSource, isWindows, safeSlice } from './util.js' import { getWebOutput, getWebOutputRaw } from './web_cache.js' /** The narrowing flags every cached-output recall command shares; the three recall-by-position flags (`--lines`, `-n`, `--context`) are optional so callers that never register them (cli_office.ts) compile unchanged. */ @@ -52,14 +54,29 @@ function parseLineRange(spec: string): { from: number; to: number } { return { from, to } } +/** How a recall command re-runs for the part of its answer the output ceiling left out. `command` is the command up to its narrowing flags (`token-goat bash-output `, `token-goat mcp-output --file "" --json-query ""`), null when the path or id has no spelling both shells keep; `capped` is false when the caller asked for everything with --full; `defaultMaxMatches` bounds a --grep given no --max-matches, --head, --tail or --full. Absent for the callers that keep no ceiling (`retrieve` and the document commands). */ +export interface RecallContinuation { + command: string | null + capped: boolean + defaultMaxMatches?: number + /** The flags that narrow inside one line too long for the ceiling, which `--lines` cannot page: `--grep` by default, and `--json-query` too for an MCP result, often one line of JSON. */ + narrowFlags?: string +} + +/** Characters of a capped recall kept free for the continuation notice printed below the fence. */ +const CONTINUATION_RESERVE_CHARS = 600 + export function _applyFiltersAndPrint( content: string, opts: RecallFilterOpts, fenceByProvenance = false, fenceTag: string = UNTRUSTED_WEB_TAG, + recall?: RecallContinuation, ): string { // Said on stderr after the body: a line inside the content fence is escaped as payload and reads as part of the text it describes. let capNotice: string | undefined + // Set when a default --max-matches (recall.defaultMaxMatches) dropped matches: said on stdout with the command for the rest, as a ceiling cut is. + let defaultCapped: { shown: number; total: number } | undefined // Fetched-page recall only. The injection scan is documented as unconditional for fetched pages, and a `web-output ` recall puts that same attacker-written text in front of the model -- but only the WebFetch post-hook fenced it, so the copy served from the cache came back bare. Scanning here rather than at store time means the fence wraps exactly what the caller sees, so a --grep/--head slice that keeps the payload is fenced and one that drops it is not. Recall of cached Bash and MCP output is scanned too, under a tag naming tool output rather than a fetched page. Neither is a page token-goat fetched, but both carry text written by a third party -- a dependency's build or test output, a remote MCP server's result -- and the recall channel put it in front of the model unmarked. The fence is decided by provenance -- `fenceByProvenance` -- and not by whether the scan matched: it used to appear only on a positive hit, which meant any payload the eight deliberately-narrow regexes miss was emitted bare. The scan now only decides whether the notice names pattern(s) and whether a stat is recorded. const emit = (text: string): string => { const shown = !fenceByProvenance || text === '' ? text : fenceUntrusted(text, fenceTag) @@ -72,6 +89,58 @@ export function _applyFiltersAndPrint( // fenceByProvenance true means this is third-party content (a fetched page, a recalled cache entry, a document the caller only named rather than authored), the same population the injection fence covers -- redact it before any narrowing so --grep's long-line clip can never cut a secret in half and leave a fragment the redactor no longer recognises. Idempotent on content already redacted at write time (bash/web/mcp caches). if (fenceByProvenance) content = redactSecrets(content, undefined, { keepLineCount: true }).text const render = (rows: RecallRow[]): string => rows.map((r) => (opts.lineNumbers === true && r.n !== null ? `${r.n}:${r.text}` : r.text)).join('\n') + // Prints `shown` through {@link emit}, held to the output ceiling when `recall` asks for one. A cut keeps whole leading rows, closes the fence, and names on stdout below it the command that returns the next lines (`--lines A-B`, with the narrowing flags kept), so an answer past the harness's inline limit costs one more call instead of a persisted file and two more. + const emitRows = (shown: RecallRow[], trailing = ''): string => { + const ceiling = recall?.capped === true ? outputCeilingChars() : null + let kept = shown + let cutLine: { n: number; of: number } | undefined + if (ceiling !== null && fencedLength(render(shown) + trailing) > ceiling) { + let budget = ceiling - CONTINUATION_RESERVE_CHARS - fencedLength('') + for (let pass = 0; pass < 8; pass++) { + kept = shown.slice(0, linesWithinChars(shown.map((r) => render([r])), budget)) + const only = kept[0] + // One line longer than the whole ceiling (a minified JSON result is often a single line): keep its head, since --lines cannot page inside a line. + if (kept.length === 1 && only !== undefined && render(kept).length > budget) { + const room = Math.max(1, budget - (render(kept).length - only.text.length) - 40) + cutLine = { n: only.n ?? 0, of: only.text.length } + kept = [{ n: only.n, text: `${safeSlice(only.text, room)} … [${only.text.length - room} chars not shown]` }] + } + const excess = fencedLength(render(kept)) + CONTINUATION_RESERVE_CHARS - ceiling + if (excess <= 0) break + budget -= excess + } + } + if (kept === shown && defaultCapped === undefined) return emit(render(shown) + trailing) + const printed = emit(render(kept)) + const notice = continuationNotice(kept, shown, ceiling !== null && kept !== shown ? ceiling : null, cutLine) + out(notice) + return `${printed}\n${notice}` + } + const fencedLength = (text: string): number => (fenceByProvenance && text !== '' ? fenceUntrusted(text, fenceTag).length : fenceByProvenance ? fenceUntrusted(' ', fenceTag).length : text.length) + // The notice below a capped recall: what it showed and the command for the rest. A --grep with no --head or --tail continues to the last line it searched; any other view continues to the last line it would have shown. + const continuationNotice = (kept: RecallRow[], shown: RecallRow[], ceiling: number | null, cutLine: { n: number; of: number } | undefined): string => { + const lastN = (rows: RecallRow[]): number | null => [...rows].reverse().find((r) => r.n !== null)?.n ?? null + const lastShown = lastN(kept) + const end = opts.grep !== undefined && opts.head === undefined && opts.tail === undefined ? searchEnd : lastN(shown) + const firstShown = kept.find((r) => r.n !== null)?.n ?? null + const what = opts.grep !== undefined ? `matches through line ${lastShown ?? '?'}` : `lines ${firstShown ?? '?'}-${lastShown ?? '?'}` + const head = ceiling !== null + ? `[token-goat: output capped at ${ceiling} chars to stay under the harness's inline output limit; showing ${what}${end !== null ? ` of ${end}` : ''}.` + : `[token-goat: showing the first ${defaultCapped!.shown} of ${defaultCapped!.total} matching lines, the default for a --grep with no --max-matches.` + const cut = cutLine !== undefined ? ` Line ${cutLine.n} alone is ${cutLine.of} chars, cut above; ${recall?.narrowFlags ?? '--grep'} narrows inside it.` : '' + if (lastShown === null || end === null || lastShown >= end) return forClient(`${head}${cut}]`) + const flags = [ + opts.section !== undefined ? `--section ${quotedArg(opts.section)}` : null, + opts.grep !== undefined ? `--grep ${quotedArg(opts.grep)}` : null, + opts.context !== undefined ? `--context ${opts.context}` : null, + opts.maxMatches !== undefined ? `--max-matches ${opts.maxMatches}` : null, + opts.lineNumbers === true ? '-n' : null, + `--lines ${lastShown + 1}-${end}`, + ] + const runnable = recall?.command !== null && recall?.command !== undefined && (opts.section === undefined || canQuoteArg(opts.section)) && (opts.grep === undefined || canQuoteArg(opts.grep)) + const next = runnable ? ` Next part: ${fencedCommand(`${recall.command} ${flags.filter((f) => f !== null).join(' ')}`)}` : ` The rest is lines ${lastShown + 1}-${end}.` + return forClient(`${head}${cut}${next}${defaultCapped !== undefined && ceiling === null ? ', or pass --max-matches for more at once' : ''}]`) + } let firstLine = 1 if (opts.section !== undefined) { const sectionResult = extractSection(content, opts.section) @@ -113,6 +182,9 @@ export function _applyFiltersAndPrint( throw new CliError('--context needs --grep: it widens each match with the lines around it') } + // The last line a --grep searched, which is where a continuation of its matches has to run to. + const searchEnd = rows.length > 0 ? rows[rows.length - 1]!.n : null + if (opts.grep !== undefined) { let pattern = opts.grep // Normalize pattern to handle -E or --extended-regexp prefix @@ -128,7 +200,8 @@ export function _applyFiltersAndPrint( }) // Said as an error, like a --section that is not found: an empty line at exit 0 reads the same as an empty cache entry or a failed recall. if (hits.length === 0) throw new CliError(`--grep matched no lines of ${rows.length}: ${echoedValue(opts.grep)}`) - const cap = opts.maxMatches !== undefined ? requireNonNegativeInt('--max-matches', opts.maxMatches) : undefined + const explicitCap = opts.maxMatches !== undefined ? requireNonNegativeInt('--max-matches', opts.maxMatches) : undefined + const cap = explicitCap ?? (opts.head === undefined && opts.tail === undefined && opts.full !== true ? recall?.defaultMaxMatches : undefined) const kept = cap !== undefined && hits.length > cap ? hits.slice(0, cap) : hits const ctx = opts.context !== undefined ? requireNonNegativeInt('--context', opts.context) : 0 const lastReal = rows.length - 1 @@ -146,7 +219,8 @@ export function _applyFiltersAndPrint( previous = i } if (cap !== undefined && hits.length > cap) { - capNotice = '[token-goat: showing first ' + cap + ' of ' + hits.length + ' matching lines; raise --max-matches for more]' + if (explicitCap !== undefined) capNotice = '[token-goat: showing first ' + cap + ' of ' + hits.length + ' matching lines; raise --max-matches for more]' + else defaultCapped = { shown: cap, total: hits.length } } rows = next } @@ -154,12 +228,12 @@ export function _applyFiltersAndPrint( // --full is the only way to get the whole stored blob back (colour codes stripped, as on every path here; `retrieve` with no flag bypasses this function for its byte-verbatim contract). The blob store itself is lossless, but every render path below elides the middle past head+tail, so without this flag an elision marker pointing a reader at `mcp-output ` promises a full report the CLI cannot actually produce -- which is exactly what hooks_agent_spawn.ts's envelope compaction relies on. Deliberately bypasses only the elision, not --section/--grep/--max-matches above: those are explicit narrowing the caller asked for. if (opts.full === true) { // The whole text (or --section's) keeps the final newline it ended with, so the verbatim blob is unchanged; a --lines or --grep slice is a set of lines, not the text's end. - return emit(render(rows) + (trailingNewline && opts.lines === undefined && opts.grep === undefined ? '\n' : '')) + return emitRows(rows, trailingNewline && opts.lines === undefined && opts.grep === undefined ? '\n' : '') } const lines = rows // An explicit --lines range is the caller asking for exactly those lines, so the default head/tail window must not cut into it. if (ranged && opts.head === undefined && opts.tail === undefined) { - return emit(render(lines)) + return emitRows(lines) } const headN = opts.head !== undefined ? requireNonNegativeInt('--head', opts.head) : 30 const tailN = opts.tail !== undefined ? requireNonNegativeInt('--tail', opts.tail) : 80 @@ -194,7 +268,7 @@ export function _applyFiltersAndPrint( noteLineCap('last', 'tail', result.length, lines.length) } - return emit(render(result)) + return emitRows(result) } /** The text of the file a `bash-output`/`mcp-output --file` recall names: a regular file only, decoded, and with a dotenv file's values redacted, since either command will print a .env it is pointed at and every other path that serves a file's text redacts them (see dotenv_redact.ts). `mtimeMs` is what `--verify-last-write` measures a write's age against. */ @@ -217,6 +291,23 @@ function readRecallFile(file: string): { text: string; mtimeMs: number } { } } +/** Matches a `bash-output --grep` prints when given no --max-matches, --head, --tail or --full, the rest named by a `--lines` continuation. Unbounded, a --grep over a 337 KB tool-results file printed 48 KB, past the harness's inline limit, so the answer was persisted to yet another file. */ +const BASH_OUTPUT_DEFAULT_MAX_MATCHES = 100 + +/** A recall's own command as a continuation repeats it: `recall` names the command and `args` the id or `--file` path, quoted only when it needs to be; null when a value has no spelling both shells keep. */ +function recallCommand(command: string, args: ({ flag?: string; value: string } | string)[]): string | null { + const words = [`token-goat ${command}`] + for (const arg of args) { + if (typeof arg === 'string') { + words.push(arg) + continue + } + if (!/^[\w.-]+$/.test(arg.value) && !canQuoteArg(arg.value)) return null + words.push(`${arg.flag !== undefined ? `${arg.flag} ` : ''}${/^[\w.-]+$/.test(arg.value) ? arg.value : quotedArg(arg.value)}`) + } + return words.join(' ') +} + export function cmdBashOutput( id: string | undefined, opts: RecallFilterOpts & { @@ -249,7 +340,11 @@ export function cmdBashOutput( process.stderr.write(`[tg: stale-write] ${msg}\n`) } } - _applyFiltersAndPrint(opts.transcript === true ? extractTranscriptText(text) : text, opts, true, UNTRUSTED_TOOL_TAG) + _applyFiltersAndPrint(opts.transcript === true ? extractTranscriptText(text) : text, opts, true, UNTRUSTED_TOOL_TAG, { + command: recallCommand('bash-output', [{ flag: '--file', value: opts.file }, ...(opts.transcript === true ? ['--transcript'] : [])]), + capped: opts.full !== true, + defaultMaxMatches: BASH_OUTPUT_DEFAULT_MAX_MATCHES, + }) return } @@ -273,7 +368,11 @@ export function cmdBashOutput( } } - _applyFiltersAndPrint(entry.output, opts, true, UNTRUSTED_TOOL_TAG) + _applyFiltersAndPrint(entry.output, opts, true, UNTRUSTED_TOOL_TAG, { + command: recallCommand('bash-output', [{ value: id }]), + capped: opts.full !== true, + defaultMaxMatches: BASH_OUTPUT_DEFAULT_MAX_MATCHES, + }) } export function cmdWebOutput( @@ -288,7 +387,10 @@ export function cmdWebOutput( if (content === null) { throw new CliError(`no cached web output for id: ${id}. The cache may have expired; re-run the WebFetch to repopulate it.`) } - _applyFiltersAndPrint(content, opts, true) + _applyFiltersAndPrint(content, opts, true, UNTRUSTED_WEB_TAG, { + command: recallCommand('web-output', [{ value: id }, ...(opts.raw === true ? ['--raw'] : [])]), + capped: opts.full !== true, + }) } function extractJsonFromMcpOutput(text: string): unknown { @@ -347,6 +449,13 @@ export function cmdMcpOutput( throw new CliError('provide an mcp-output or --file ') } + const source = opts.file !== undefined ? { flag: '--file', value: opts.file } : { value: id! } + const recall: RecallContinuation = { + command: recallCommand('mcp-output', opts.jsonQuery !== undefined ? [source, { flag: '--json-query', value: opts.jsonQuery }] : [source]), + capped: opts.full !== true, + narrowFlags: '--json-query or --grep', + } + if (opts.jsonQuery !== undefined) { const data = extractJsonFromMcpOutput(content) let head: number | undefined @@ -364,7 +473,7 @@ export function cmdMcpOutput( out(displaySafeJson(fenceJsonStrings(val, fenceToolFieldIfMatched), 0)) return } - _applyFiltersAndPrint(displaySafeJson(val), printOpts, true, UNTRUSTED_TOOL_TAG) + _applyFiltersAndPrint(displaySafeJson(val), printOpts, true, UNTRUSTED_TOOL_TAG, recall) return } @@ -391,10 +500,10 @@ export function cmdMcpOutput( if (queryResult.truncated) { lines.push(traversalLimitNotice(totalCount)) } - _applyFiltersAndPrint(lines.join('\n'), printOpts, true, UNTRUSTED_TOOL_TAG) + _applyFiltersAndPrint(lines.join('\n'), printOpts, true, UNTRUSTED_TOOL_TAG, recall) } return } - _applyFiltersAndPrint(content, opts, true, UNTRUSTED_TOOL_TAG) + _applyFiltersAndPrint(content, opts, true, UNTRUSTED_TOOL_TAG, recall) } diff --git a/src/cli_skills.ts b/src/cli_skills.ts index 13dde0b24..93b625181 100644 --- a/src/cli_skills.ts +++ b/src/cli_skills.ts @@ -6,8 +6,9 @@ import { formatCommandError } from './command_error.js' import { emitErr } from './emit.js' import { extractMarkdownHeadings } from './hints/markdown_hints.js' import { buildLineDiff } from './hooks_read_slice.js' -import { displaySafeJson, displaySafeText } from './paths.js' -import { findSpecSeparator } from './read_commands.js' +import { displaySafeJson, displaySafeText, normalizePath } from './paths.js' +import { capFileLines } from './overflow_guard.js' +import { findSpecSeparator, outputCeilingChars } from './read_commands.js' import { didYouMeanLines, filterSimilarHeadings } from './read_suggest.js' import { listSections } from './section_reader.js' import { getSessionId } from './session.js' @@ -15,7 +16,7 @@ import { COMPACT_END_MARKER, contentHash, extractCompactFromMarker, - extractNamedSection, + extractNamedSectionAt, formatAge, getSkillFilePath, incrementSkillHit, @@ -40,14 +41,14 @@ export async function cmdSkillBody(name: string, opts: { compact?: boolean }): P if (opts.compact === true) { const compact = usableCompactSlice(body) if (compact !== null) { - out(compact) + out(capToCeiling(compact, filePath, body.split('\n').findIndex((line) => line.trim() !== '') + 1, 'compact slice')) const bytesSaved = Buffer.byteLength(body, 'utf8') - Buffer.byteLength(compact, 'utf8') if (bytesSaved > 0) recordStat('skill_body:compact', bytesSaved, savedTokensFromBytes(bytesSaved), undefined, name) } else { out(await compactStandIn(name, filePath, body)) } } else { - out(body) + out(capToCeiling(body, filePath, 1, 'skill body')) } await incrementSkillHit(name) } @@ -65,7 +66,7 @@ async function compactStandIn(name: string, filePath: string, body: string): Pro const tree = planHeadingTree(body, bodyBytes, name, filePath) if (tree === null) { emitErr(formatCommandError(`${subject} has ${why} and too few headings to map, so --compact printed the full body.`)) - return body + return capToCeiling(body, filePath, 1, 'skill body') } emitErr(formatCommandError(`${subject} has ${why}; --compact printed its heading tree instead of the body.`)) const firstHeading = extractMarkdownHeadings(body).find((h) => h.level > 1)?.text @@ -73,6 +74,13 @@ async function compactStandIn(name: string, filePath: string, body: string): Pro return `${subject} has ${why}; ${tree.phrase} Load one section with \`token-goat skill-section ${quotedArg(name)} ${headingArg}\`, or drop --compact for the full body.\n\n${tree.sectionsList}` } +/** `text`, which is lines `firstLine` on of the skill file at `filePath`, held to the output ceiling ({@link outputCeilingChars}) and paged past it with a `read "file@A-B"` of the next lines. A skill body is printed whole, and humanizer's ran to 49 KB: past the harness's inline limit it was persisted to a file the agent then spent two more calls reading back. */ +function capToCeiling(text: string, filePath: string, firstLine: number, what: string): string { + const ceiling = outputCeilingChars() + const cut = ceiling === null ? null : capFileLines(text.split('\n'), firstLine, normalizePath(filePath), ceiling, what) + return cut === null ? text : `${cut.kept.join('\n')}\n${cut.notice}` +} + export async function cmdSkillCompact(name: string | undefined, opts: { path?: string; all?: boolean }): Promise { const sessionId = getSessionId() @@ -338,7 +346,7 @@ export async function cmdSkillSection(nameHeading: string, headingArg?: string): throw new CliError(`skill ${echoedValue(skillName)} not found`) } const body = decodeSource(fs.readFileSync(filePath)) - const extracted = extractNamedSection(body, heading) + const extracted = extractNamedSectionAt(body, heading) if (!extracted) { const allHeadings = listSections(filePath) const wanted = stripLower(heading) @@ -352,5 +360,5 @@ export async function cmdSkillSection(nameHeading: string, headingArg?: string): else messages.push(`Try: token-goat outline ${quotedArg(filePath)}`) throw new CliError(messages) } - out(extracted) + out(capToCeiling(extracted.text, filePath, extracted.firstLine, 'skill section')) } diff --git a/src/overflow_guard.ts b/src/overflow_guard.ts index d7ee0916f..67a748f82 100644 --- a/src/overflow_guard.ts +++ b/src/overflow_guard.ts @@ -3,6 +3,7 @@ import { stripAnsiEscapes } from './render/ansi.js' import { safeSlice } from './util.js' import { forClient } from './mcp_client_text.js' +import { canQuoteArg, echoedValue, fencedCommand, quotedArg } from './hint_suggestion_guard.js' import type { ContentClass } from './token_estimate.js' import { classifyContent, guardDivisor } from './token_estimate.js' @@ -66,6 +67,38 @@ export function trimToBudget(text: string, budgetTokens: number, command?: strin return kept.join('\n') + '\n' + forClient(marker) } +/** Characters of a token-goat command's own answer that stay inline in Claude Code's Bash tool. CAPTURE from this machine's session transcripts on Claude Code 2.1.29x: results up to 26,588 characters came back inline, while one whose stdout reached 29,926 was persisted to a file and the model shown a 2 KB preview of it. A token-goat answer past that line cost the agent a second call to read the persisted file, and often a third: `section` returned 52.5 KB, `skill-body humanizer` 49 KB and `bash-output --file --grep` 48 KB. 24,000 leaves room for the continuation notice and for stderr. */ +export const HARNESS_INLINE_CEILING_CHARS = 24_000 + +/** Characters a `token-goat read "file@A-B"` header costs, kept out of a continuation range so the read it names fits the ceiling too. */ +const READ_HEADER_RESERVE_CHARS = 120 + +/** How many leading `lines`, joined with line breaks, fit in `maxChars`. Never fewer than one, so a capped answer is never empty; a single line longer than the ceiling is the caller's to cut. */ +export function linesWithinChars(lines: readonly string[], maxChars: number): number { + let used = 0 + for (let i = 0; i < lines.length; i++) { + used += lines[i]!.length + (i > 0 ? 1 : 0) + if (used > maxChars) return Math.max(1, i) + } + return lines.length +} + +/** `lines`, which are lines `firstLine` on of `file`, cut to the leading lines that fit `maxChars` less `reserve` (what the caller prints around them) with room for a closing notice, which names the next part as a `token-goat read "file@A-B"` whose range fits the ceiling itself. Null when the lines already fit. `what` names the output in the notice ("section", "skill body"). A path no quote mark holds in both shells gets the line range without the command. */ +export function capFileLines(allLines: readonly string[], firstLine: number, file: string, maxChars: number, what: string, reserve = 0): { kept: string[]; lastLine: number; notice: string } | null { + if (allLines.join('\n').length + reserve <= maxChars) return null + // Text ending in a line break splits into a trailing "" that is no line of the file; counting it would page one line past the end. + const lines = allLines.length > 1 && allLines[allLines.length - 1] === '' ? allLines.slice(0, -1) : allLines + const kept = lines.slice(0, linesWithinChars(lines, maxChars - reserve - (300 + file.length))) + const lastLine = firstLine + kept.length - 1 + const finalLine = firstLine + lines.length - 1 + const nextEnd = lastLine + linesWithinChars(lines.slice(kept.length), maxChars - READ_HEADER_RESERVE_CHARS) + const head = `[token-goat: ${what} capped at ${maxChars} chars to stay under the harness's inline output limit; showing lines ${firstLine}-${lastLine} of ${firstLine}-${finalLine}.` + const next = canQuoteArg(file) + ? ` Next part: ${fencedCommand(`token-goat read ${quotedArg(`${file}@${lastLine + 1}-${nextEnd}`)}`)}${nextEnd < finalLine ? `, then on to line ${finalLine}` : ''}.]` + : ` The rest is lines ${lastLine + 1}-${finalLine} of ${echoedValue(file)}.]` + return { kept, lastLine, notice: forClient(head + next) } +} + /** Result of capping a JSON-serializable array to a token budget. */ export interface JsonRowCapResult { items: T[] diff --git a/src/read_commands.ts b/src/read_commands.ts index a37c68c0e..066342576 100644 --- a/src/read_commands.ts +++ b/src/read_commands.ts @@ -29,7 +29,8 @@ import { UNTRUSTED_GITHUB_TAG } from './injection_scan.js' import { redactSecrets } from './secret_redact.js' import { fenceIfScanMatched } from './untrusted_fence.js' import { guardThenFence } from './fence_cap.js' -import { trimToBudget, capJsonRows, type JsonRowCapResult } from './overflow_guard.js' +import { trimToBudget, capJsonRows, HARNESS_INLINE_CEILING_CHARS, type JsonRowCapResult } from './overflow_guard.js' +import { guardDivisor } from './token_estimate.js' import { enclosingSymbol, ALL_SYMBOLS_IN_FILE_LIMIT } from './graph_commands.js' import { MAX_ZIP_INPUT_BYTES, ZipInputTooLargeError } from './zip_bounds.js' import { @@ -492,6 +493,12 @@ export function guardText(text: string, command: string): string { return cfg.overflow_guard.enabled ? trimToBudget(text, cfg.overflow_guard.max_tokens, command) : text } +/** The character ceiling a command that pages its answer holds it to ({@link HARNESS_INLINE_CEILING_CHARS}), or less when `overflow_guard.max_tokens` asks for less, at the guard's own ~3 chars a token; null when `overflow_guard` is off, the same switch that turns {@link guardText} off. */ +export function outputCeilingChars(): number | null { + const cfg = loadConfig() + return cfg.overflow_guard.enabled ? Math.min(HARNESS_INLINE_CEILING_CHARS, cfg.overflow_guard.max_tokens * guardDivisor()) : null +} + /** Cap `text` and fence it under {@link UNTRUSTED_GITHUB_TAG}. A PR's title, description, review comments, and diff are all authorable by anyone who opened the PR or left the comment, so the fence follows that provenance and not the scan result. The scan still runs, purely to name matched pattern(s) in the notice and record the stat. Used by every printed `pr-slice` emit site, each of which has already redacted its text. The `--json` sites use {@link fenceGithubFieldIfMatched} instead -- see the note there. */ function guardAndFenceGithubText(text: string): string { return guardThenFence(text, 'pr-slice', UNTRUSTED_GITHUB_TAG) diff --git a/src/read_section.ts b/src/read_section.ts index 83cdaf0a6..769403859 100644 --- a/src/read_section.ts +++ b/src/read_section.ts @@ -6,6 +6,8 @@ import { eachUnfencedLine } from './markdown_lines.js' import { detectLanguage } from './parser_types.js' import { displaySafeJson } from './paths.js' import { guardText, healStaleIndex, readFileText, recordReadStat, resolveAgainstProjectRoot, sumFileSizes } from './read_commands.js' +import { forClient } from './mcp_client_text.js' +import { capFileLines, linesWithinChars } from './overflow_guard.js' import { namedSpecsMergeable, stripHtmlIdSpelling, parseCrossFileMultiSpec } from './read_spec.js' import { didYouMean, filterSimilarHeadings } from './read_suggest.js' import { listSections, readSection, type SectionResult } from './section_reader.js' @@ -38,6 +40,8 @@ export interface SectionOptions { maxLines?: number /** Keep only the section body lines matching this pattern (regex, literal substring when invalid), each under its nearest sub-heading. */ grep?: string + /** Characters the printed section is held to, paging past them with a `read "file@A-B"` continuation; the CLI passes {@link outputCeilingChars}, and the MCP tool passes none, so its answers are unchanged. A multi-section call splits it across its sections ({@link perSectionOpts}). */ + maxChars?: number } // Keeps the section's own first line plus each matching line, preceded once by the nearest heading line above it, so a surviving line stays attributable to its sub-section. @@ -196,14 +200,29 @@ export function runSection(opts: SectionOptions): { text: string; code: number } const redirectNote = result.redirectedFrom !== undefined ? ` (redirected from: ${echoedValue(result.redirectedFrom)})` : '' - const text = guardText( - `# ${result.heading} — ${filePath}:${result.lineStart}-${lineEnd}${redirectNote}${content.length > 0 ? `\n${content}` : ''}${truncatedNotice}${grepNote}`, - 'heading', - ) + const header = (end: number): string => `# ${result.heading} — ${filePath}:${result.lineStart}-${end}${redirectNote}` + let body = `${header(lineEnd)}${content.length > 0 ? `\n${content}` : ''}${truncatedNotice}${grepNote}` + // Paged, not just cut: a section past the harness's inline limit was persisted to a file the agent then had to read back, so the cut names the read that returns the next lines. A --grep view keeps scattered lines with no file range to page through, so it is cut with a pointer at narrowing instead. + if (opts.maxChars !== undefined && body.length > opts.maxChars) { + if (opts.grep === undefined) { + const cut = capFileLines(result.content.split('\n'), result.lineStart, filePath, opts.maxChars, 'section', header(lineEnd).length + 1) + if (cut !== null) body = `${header(cut.lastLine)}\n${cut.kept.join('\n')}\n${cut.notice}` + } else { + const lines = content.split('\n') + const kept = lines.slice(0, linesWithinChars(lines, opts.maxChars - header(lineEnd).length - 300)) + body = `${header(lineEnd)}\n${kept.join('\n')}\n${forClient(`[token-goat: section capped at ${opts.maxChars} chars to stay under the harness's inline output limit; showing ${kept.length} of the ${lines.length} lines --grep kept. Narrow the pattern, or read a sub-heading.]`)}` + } + } + const text = guardText(body, 'heading') if (opts.suppressStat !== true) recordReadStat(kind, fullSourceBytes, text, heading) return { text, code: 0 } } +/** `opts` for one of `count` sections printed in one answer: an even share of `maxChars`, less what its label line costs, so the whole answer stays under the ceiling and a cut section still names its own next part. */ +function perSectionOpts>(opts: T, count: number): T { + return opts.maxChars === undefined || count < 2 ? opts : { ...opts, maxChars: Math.max(500, Math.floor(opts.maxChars / count) - 200) } +} + export function runSectionMulti( specFilePath: string, resolvedFilePath: string, @@ -246,7 +265,7 @@ export function runSectionMulti( }) } - const sub = runSection({ ...opts, spec: `${specFilePath}::${heading}`, suppressStat: true }) + const sub = runSection({ ...perSectionOpts(opts, headings.length), spec: `${specFilePath}::${heading}`, suppressStat: true }) if (sub.code === 0) anyFound = true if (opts.json === true) { jsonOut[heading] = sub.code === 0 ? (JSON.parse(sub.text) as unknown) : { error: sub.text } @@ -307,7 +326,7 @@ export function runSectionCrossFile(pairs: { file: string; symbol: string }[], o includedByFile.set(resolved, fileIncluded) } - const sub = runSection({ ...opts, spec: `${file}::${heading}`, suppressStat: true }) + const sub = runSection({ ...perSectionOpts(opts, pairs.length), spec: `${file}::${heading}`, suppressStat: true }) if (sub.code === 0) anyFound = true if (opts.json === true) { jsonOut[key] = sub.code === 0 ? (JSON.parse(sub.text) as unknown) : { error: sub.text } @@ -338,7 +357,7 @@ export function runSectionSpecs(specs: string[], opts: Omit = {} const textBlocks: string[] = [] for (const spec of specs) { - const sub = runSection({ ...opts, spec }) + const sub = runSection({ ...perSectionOpts(opts, specs.length), spec }) if (sub.code === 0) anyFound = true if (opts.json === true) jsonOut[spec] = sub.code === 0 ? (JSON.parse(sub.text) as unknown) : { error: sub.text } else textBlocks.push(`${spec}:\n${sub.text}`) diff --git a/src/skill_cache.ts b/src/skill_cache.ts index e9863f1e1..32afd78a0 100644 --- a/src/skill_cache.ts +++ b/src/skill_cache.ts @@ -264,6 +264,11 @@ function parseSectionOrdinal(heading: string): [baseHeading: string, ordinal: nu } export function extractNamedSection(body: string, heading: string): string | null { + return extractNamedSectionAt(body, heading)?.text ?? null +} + +/** {@link extractNamedSection} with the 1-based line of `body` its text starts on, so `skill-section` can page an oversized section with a `read "file@A-B"` of the lines that follow. */ +export function extractNamedSectionAt(body: string, heading: string): { text: string; firstLine: number } | null { if (!body || !heading) return null const [baseHeading, ordinal] = parseSectionOrdinal(heading) @@ -291,8 +296,9 @@ export function extractNamedSection(body: string, heading: string): string | nul if (startIdx === -1) return null - const text = collectSectionBody(lines, startIdx, matchedLevel).join('\n').trim() - return text || null + const bodyLines = collectSectionBody(lines, startIdx, matchedLevel) + const text = bodyLines.join('\n').trim() + return text ? { text, firstLine: startIdx + bodyLines.findIndex((line) => line.trim() !== '') + 1 } : null } export function extractChecklistSection(body: string): string | null { diff --git a/tests/cli.test.ts b/tests/cli.test.ts index d6b6cf3be..6af53517a 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -291,17 +291,18 @@ describe('token-goat CLI', () => { } }, 30000) - it('bash-output --file --grep alone (no --head/--tail) still applies default elision on a large match set', async () => { + // bash-output's --grep with no --max-matches, --head or --tail stops at its default of 100 matches and names the --lines continuation for the rest; it used to show the first 30 and last 80 behind an elision marker whose hint was --full, the one flag that prints every match. + it('bash-output --file --grep alone (no --head/--tail) stays bounded on a large match set and names the rest', async () => { const tmpFile = path.join(os.tmpdir(), `tg-grep-only-elide-${Date.now()}.txt`) const lines = Array.from({ length: 200 }, (_, i) => `MATCH line ${i + 1}`).join('\n') fs.writeFileSync(tmpFile, lines, 'utf8') try { const r = await run(['bash-output', '--file', tmpFile, '--grep', 'MATCH']) expect(r.status).toBe(0) - expect(r.stdout).toContain('...(elided lines ') - expect(r.stdout).toContain('MATCH line 1') - expect(r.stdout).toContain('MATCH line 200') - expect(r.stdout).not.toContain('MATCH line 100') + expect(r.stdout).toContain('MATCH line 1\n') + expect(r.stdout).toContain('MATCH line 100\n') + expect(r.stdout).not.toContain('MATCH line 101\n') + expect(r.stdout).toContain('--grep "MATCH" --lines 101-200') } finally { fs.rmSync(tmpFile, { force: true }) } diff --git a/tests/guards/display_safe_sink_coverage.test.ts b/tests/guards/display_safe_sink_coverage.test.ts index 8d6b2c378..07250dd28 100644 --- a/tests/guards/display_safe_sink_coverage.test.ts +++ b/tests/guards/display_safe_sink_coverage.test.ts @@ -259,6 +259,13 @@ const ESCAPING_NOT_OWED: ReadonlyMap = new Map([ 'Scoped to the serializer only. The same property in formatXmlOutline IS escaped, because ' + "that line is token-goat's own summary with a child count appended rather than a reproduction.", ], + [ + 'cli_skills.ts:extracted.text', + 'the skill section skill-section was asked to print: the payload the reader asked for, printed ' + + 'verbatim as it always was (it read out(extracted) before capToCeiling wrapped it to page an ' + + 'oversized section). capToCeiling only stops at a line boundary and appends token-goat\'s own ' + + 'continuation notice, whose path is quoted by quotedArg.', + ], [ 'session_store_schema.ts:result.text', 'the output text produced by describeTarget: the payload the reader asked for, composed of ' + diff --git a/tests/guards/hint_templates_survive_suggestion_guard.test.ts b/tests/guards/hint_templates_survive_suggestion_guard.test.ts index 2611b8baf..e35dcf315 100644 --- a/tests/guards/hint_templates_survive_suggestion_guard.test.ts +++ b/tests/guards/hint_templates_survive_suggestion_guard.test.ts @@ -376,7 +376,7 @@ const SAFE_BARE: ReadonlyArray<{ key: string; reason: string }> = [ { key: 'answer_router.ts::ANSWER_DELEGATE_LIMIT', reason: 'a numeric constant' }, { key: 'read_git.ts::suggestedRef', reason: '`HEAD~n` for a number n, or the fixed empty-tree hash' }, { key: 'hooks_bash.ts::monitoringHint', reason: 'a flag string from the fixed MONITORING_COMMAND_PATTERNS table (src/hints/lang_patterns.ts)' }, - ...['answer_router.ts::command', 'answer_router.ts::cls.intent', 'bash_extractors.ts::fmt', 'hint_target.ts::format', 'hooks_read.ts::fmt', 'read_spec.ts::command', 'read_spec.ts::commandName', 'read_suggest.ts::command', 'read_symbol.ts::hit.command', 'cli_recall.ts::RECALL_COMMAND[hit.cacheType]', 'hooks_bash.ts::tgRead.sub'].map((key) => ({ key, reason: 'a command name (or `json`/`yaml` prefix) from a fixed set in the source, never user text' })), + ...['answer_router.ts::command', 'cli_cached_output.ts::command', 'answer_router.ts::cls.intent', 'bash_extractors.ts::fmt', 'hint_target.ts::format', 'hooks_read.ts::fmt', 'read_spec.ts::command', 'read_spec.ts::commandName', 'read_suggest.ts::command', 'read_symbol.ts::hit.command', 'cli_recall.ts::RECALL_COMMAND[hit.cacheType]', 'hooks_bash.ts::tgRead.sub'].map((key) => ({ key, reason: 'a command name (or `json`/`yaml` prefix) from a fixed set in the source, never user text' })), { key: 'cli_skills.ts::headingArg', reason: 'a real heading of the skill already passed through quotedArg (kept only when displaySafeText leaves it unchanged and canQuoteArg holds), else the fixed `""` placeholder, as hooks_skill.ts skillSectionArg returns' }, { key: 'cli_install.ts::leftover.flag', reason: 'an install flag such as `--codex` from the fixed leftoverIntegrations table' }, ...['cli_install.ts::removal.label', 'cli_install.ts::leftover.label'].map((key) => ({ key, reason: 'a fixed integration label in a status line (`Removed token-goat Codex CLI integration.`), not a command' })), diff --git a/tests/guards/truncators_are_classified.test.ts b/tests/guards/truncators_are_classified.test.ts index f1e5a2fff..404b86ff7 100644 --- a/tests/guards/truncators_are_classified.test.ts +++ b/tests/guards/truncators_are_classified.test.ts @@ -298,6 +298,23 @@ const TRUNCATOR_CLASSIFICATION: ReadonlyMap = new Map([ reason: 'fold_structure.ts caps FoldRow rows built from Read-tool/cat file-content folding, not tool output.', }, ], + [ + 'capFileLines', + { + bucket: 'read-tool-or-transcript-content-not-in-scope-of-the-tool-output-redaction-boundary', + reason: + 'Pages a section of a file the caller named (read_section.ts runSection) or an installed skill file ' + + '(cli_skills.ts capToCeiling) by whole lines at the output ceiling. Both are the user\'s own files, printed ' + + 'unredacted by section and skill-body before this cap existed; it only stops sooner, at a line boundary.', + }, + ], + [ + 'capToCeiling', + { + bucket: 'read-tool-or-transcript-content-not-in-scope-of-the-tool-output-redaction-boundary', + reason: 'cli_skills.ts\'s one caller of capFileLines, over a skill file skill-body and skill-section already print whole.', + }, + ], [ 'clipToDeliveryCap', { diff --git a/tests/helpers/matrix_cases.ts b/tests/helpers/matrix_cases.ts index c7eeb958d..809a2da14 100644 --- a/tests/helpers/matrix_cases.ts +++ b/tests/helpers/matrix_cases.ts @@ -1384,6 +1384,13 @@ export const cases: Record void | Promise> = { const r = run(['bash-output', '--file', 'pkg.json']) expect(r.status, r.stderr).toBe(0) expect(r.stdout).toContain('3.2.1') + // An answer past the harness's inline limit is held under it, closes its fence, and names the --lines that return the rest. + const big = path.join(mkIsolated('tg-matrix-bo-'), 'big.txt') + fs.writeFileSync(big, `${Array.from({ length: 600 }, (_, i) => `row ${i + 1} ${'filler '.repeat(14)}`).join('\n')}\n`) + const capped = run(['bash-output', '--file', big, '--head', '600']) + expect(capped.status, capped.stderr).toBe(0) + expect(capped.stdout.length).toBeLessThanOrEqual(24_000) + expect(capped.stdout).toMatch(/<\/untrusted-tool-output>\n\[token-goat: output capped at 24000 chars[^\n]*--lines \d+-600`\]/) }, 'web-output': () => { // Reachability: process-local cache is always empty in a fresh process, so a bogus id is a graceful miss (exit 1), not an "unknown command" / crash. diff --git a/tests/output_ceiling_continuation.test.ts b/tests/output_ceiling_continuation.test.ts new file mode 100644 index 000000000..16f5f6402 --- /dev/null +++ b/tests/output_ceiling_continuation.test.ts @@ -0,0 +1,181 @@ +/** Regression: token-goat's own answers ran past Claude Code's inline Bash limit (measured on 2.1.29x: 26,588 chars came back inline, a 29,926-char stdout was persisted with a 2 KB preview), so `section` (52.5 KB), `skill-body humanizer` (49 KB) and `bash-output --file --grep` (48 KB) were each persisted to a file the agent then spent more calls reading back. section, skill-body, skill-section, bash-output, web-output and mcp-output now hold their answer to HARNESS_INLINE_CEILING_CHARS at a line boundary, close any fence first, and name a continuation command. Each case runs the built bundle, then runs the continuation it printed and checks that it returns the next lines. */ +import { spawnSync } from 'node:child_process' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' + +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { BUNDLE } from './helpers/bundle.js' +import { storeMcpOutput } from '../src/mcp_cache.js' +import { HARNESS_INLINE_CEILING_CHARS } from '../src/overflow_guard.js' +import { shlexSplit } from '../src/tool_filters/helpers.js' +import { storeWebOutput } from '../src/web_cache.js' + +// HAND-DERIVED: numbered lines of ~110 chars, so a few hundred of them clear the ceiling and every line names its own position. +const padded = (label: string, i: number): string => `${label} ${String(i).padStart(4, '0')} ${'filler '.repeat(14)}end` + +let root: string +let env: NodeJS.ProcessEnv + +function run(args: string[]): { status: number | null; stdout: string; stderr: string } { + const r = spawnSync(process.execPath, [BUNDLE, ...args], { cwd: root, env, encoding: 'utf8', timeout: 60000 }) + return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' } +} + +/** The command a capped answer names for its next part, run through the bundle. */ +function runContinuation(stdout: string): { status: number | null; stdout: string; stderr: string; command: string } { + const command = /Next part: `(token-goat [^`\n]+)`/.exec(stdout)?.[1] + expect(command, `no continuation named in: ${stdout.slice(-400)}`).toBeDefined() + const [, ...args] = shlexSplit(command!) + return { ...run(args), command: command! } +} + +function expectCapped(stdout: string): void { + expect(stdout.length).toBeLessThanOrEqual(HARNESS_INLINE_CEILING_CHARS) + expect(stdout).toContain(`capped at ${HARNESS_INLINE_CEILING_CHARS} chars`) +} + +beforeAll(() => { + root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), 'tg-output-ceiling-'))) + fs.writeFileSync(path.join(root, 'big.md'), `# Big\n\n## Long part\n\n${Array.from({ length: 500 }, (_, i) => padded('doc', i + 1)).join('\n')}\n\n## After\n\ntail\n`) + const skillDir = path.join(root, 'home', '.claude', 'skills', 'tgbigskill') + fs.mkdirSync(skillDir, { recursive: true }) + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), `# Big Skill\n\n## Long part\n\n${Array.from({ length: 500 }, (_, i) => padded('skill', i + 1)).join('\n')}\n\n## After\n\ntail\n`) + fs.writeFileSync(path.join(root, 'tool-output.txt'), `${Array.from({ length: 1000 }, (_, i) => padded('row', i + 1)).join('\n')}\n`) + env = { + ...process.env, + HOME: path.join(root, 'home'), + USERPROFILE: path.join(root, 'home'), + } +}) + +afterAll(() => { + fs.rmSync(root, { recursive: true, force: true }) +}) + +describe('section', () => { + it('pages a section past the ceiling with a read of the next lines', () => { + const r = run(['section', 'big.md::Long part']) + expect(r.status, r.stderr).toBe(0) + expectCapped(r.stdout) + const lastShown = Number(/^# Long part — big\.md:3-(\d+)$/m.exec(r.stdout)?.[1]) + expect(r.stdout).toContain(`doc ${String(lastShown - 4).padStart(4, '0')} `) + + const next = runContinuation(r.stdout) + expect(next.command).toMatch(/^token-goat read "big\.md@/) + expect(next.status, next.stderr).toBe(0) + expect(next.stdout.length).toBeLessThanOrEqual(HARNESS_INLINE_CEILING_CHARS) + expect(next.stdout.split('\n')[1]).toBe(padded('doc', lastShown - 3)) + }) +}) + +describe('skill-body and skill-section', () => { + it('skill-body pages a body past the ceiling with a read of the skill file', () => { + const r = run(['skill-body', 'tgbigskill']) + expect(r.status, r.stderr).toBe(0) + expectCapped(r.stdout) + const shown = r.stdout.split('\n') + const lastLine = shown[shown.indexOf(shown.find((l) => l.startsWith('[token-goat: skill body capped'))!) - 1]! + + const next = runContinuation(r.stdout) + expect(next.status, next.stderr).toBe(0) + expect(next.stdout.length).toBeLessThanOrEqual(HARNESS_INLINE_CEILING_CHARS) + const lastIndex = Number(/^skill (\d+) /.exec(lastLine)?.[1]) + expect(next.stdout.split('\n')[1]).toBe(padded('skill', lastIndex + 1)) + }) + + it('skill-section pages a section past the ceiling the same way', () => { + const r = run(['skill-section', 'tgbigskill', 'Long part']) + expect(r.status, r.stderr).toBe(0) + expectCapped(r.stdout) + const lastIndex = Number(/^skill (\d+) .*\n\[token-goat: skill section capped/m.exec(r.stdout)?.[1]) + + const next = runContinuation(r.stdout) + expect(next.status, next.stderr).toBe(0) + expect(next.stdout.split('\n')[1]).toBe(padded('skill', lastIndex + 1)) + }) +}) + +describe('the cached-output recalls', () => { + /** The last numbered row inside the fence and the first one of its continuation. */ + const rowNumbers = (stdout: string, label: string): number[] => [...stdout.matchAll(new RegExp(`^${label} (\\d+) `, 'gm'))].map((m) => Number(m[1])) + + it('bash-output --head past the ceiling closes the fence, then names --lines for the rest', () => { + const r = run(['bash-output', '--file', path.join(root, 'tool-output.txt'), '--head', '400']) + expect(r.status, r.stderr).toBe(0) + expectCapped(r.stdout) + expect(r.stdout.indexOf('')).toBeLessThan(r.stdout.indexOf('[token-goat: output capped')) + const shown = rowNumbers(r.stdout, 'row') + + const next = runContinuation(r.stdout) + expect(next.command).toContain(`--lines ${shown.at(-1)! + 1}-400`) + expect(next.status, next.stderr).toBe(0) + expect(rowNumbers(next.stdout, 'row')[0]).toBe(shown.at(-1)! + 1) + }) + + it('bash-output --grep keeps the pattern in its continuation and resumes after the last match shown', () => { + // Lines of ~370 chars: the 100 matches a bare --grep prints run past the ceiling, the shape of the 48 KB `--grep 'listing/[0-9]+'` recall. + const file = path.join(root, 'long-matches.txt') + fs.writeFileSync(file, `${Array.from({ length: 300 }, (_, i) => `hit ${String(i + 1).padStart(4, '0')} listing/${i} ${'filler '.repeat(50)}`).join('\n')}\n`) + const r = run(['bash-output', '--file', file, '--grep', 'listing/[0-9]+']) + expect(r.status, r.stderr).toBe(0) + expectCapped(r.stdout) + const shown = rowNumbers(r.stdout, 'hit') + expect(shown.length).toBeGreaterThan(0) + + const next = runContinuation(r.stdout) + expect(next.command).toContain(`--grep "listing/[0-9]+" --lines ${shown.at(-1)! + 1}-300`) + expect(next.status, next.stderr).toBe(0) + expect(rowNumbers(next.stdout, 'hit')[0]).toBe(shown.at(-1)! + 1) + }) + + it('bash-output --grep with no --max-matches stops at 100 matches and names the rest', () => { + const file = path.join(root, 'short-matches.txt') + fs.writeFileSync(file, `${Array.from({ length: 250 }, (_, i) => `MATCH ${i + 1}`).join('\n')}\n`) + const r = run(['bash-output', '--file', file, '--grep', 'MATCH']) + expect(r.status, r.stderr).toBe(0) + expect(r.stdout).toContain('MATCH 100\n') + expect(r.stdout).not.toContain('MATCH 101\n') + expect(r.stdout).toContain('showing the first 100 of 250 matching lines') + + const next = runContinuation(r.stdout) + expect(next.command).toContain('--lines 101-250') + expect(next.stdout).toContain('MATCH 101\n') + expect(next.stdout).toContain('MATCH 200\n') + }) + + it('bash-output --full still prints everything it was asked for', () => { + const r = run(['bash-output', '--file', path.join(root, 'tool-output.txt'), '--full']) + expect(r.status, r.stderr).toBe(0) + expect(r.stdout).toContain(padded('row', 1000)) + expect(r.stdout).not.toContain('capped at') + }) + + it('web-output pages a cached page past the ceiling by its id', () => { + const id = storeWebOutput('https://example.com/output-ceiling', Array.from({ length: 400 }, (_, i) => padded('web', i + 1)).join('\n')) + const r = run(['web-output', id, '--head', '400']) + expect(r.status, r.stderr).toBe(0) + expectCapped(r.stdout) + const shown = rowNumbers(r.stdout, 'web') + + const next = runContinuation(r.stdout) + expect(next.command).toContain(`token-goat web-output ${id}`) + expect(next.status, next.stderr).toBe(0) + expect(rowNumbers(next.stdout, 'web')[0]).toBe(shown.at(-1)! + 1) + }) + + it('mcp-output pages a cached MCP result past the ceiling by its id', () => { + const id = storeMcpOutput('output-ceiling-session', 'mcp__probe__list', { q: 'all' }, Array.from({ length: 400 }, (_, i) => padded('mcp', i + 1)).join('\n')) + expect(id).not.toBeNull() + const r = run(['mcp-output', id!, '--head', '400']) + expect(r.status, r.stderr).toBe(0) + expectCapped(r.stdout) + const shown = rowNumbers(r.stdout, 'mcp') + + const next = runContinuation(r.stdout) + expect(next.command).toContain(`token-goat mcp-output ${id!}`) + expect(next.status, next.stderr).toBe(0) + expect(rowNumbers(next.stdout, 'mcp')[0]).toBe(shown.at(-1)! + 1) + }) +}) From 3872e24336bf3f76a092d6206b848d1ed064c6fc Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 05:53:25 -0500 Subject: [PATCH 11/92] fix(hooks): let cat of a small file run, and name a command that runs for a file with nothing to name A whole-file cat, bat, type or Get-Content was refused at any size while the Read tool took the same file, and across the sampled refusals of 0.8-12.6 KB notes the agent read the file whole anyway six times out of eight. Below the Read gate's own threshold for the type (hints/file_type_handler.ts, 20 KB for every type the Bash hook checks) the dump now runs as written, as does a sed or awk range over the whole file; an .env file keeps the refusal at any size, and a multi-file cat is refused for its files past the floor only. The whole-file ladder printed a placeholder whenever no name was found: section "file::SectionHeading" for a heading-less doc, config-get "file" "KEY_NAME" for an env or flat config file, a grep for or , read "file::table_name" for SQL, all of which exit 1 or need editing first. Each now leads with token-goat read "file::1-80" through one helper, which hint_target.ts's sectionOrRangeCommand now shares. Tests that asserted a placeholder, or used a fixture smaller than the new floor to provoke a refusal, are updated to the new command or padded past the floor. --- README.md | 2 +- src/bash_extractors.ts | 17 ++- src/hint_suggestion_guard.ts | 17 ++- src/hint_target.ts | 7 +- src/hooks_bash.ts | 40 ++++- tests/bash_cat_read_gate_floor.test.ts | 139 ++++++++++++++++++ .../bash_deny_names_a_runnable_target.test.ts | 50 +++++-- tests/hint_target.test.ts | 8 +- ...rive_letter_key_open_its_wsl_mount.test.ts | 7 +- tests/hooks_bash.test.ts | 54 +++---- tests/rewrite_permission_e2e.test.ts | 3 +- tests/subagent_compaction_epoch.test.ts | 3 +- tests/suggestion_guard_quote_scan.test.ts | 5 +- 13 files changed, 285 insertions(+), 67 deletions(-) create mode 100644 tests/bash_cat_read_gate_floor.test.ts diff --git a/README.md b/README.md index 649466626..2fa66f98d 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ The fastest way to reduce AI token costs is fixing these six, not writing shorte | Same large skill invoked twice in a session | PreToolUse hook blocks the reload; serves cached compact (~400 tokens) via `additionalContext` instead of the full 40–65k body. Allows the reload if compaction fired since the last load | | Skill invoked with `first_load_compact=true` and `` present | First load also blocked; only the curated compact section is served. Full body available via `token-goat skill-body ` on demand | | Same docs URL fetched twice in a session with the same question | Re-fetch blocked at any context pressure; cached body available via `token-goat web-output `. Keyed on the URL and the prompt together, since a WebFetch answer is specific to the question asked, so the same page fetched with a different question is left alone | -| `cat src/auth.py` or `Get-Content module.py` run via Bash | Pre-Bash hook detects whole-file reads of indexed source files and suggests `token-goat read "file::Symbol"`, `skeleton`, or `section` — covers `cat`, `bat`, `type`, PowerShell `Get-Content`/`gc` | +| `cat src/auth.py` or `Get-Content module.py` run via Bash | Pre-Bash hook detects whole-file reads of indexed source files and suggests `token-goat read "file::Symbol"`, `skeleton`, or `section` — covers `cat`, `bat`, `type`, PowerShell `Get-Content`/`gc`. A file under the size the Read gate hands over whole (20 KB for every type checked here) runs as written, except a `.env` file. A file with no heading, key or table to name gets `token-goat read "file::1-80"`, its first lines | | `rg pattern src/` or `grep -rn` run via Bash (first time) | Pre-Bash hook suggests `token-goat symbol ` and `token-goat semantic ""` as indexed alternatives to a full directory walk | | `rg "^def" src/file.py` or `grep "class " module.ts` — structural search on a single source file | Pre-Bash hook redirects to `token-goat skeleton "file"` or `outline "file"` — all symbols with line numbers, no full-file read | | `rg` or `grep` run twice with the same pattern | Pre-Bash dedup hint fires on repeated `rg`/`grep`/`ag` calls the same way it fires on the native Grep tool; repeat searches return a cached match-count hint instead of re-running | diff --git a/src/bash_extractors.ts b/src/bash_extractors.ts index 632bae11d..3bbd781db 100644 --- a/src/bash_extractors.ts +++ b/src/bash_extractors.ts @@ -9,7 +9,7 @@ import { hasBareBackgroundOrNewline, hasUnquotedOperator, isRedirectAmpersand } import { dropFileLineRangesIfChanged, getFileLineRanges } from './session.js' import { escapeRegExp } from './util.js' import { FALSY_ENV_VALUES } from './env.js' -import { leadWithCommand, docSectionHint, grepLinesHint, configGetCommand, quotedArg, quotedArgs } from './hint_suggestion_guard.js' +import { leadWithCommand, docSectionHint, headLinesHint, configGetCommand, quotedArg, quotedArgs } from './hint_suggestion_guard.js' import type { HintTarget } from './hint_target.js' // Defined beside preToolPathDeclined, the rule it applies, so hint_target.ts can gate a path without pulling this module into the core bundle. @@ -35,34 +35,37 @@ function structuredDataHint(hintPath: string, target: string | null, reason = '' return leadWithCommand(query(path), 'to read that value, or `' + outline + '` for every top-level key with its type and size', reason) } -/** The SQL branch every whole-file read shares: `read` returns one CREATE TABLE / CREATE TYPE block by the name the SQL adapter indexes it under. It used to print `section "file::table_name"`, which cannot run at all -- `section` finds markdown/TOML/INI headers and exits 1 with "has no headings" on a .sql file (measured 2026-09-24 against the built binary). */ +/** The SQL branch every whole-file read shares: `read` returns one CREATE TABLE / CREATE TYPE block by the name the SQL adapter indexes it under. It used to print `section "file::table_name"`, which cannot run at all -- `section` finds markdown/TOML/INI headers and exits 1 with "has no headings" on a .sql file (measured 2026-09-24 against the built binary). With no table to name, `read "file::table_name"` exits 1 as well, so the file's first lines are offered instead. */ export function sqlTableHint(hintPath: string, target: HintTarget, reason = ''): string { + if (!target.real) return headLinesHint(hintPath, reason) return leadWithCommand('token-goat read ' + quotedArg(hintPath + '::' + target.name), 'to pull one CREATE TABLE / CREATE TYPE block', reason) } /** Shared non-SQL surgical-read hint ladder for whole-file dump commands (`cat`, a PowerShell `Get-Content` wrapper, `wsl cat`) -- each caller handles its own SQL branch and passes its lead-in as `reason`, which follows the command. */ // Every caller passes a hintPath already through displaySafePath, because the path here comes out of the shell command's own arguments and so is whatever a repository named its files, while the hint is delivered on the context channel, which unlike the deny channel neither fences its payload nor escapes the markers token-goat speaks in. Sanitizing at the fifteen assignment sites rather than at the thirty interpolations below is what keeps that invariant checkable, and it is the identity function on every path that does not contain a marker or a control character, so the index lookups keyed on the same value are unaffected for any real file. export function surgicalHintFor(hintPath: string, isEnv: boolean, isConfig: boolean, isDoc: boolean, isXml: boolean, target: HintTarget, reason = ''): string { - // The whole-file branches are a hard deny, so the command they print is the agent's only next move, and the placeholders they used to print do not run: verified against the built binary on 2026-09-21, `token-goat section "CHANGELOG.md::SectionHeading"` exits 1 with "Section 'SectionHeading' not found" and `token-goat config-get "package.json" KEY_NAME` exits 1 with "Key 'KEY_NAME' not found". `target.name` is a name that file really holds whenever `target.real`, so the command runs verbatim; otherwise it is hint_target.ts's placeholder. + // The whole-file branches are a hard deny, so the command they print is the agent's only next move, and the placeholders they used to print do not run: verified against the built binary on 2026-09-21, `token-goat section "CHANGELOG.md::SectionHeading"` exits 1 with "Section 'SectionHeading' not found" and `token-goat config-get "package.json" KEY_NAME` exits 1 with "Key 'KEY_NAME' not found". `target.name` is a name that file really holds whenever `target.real`, so the command runs verbatim; otherwise the hint leads with a command that needs no name: an outline where the format has one for any file, else a read of the first lines, which runs on every text file. // // A config file's name goes where its format takes it: a JSON/YAML key to the query command, a TOML/INI table to `section` (measured, `config-get "cfg.toml" tool` exits 1 on a table while `section "cfg.toml::tool"` returns it), a .properties key to config-get. A JSON property is never put in a `section` slot: `token-goat section "package.json::name"` exits 1. const outline = outlineCommand(hintPath) if (isXml) return leadWithCommand('token-goat xml-outline ' + quotedArg(hintPath), 'to inspect structure, or `token-goat xml-query ' + quotedArgs(hintPath, '').join(' ') + '` to query specific nodes', reason) - if (isEnv) return leadWithCommand(configGetCommand(hintPath, target.name), 'to read a specific variable', reason) - // .properties has no outline extractor (measured 2026-10-03: `outline app.properties` exits 1), so it gets config-get for a key it holds and a grep otherwise. + if (isEnv) return target.real ? leadWithCommand(configGetCommand(hintPath, target.name), 'to read a specific variable', reason) : headLinesHint(hintPath, reason) + // .properties has no outline extractor (measured 2026-10-03: `outline app.properties` exits 1), so it gets config-get for a key it holds and its first lines otherwise. if (isConfig && /\.properties$/i.test(hintPath)) { return target.real ? leadWithCommand(configGetCommand(hintPath, target.name), 'to read a specific value', reason) - : grepLinesHint('', hintPath, reason) + : headLinesHint(hintPath, reason) } if (isConfig) { const structured = structuredDataHint(hintPath, target.real ? target.name : null, reason) if (structured !== null) return structured + // A flat .conf holds no name outline can list (measured: `outline a.conf` exits 1 with "No indexed symbols found"), and a file with no name found has none for it either. + if (!target.real) return headLinesHint(hintPath, reason) return target.slice === 'key' ? leadWithCommand(configGetCommand(hintPath, target.name), 'to read a specific value, or `' + outline + '` for every key with line ranges', reason) : leadWithCommand('token-goat section ' + quotedArg(hintPath + '::' + target.name), 'to read one table, or `' + outline + '` for every key with line ranges', reason) } - if (isDoc) return docSectionHint(hintPath, target.name, reason) + if (isDoc) return target.real ? docSectionHint(hintPath, target.name, reason) : headLinesHint(hintPath, reason) // A source file with no nameable symbol leads with outline, which always runs, rather than a `read "file::SymbolName"` that never does. return target.real ? leadWithCommand('token-goat read ' + quotedArg(hintPath + '::' + target.name), 'to read one function or class, or `' + outline + '` for all of them', reason) diff --git a/src/hint_suggestion_guard.ts b/src/hint_suggestion_guard.ts index 0cf916e4a..c42bd5387 100644 --- a/src/hint_suggestion_guard.ts +++ b/src/hint_suggestion_guard.ts @@ -281,14 +281,27 @@ export function grepLinesHint(pattern: string, shownPath: string, reason = ''): return leadWithCommand('token-goat grep ' + quotedPattern + ' ' + quotedPath + ' -C 3', 'to read the matching lines', reason) } +/** Lines a {@link headLinesCommand} range read covers, when the file has nothing in it to name. */ +export const HEAD_LINES_READ = 80 + +/** The command that reads the first {@link HEAD_LINES_READ} lines of `shownPath`, which runs on every text file: one with no heading to name (`section` exits 1 on it, and `::HeadingName` is a name no file holds), and one of a type neither `section` nor `outline` serves. */ +export function headLinesCommand(shownPath: string): string { + return 'token-goat read ' + quotedArg(shownPath + '::1-' + HEAD_LINES_READ) +} + +/** The hint leading with {@link headLinesCommand}, for a file a hint has no part of to name. */ +export function headLinesHint(shownPath: string, reason = ''): string { + return leadWithCommand(headLinesCommand(shownPath), 'to read its first ' + HEAD_LINES_READ + ' lines', reason) +} + /** `config-get` for one key of one file, the key quoted with the path's mark ({@link quotedArgs}), the form {@link stripUnsafeSuggestions} checks: a bare key sits outside the quotes, where the relay judges it only by its outside-quote allowlist and a shell splits it at a space. hint_target.ts refuses a key holding `"`, so these quotes always hold. */ export function configGetCommand(shownPath: string, key: string): string { return 'token-goat config-get ' + quotedArgs(shownPath, key).join(' ') } -/** The hint for one section of a prose document: `section` with the `outline` alternative only where it runs, and a plain grep for a file type neither serves. */ +/** The hint for one section of a prose document: `section` with the `outline` alternative only where it runs, and a read of the first lines for a file type neither serves, whose grep needed a pattern the hint could only print as ``. */ export function docSectionHint(shownPath: string, heading: string, reason = ''): string { const nav = docNavigation(shownPath) - if (!nav.section) return grepLinesHint('', shownPath, reason) + if (!nav.section) return headLinesHint(shownPath, reason) return leadWithCommand('token-goat section ' + quotedArg(shownPath + '::' + heading), 'to read one section' + (nav.outline ? ', or ' + fencedCommand('token-goat outline ' + quotedArg(shownPath)) + ' for every heading with line ranges' : ''), reason) } diff --git a/src/hint_target.ts b/src/hint_target.ts index 6d00b591f..886b99b97 100644 --- a/src/hint_target.ts +++ b/src/hint_target.ts @@ -8,7 +8,7 @@ import { indexMatchesDisk } from './index_freshness.js' import { commandPathIsTouchable } from './vscode_path_gate.js' import { extractMarkdownHeadings, formatHeadingTreeParts, type MarkdownHeading } from './hints/markdown_hints.js' import { extractQuickSymbolSamples } from './hooks_read.js' -import { stripUnsafeSuggestions, leadWithCommand, grepLinesHint, configGetCommand, quotedArg, quotedArgs } from './hint_suggestion_guard.js' +import { stripUnsafeSuggestions, leadWithCommand, grepLinesHint, configGetCommand, headLinesCommand, quotedArg, quotedArgs } from './hint_suggestion_guard.js' import { getCompactedAt, markHintShown, wasHintShown } from './session.js' import { shortFingerprint } from './fingerprint.js' import { sessionStateKey, type HookEvent } from './hook_registry.js' @@ -280,12 +280,9 @@ export function sliceCommand(shownPath: string, target: HintTarget): string { } } -/** Lines a {@link sectionOrRangeCommand} range read starts with when the file has no heading to name. */ -const NO_HEADING_RANGE_LINES = 80 - /** The command that reads one part of `shownPath` when the hint is about a section: `section` on `heading` when it is a real heading, else a read of the file's first lines. A file with no headings (`section` exits 1 on it, and `::HeadingName` is a name no file holds) still gets a command that runs. */ export function sectionOrRangeCommand(shownPath: string, heading: HintTarget): string { - return heading.real ? sliceCommand(shownPath, heading) : 'token-goat read ' + quotedArg(shownPath + '::1-' + NO_HEADING_RANGE_LINES) + return heading.real ? sliceCommand(shownPath, heading) : headLinesCommand(shownPath) } /** Lock files whose bytes are a JSON or YAML document under an extension the format commands would not infer. */ diff --git a/src/hooks_bash.ts b/src/hooks_bash.ts index 2843be4dd..da34d8a6b 100644 --- a/src/hooks_bash.ts +++ b/src/hooks_bash.ts @@ -89,10 +89,26 @@ import { compressionOptedOut, } from './bash_extractors.js' import { countTextLines, SLICE_ESTIMATE_SCAN_CAP_BYTES } from './hooks_read_slice.js' +import { FILE_TYPE_THRESHOLDS } from './hints/file_type_handler.js' /** Token budget for an inline interpreter file read run under the wrapper: room for one registry entry or config block printed whole, well under a whole-file dump. */ const INTERPRETER_READ_TOKEN_CAP = 2000 +/** The size below which the Read gate hands a file of each type a whole-file refusal covers over whole, from hints/file_type_handler.ts: its own entry for the types it dispatches, the plain-text one for prose and source, which that table leaves to gates that do not refuse a first read this size. */ +const READ_GATE_FLOOR_BYTES: Readonly> = { + json: FILE_TYPE_THRESHOLDS.json, + yaml: FILE_TYPE_THRESHOLDS.yaml, + yml: FILE_TYPE_THRESHOLDS.yaml, + xml: FILE_TYPE_THRESHOLDS.xml, + dtsx: FILE_TYPE_THRESHOLDS.dtsx, + ampkg: FILE_TYPE_THRESHOLDS.ampkg, + xaml: FILE_TYPE_THRESHOLDS.xaml, +} + +function readGateFloorBytes(filePath: string): number { + return READ_GATE_FLOOR_BYTES[path.extname(filePath).slice(1).toLowerCase()] ?? FILE_TYPE_THRESHOLDS.txt +} + // Runs an inline interpreter file read under a token cap instead of refusing it. Many are projections, `json.load(open('r.json'))['118615']`, that print a few lines no token-goat command beats by a second round trip; refused, each cost the agent a retry and often a second refusal before it found a spelling the hook let through. A projection now passes whole, and a whole-file dump comes back capped with its recall id and the same surgical command the refusal named. The refusal stays wherever the wrapper cannot run: compression off, a pipeline or chain, a background `&`, a script past the command-line ceiling, VS Code. The hint passes the suggestion guard here because the relay only guards a refusal: printed after the capped output, a path that broke its quoting would reach the model unguarded. function cappedInterpreterRead(event: HookEvent, rawCmd: string, cmd: string, hint: string, denial: string): HookOutput { return maybeCompressRewrite(event, rawCmd, cmd, { maxTokens: INTERPRETER_READ_TOKEN_CAP, hint: stripUnsafeSuggestions(hint) }) ?? denyOutput(denial) @@ -244,6 +260,16 @@ function preBashHandlerInner(event: HookEvent): HookOutput { for (const p of typeof paths === 'string' ? [paths] : paths) logSuppressedDetection('bash_redirect', event.sessionId, p) return passOutput() } + // A whole-file dump of a file the Read gate would hand over whole is not refused: of eight measured `cat` refusals of 0.8-12.6 KB notes, six were followed by a Read or sed of the same file, a turn spent to arrive at the same bytes, and the command the refusal named was run none of the eight times. An .env file keeps the refusal at any size, as the Read gate refuses its re-read at any size: what it holds is better asked for one variable at a time. Stats the file, so it is gated like every other path this hook touches before the command is approved. + const underReadGateFloor = (hintPath: string, isEnv: boolean): boolean => { + if (isEnv || !commandPathIsTouchable(hintPath, event)) return false + try { + const st = statSync(hostPathOfTypedPath(hintPath, hintCwd)) + return st.isFile() && st.size < readGateFloorBytes(hintPath) + } catch { + return false + } + } // The size half of the condition each of the five `token-goat bash-output ` recall branches below repeated, plus the decline record none of them kept: a candidate that clears the dedup minimum but fails the savings floor was refused on price, which is a decision and has to leave a trace for the same reason declineUnpriced does. Callers still test their own id and entry for null, both to keep TypeScript's narrowing inside the branch and because neither absence is this gate declining anything -- no cached output, or an entry too stale to offer, never reached a price comparison, and recording it would credit the gate with refusing work it never had. The correlator is the cache id, which is what classifyBashHint reads back out of a recall hint that did get shown, so the declined and shown rows key alike. const recallWorthShowing = (outputId: string, entry: { sizeBytes: number }): boolean => { if (entry.sizeBytes < loadConfig().hints.bash_dedup_min_bytes) return false @@ -365,7 +391,7 @@ function preBashHandlerInner(event: HookEvent): HookOutput { dropFileLineRangesIfChanged(sedDedupKey) // A range from line 1 to the last line is `cat` spelled another way, and gets `cat`'s answer: pricing it against a surgical read of the same lines finds no saving, since that read is the whole file too, so `awk 'NR>=1 && NR<=324'` over a 324-line, 54KB skill passed with no word while `cat` of it was refused. Checked before the range is recorded, so a refusal does not count the lines as served. const whole = singleLineRangeRead === null ? null : wholeFileRange(filePath, hintPath, hintCwd, ranges, tool) - if (whole !== null && findRangeOverlap(getFileLineRanges(sedDedupKey), whole.start, whole.end) === null) { + if (whole !== null && findRangeOverlap(getFileLineRanges(sedDedupKey), whole.start, whole.end) === null && !underReadGateFloor(hintPath, whole.cat.isEnv)) { recordStat('session_hint', 0, 0) if (whole.cat.isSql) return pathHint(hintPath, sqlTableHint(hintPath, targetFor(hintPath, 'table'), whole.reason)) const hint = surgicalHintFor(hintPath, whole.cat.isEnv, whole.cat.isConfig, whole.cat.isDoc, whole.cat.isXml, targetFor(hintPath, whole.cat.isEnv ? 'key' : undefined), whole.reason) @@ -452,6 +478,7 @@ function preBashHandlerInner(event: HookEvent): HookOutput { if (catResult !== null) { const { filePath, isDoc, isEnv, isConfig, isSql, isXml, cmd0, advisoryOnly } = catResult const hintPath = displaySafePath(cdStripped ? resolveCdHintPath(rawCmd, filePath, hintCwd) : filePath) + if (underReadGateFloor(hintPath, isEnv)) return declineUnpriced(hintPath) recordStat('session_hint', 0, 0) const reason = '`' + cmd0 + '` loads the entire file into context.' if (isSql) return pathHint(hintPath, sqlTableHint(hintPath, targetFor(hintPath, 'table'), reason)) @@ -460,13 +487,16 @@ function preBashHandlerInner(event: HookEvent): HookOutput { return cdStripped || advisoryOnly ? pathHint(hintPath, hint) : denyOutput(hint) } - const catMulti = extractCatFilesMulti(cmd) - if (catMulti !== null) { + const catMultiAll = extractCatFilesMulti(cmd) + if (catMultiAll !== null) { + const shownOf = (filePath: string): string => displaySafePath(cdStripped ? resolveCdHintPath(rawCmd, filePath, hintCwd) : filePath) + const catMulti = catMultiAll.filter((m) => !underReadGateFloor(shownOf(m.filePath), m.isEnv)) + if (catMulti.length === 0) return declineUnpriced(catMultiAll.map((m) => shownOf(m.filePath))) recordStat('session_hint', 0, 0) const cmd0 = catMulti[0]!.cmd0 // One command-led line per file, each carrying its own path, so the first thing in the message is a command that runs. const perPath = catMulti.map(({ filePath, isDoc, isEnv, isConfig, isSql, isXml }) => { - const hintPath = displaySafePath(cdStripped ? resolveCdHintPath(rawCmd, filePath, hintCwd) : filePath) + const hintPath = shownOf(filePath) return isSql ? sqlTableHint(hintPath, targetFor(hintPath, 'table')) : surgicalHintFor(hintPath, isEnv, isConfig, isDoc, isXml, targetFor(hintPath, isEnv ? 'key' : undefined)) @@ -479,6 +509,7 @@ function preBashHandlerInner(event: HookEvent): HookOutput { if (psGetContentResult !== null) { const { filePath, isDoc, isEnv, isConfig, isSql, isXml } = psGetContentResult const hintPath = displaySafePath(cdStripped ? resolveCdHintPath(rawCmd, filePath, hintCwd) : filePath) + if (underReadGateFloor(hintPath, isEnv)) return declineUnpriced(hintPath) recordStat('session_hint', 0, 0) const lead = '`Get-Content` via a `powershell -Command` wrapper bypasses read hooks and loads the entire file into context.' if (isSql) { @@ -493,6 +524,7 @@ function preBashHandlerInner(event: HookEvent): HookOutput { if (wslCatResult !== null) { const { filePath, isDoc, isEnv, isConfig, isSql, isXml } = wslCatResult const hintPath = displaySafePath(cdStripped ? resolveCdHintPath(rawCmd, filePath, hintCwd) : filePath) + if (underReadGateFloor(hintPath, isEnv)) return declineUnpriced(hintPath) recordStat('session_hint', 0, 0) const reason = '`cat` loads the entire file into context.' if (isSql) return pathHint(hintPath, sqlTableHint(hintPath, targetFor(hintPath, 'table'), reason)) diff --git a/tests/bash_cat_read_gate_floor.test.ts b/tests/bash_cat_read_gate_floor.test.ts new file mode 100644 index 000000000..d7dd69b29 --- /dev/null +++ b/tests/bash_cat_read_gate_floor.test.ts @@ -0,0 +1,139 @@ +// A whole-file dump of a file the Read gate would hand over whole passes, and the refusal that remains names a command that runs. CAPTURE (session transcripts 2026-08 to 2026-10): eight `cat` refusals of 0.8-12.6 KB files, most of them heading-less memory notes, each named `token-goat section "::SectionHeading"`, which exits 1 with "has no headings"; none was run, and six were followed by a Read or sed of the same whole file. The fixtures are HAND-DERIVED to those shapes: an 806-byte note with no heading (the size of the first captured one), a doc past the floor with and without headings, and data files on either side of their type's threshold. +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { FILE_TYPE_THRESHOLDS } from '../src/hints/file_type_handler.js' +import { preBashHandler } from '../src/hooks_bash.js' +import { clearModuleCaches } from '../src/reset.js' +import type { HookOutput } from '../src/types.js' +import { ROOT, runBundle, tgIsolatedEnv } from './helpers/bundle.js' +import { makeHookEvent } from './helpers/hook-event.js' + +let proj: string +let home: string +let seq = 0 + +function bash(command: string): HookOutput { + seq += 1 + return preBashHandler(makeHookEvent({ toolName: 'Bash', toolInput: { command }, sessionId: `cat-floor-${seq}`, raw: { cwd: proj } })) +} + +function write(name: string, body: string): string { + fs.writeFileSync(path.join(proj, name), body, 'utf8') + return name +} + +function denial(out: HookOutput): string { + if (out.hookType !== 'deny') throw new Error(`expected a refusal, got ${JSON.stringify(out)}`) + return out.message +} + +/** The command a refusal leads with. */ +function ledCommand(message: string): string { + const m = /`(token-goat [^`]+)`/.exec(message) + if (m === null) throw new Error('no command in: ' + message) + return m[1] ?? '' +} + +/** Runs a printed `token-goat ""` through the built bundle, from the project directory the hook resolved the path against. */ +function runPrinted(command: string): { status: number | null; stdout: string; stderr: string } { + const m = /^token-goat (\S+) "([^"]+)"$/.exec(command) + if (m === null) throw new Error('unexpected command shape: ' + command) + return runBundle([m[1] ?? '', m[2] ?? ''], { cwd: proj, env: tgIsolatedEnv(home, { TOKEN_GOAT_HOME: home }) }) +} + +const NOTE_LINE = 'Shopify Admin API notes: rate limits, the bulk endpoint and the token that goes in the header.\n' +const headinglessNote = (bytes: number): string => NOTE_LINE.repeat(Math.ceil(bytes / NOTE_LINE.length)).slice(0, bytes) +const headedDoc = (bytes: number): string => { + let out = '# Artwork Policy\n\n' + for (let i = 1; out.length < bytes; i++) out += `## Clause ${i}\n\n${NOTE_LINE.repeat(4)}\n` + return out +} + +beforeEach(() => { + clearModuleCaches() + // Under the repo's ignored .tmp, not the system temp: the Bash hook lets every read of a system-temp file through, so nothing there is ever refused. + fs.mkdirSync(path.join(ROOT, '.tmp'), { recursive: true }) + proj = fs.mkdtempSync(path.join(ROOT, '.tmp', 'tg-cat-floor-')) + home = fs.mkdtempSync(path.join(os.tmpdir(), 'tg-cat-floor-home-')) +}) + +afterEach(() => { + for (const d of [proj, home]) fs.rmSync(d, { recursive: true, force: true }) + clearModuleCaches() +}) + +describe('cat of a file the Read gate would hand over whole', () => { + it('passes an 806-byte heading-less note', () => { + const note = write('reference_shopify_api.md', headinglessNote(806)) + expect(bash(`cat ${note}`).hookType).toBe('pass') + }) + + it('passes every spelling of the whole-file dump the refusal covers', () => { + const note = write('note.md', headinglessNote(806)) + for (const command of [`cat "${note}"`, `bat ${note}`, `cat ${note} 2>&1`, `cat ${note} ${write('other.md', headinglessNote(900))}`, `sed -n '1,9p' ${note}`]) { + expect(bash(command).hookType, command).not.toBe('deny') + } + }) + + it('passes a JSON file under its type threshold and refuses one past it', () => { + const small = write('small.json', JSON.stringify({ rows: 'x'.repeat(FILE_TYPE_THRESHOLDS.json - 100) })) + const big = write('big.json', JSON.stringify({ rows: 'x'.repeat(FILE_TYPE_THRESHOLDS.json + 100) })) + expect(bash(`cat ${small}`).hookType).toBe('pass') + expect(denial(bash(`cat ${big}`))).toContain('token-goat json-') + }) + + it('still refuses a small .env file, which the Read gate also refuses to re-read at any size', () => { + write('.env', 'DATABASE_URL=postgres://h/db\n') + expect(denial(bash('cat .env'))).toContain('token-goat config-get ".env" "DATABASE_URL"') + }) + + it('refuses a multi-file cat for the files past the floor and names only those', () => { + const small = write('small.md', headinglessNote(800)) + const big = write('big.md', headedDoc(200_000)) + const message = denial(bash(`cat ${small} ${big}`)) + expect(message).toContain('big.md') + expect(message).not.toContain('small.md') + }) +}) + +describe('the refusal that remains names a command that runs', () => { + it('names a real heading for a 200 KB doc with headings', () => { + const doc = write('policy.md', headedDoc(200_000)) + const command = ledCommand(denial(bash(`cat ${doc}`))) + expect(command).toBe('token-goat section "policy.md::Clause 1"') + const ran = runPrinted(command) + expect(ran.status, ran.stderr).toBe(0) + expect(ran.stdout).toContain('Shopify Admin API notes') + }) + + it('names a read of the first lines for a heading-less doc past the floor, never a section placeholder', () => { + const note = write('project_taobao_intake.md', headinglessNote(30_000)) + const message = denial(bash(`cat ${note}`)) + expect(message).not.toContain('SectionHeading') + const command = ledCommand(message) + expect(command).toBe('token-goat read "project_taobao_intake.md::1-80"') + const ran = runPrinted(command) + expect(ran.status, ran.stderr).toBe(0) + expect(ran.stdout).toContain('Shopify Admin API notes') + }) + + it('names a read of the first lines for a large .txt file, never a grep for ``', () => { + const notes = write('notes.txt', headinglessNote(30_000)) + const message = denial(bash(`cat ${notes}`)) + expect(message).not.toContain('') + const ran = runPrinted(ledCommand(message)) + expect(ran.status, ran.stderr).toBe(0) + }) + + it('names a read of the first lines for a whole-file sed range over a large heading-less doc', () => { + const lines = 320 + const note = write('ranged.md', NOTE_LINE.repeat(lines)) + expect(fs.statSync(path.join(proj, note)).size).toBeGreaterThan(FILE_TYPE_THRESHOLDS.txt) + const message = denial(bash(`sed -n '1,${lines}p' ${note}`)) + expect(ledCommand(message)).toBe('token-goat read "ranged.md::1-80"') + }) +}) diff --git a/tests/bash_deny_names_a_runnable_target.test.ts b/tests/bash_deny_names_a_runnable_target.test.ts index f6a530134..5b0631b62 100644 --- a/tests/bash_deny_names_a_runnable_target.test.ts +++ b/tests/bash_deny_names_a_runnable_target.test.ts @@ -7,7 +7,7 @@ import * as path from 'node:path' import { beforeEach, afterEach, describe, expect, it } from 'vitest' import { hintTarget, type HintTarget } from '../src/hint_target.js' -import { surgicalHintFor } from '../src/bash_extractors.js' +import { sqlTableHint, surgicalHintFor } from '../src/bash_extractors.js' import { stripUnsafeSuggestions } from '../src/hint_suggestion_guard.js' import { indexFileSync } from '../src/parser.js' import { globalDbPath, configPath } from '../src/constants.js' @@ -136,12 +136,28 @@ describe('surgicalHintFor with a resolved target', () => { expect(hint).toBe('Run `token-goat section "doc.md::First Heading"` to read one section, or `token-goat outline "doc.md"` for every heading with line ranges.\n`cat` loads the entire file into context.') }) - it('keeps the placeholder name when no target could be resolved', () => { + it('leads with a read of the first lines when no heading could be resolved, never the placeholder that fails', () => { const hint = surgicalHintFor('doc.md', false, false, true, false, none('section', 'SectionHeading')) - expect(hint).toBe('Run `token-goat section "doc.md::SectionHeading"` to read one section, or `token-goat outline "doc.md"` for every heading with line ranges.') + expect(hint).toBe('Run `token-goat read "doc.md::1-80"` to read its first 80 lines.') expect(PLACEHOLDER_SECTION_FAILS).toContain('not found') }) + // HAND-DERIVED: each of these used to lead with its slice's placeholder (`config-get ".env" "KEY_NAME"`, a grep for ``, `section "conf.toml::SectionHeading"`, `read "s.sql::table_name"`, a grep for `` on a .txt), and none of those runs as printed. Run against the built bundle in the describe below. + it.each([ + ['an env file', '.env', true, false, false, none('key', 'KEY_NAME')], + ['a .properties file', 'app.properties', false, true, false, none('key', 'KEY_NAME')], + ['a TOML file', 'conf.toml', false, true, false, none('section', 'SectionHeading')], + ['a flat .conf file', 'app.conf', false, true, false, none('key', 'KEY_NAME')], + ['a .txt file, whatever heading was found', 'notes.txt', false, false, true, real('Install', 'section')], + ])('leads %s with no resolvable name with the read of its first lines', (_, file, isEnv, isConfig, isDoc, target) => { + expect(surgicalHintFor(file, isEnv, isConfig, isDoc, false, target)).toBe('Run `token-goat read "' + file + '::1-80"` to read its first 80 lines.') + }) + + it('leads a SQL file with no table to name with the read of its first lines', () => { + expect(sqlTableHint('s.sql', none('table', 'table_name'))).toBe('Run `token-goat read "s.sql::1-80"` to read its first 80 lines.') + expect(sqlTableHint('s.sql', real('users', 'table'))).toBe('Run `token-goat read "s.sql::users"` to pull one CREATE TABLE / CREATE TYPE block.') + }) + it('sends a TOML table to section, which config-get cannot read', () => { const hint = surgicalHintFor('conf.toml', false, true, false, false, real('server', 'section')) expect(hint).toBe('Run `token-goat section "conf.toml::server"` to read one table, or `token-goat outline "conf.toml"` for every key with line ranges.') @@ -247,12 +263,15 @@ describe('the commands a deny names run against the file', () => { for (const d of [home, proj]) fs.rmSync(d, { recursive: true, force: true }) }) + /** `body` with comment lines appended past the size `cat` passes under, so the call is refused at all. */ + const pastCatFloor = (body: string): string => body + '# a comment line that carries the file past the size cat passes under\n'.repeat(400) + it.each([ ['application.properties', 'spring.datasource.url=jdbc:x\nserver.port=8080\n', 'jdbc:x'], ['.env.local', 'export API_KEY=abc\nexport PORT=1\n', DOTENV_VALUE_PLACEHOLDER], ['.env', 'DATABASE_URL=postgres://h/db\nPORT=2\n', DOTENV_VALUE_PLACEHOLDER], ])('cat of %s is denied with commands that all exit 0, the first returning the value', (file, body, value) => { - fs.writeFileSync(path.join(proj, file), body, 'utf8') + fs.writeFileSync(path.join(proj, file), pastCatFloor(body), 'utf8') const commands = namedCommands(hook('Bash', { command: 'cat ' + file })) expect(commands.length).toBeGreaterThan(0) const results = commands.map((c) => ({ c, ...run(c) })) @@ -262,13 +281,18 @@ describe('the commands a deny names run against the file', () => { }) it('never offers outline for a .properties file, which has no extractor', () => { - fs.writeFileSync(path.join(proj, 'app.properties'), 'a.b=1\n', 'utf8') - expect(namedCommands(hook('Bash', { command: 'cat app.properties' })).join('\n')).not.toContain('outline') + fs.writeFileSync(path.join(proj, 'app.properties'), pastCatFloor('a.b=1\n'), 'utf8') + const joined = namedCommands(hook('Bash', { command: 'cat app.properties' })).join('\n') + expect(joined).toContain('token-goat config-get') + expect(joined).not.toContain('outline') }) it('names no config-get or placeholder key for an empty .properties file, where no key exists to name', () => { - fs.writeFileSync(path.join(proj, 'empty.properties'), '# nothing here\n', 'utf8') - const joined = namedCommands(hook('Bash', { command: 'cat empty.properties' })).join('\n') + fs.writeFileSync(path.join(proj, 'empty.properties'), pastCatFloor('# nothing here\n'), 'utf8') + const commands = namedCommands(hook('Bash', { command: 'cat empty.properties' })) + const joined = commands.join('\n') + expect(commands.length).toBeGreaterThan(0) + expect(commands.filter((c) => run(c).status !== 0)).toEqual([]) expect(joined).not.toContain('KEY_NAME') expect(joined).not.toContain('config-get') expect(joined).not.toContain('outline') @@ -276,20 +300,22 @@ describe('the commands a deny names run against the file', () => { // CAPTURE 2026-10-03 against the built bundle: `section "d.rst::Sub"` returns the section, `outline d.rst` exits 1 (no extractor), `section` on a .txt exits 1 ("has no headings"). it('cat of a .rst file names section but never outline, and the named commands run', () => { - fs.writeFileSync(path.join(proj, 'd.rst'), 'Title\n=====\n\nIntro\n\nSub\n---\n\nbody\n'.repeat(1), 'utf8') + fs.writeFileSync(path.join(proj, 'd.rst'), 'Title\n=====\n\nIntro\n\nSub\n---\n\nbody\n' + 'Prose that carries the file past the size cat passes under.\n'.repeat(400), 'utf8') const commands = namedCommands(hook('Bash', { command: 'cat d.rst' })) expect(commands.length).toBeGreaterThan(0) expect(commands.join('\n')).not.toContain('outline') expect(commands.filter((c) => run(c).status !== 0)).toEqual([]) }) - it('cat of a .txt file names no section or outline, only a grep that runs', () => { - fs.writeFileSync(path.join(proj, 'n.txt'), 'alpha\nbeta\n'.repeat(400), 'utf8') + it('cat of a .txt file names no section or outline, only a read of its first lines that runs as printed', () => { + fs.writeFileSync(path.join(proj, 'n.txt'), 'alpha\nbeta\n'.repeat(2000), 'utf8') const commands = namedCommands(hook('Bash', { command: 'cat n.txt' })) const joined = commands.join('\n') + expect(commands.length).toBeGreaterThan(0) expect(joined).not.toContain('outline') expect(joined).not.toContain('token-goat section') - expect(commands.filter((c) => run(c.replace('', 'beta')).status !== 0)).toEqual([]) + expect(joined).not.toContain('') + expect(commands.filter((c) => run(c).status !== 0)).toEqual([]) }) it('the Grep tool on a .txt file gets no structural hint', () => { diff --git a/tests/hint_target.test.ts b/tests/hint_target.test.ts index b790249a2..6ca091805 100644 --- a/tests/hint_target.test.ts +++ b/tests/hint_target.test.ts @@ -211,7 +211,8 @@ describe('a repeated identical deny, through the real relay', () => { }) it('gives the second copy as the command and one line, and the first copy in full', async () => { - write('guide.md', doc(REAL, 200)) + // Past the size `cat` passes under, so the call is refused at all. + write('guide.md', doc(REAL, 4000)) const session = uniq('repeat') const first = denyOf(await relayInProcess('pre_tool_use', catPre(session, 'guide.md'))) const second = denyOf(await relayInProcess('pre_tool_use', catPre(session, 'guide.md'))) @@ -262,11 +263,12 @@ const SITES: Site[] = [ { name: 'hooks_bash.ts cat deny', real: (h) => 'Run `token-goat section "guide.md::' + h + '"`', - fallback: 'Run `token-goat section "guide.md::SectionHeading"`', + fallback: 'Run `token-goat read "guide.md::1-80"`', guarded: true, fallbackHeadings: [], run: async (headings) => { - write('guide.md', doc(headings)) + // Past the size `cat` passes under, which the other sites' default padding is not. + write('guide.md', doc(headings, 4000)) const out = preBashHandler(makeHookEvent({ toolName: 'Bash', toolInput: { command: 'cat guide.md' }, sessionId: uniq('bash'), raw: { cwd: dir } })) if (out.hookType !== 'deny') throw new Error('expected a deny, got ' + out.hookType) return out.message diff --git a/tests/hook_reads_of_a_drive_letter_key_open_its_wsl_mount.test.ts b/tests/hook_reads_of_a_drive_letter_key_open_its_wsl_mount.test.ts index da9eae989..bfdebcb0e 100644 --- a/tests/hook_reads_of_a_drive_letter_key_open_its_wsl_mount.test.ts +++ b/tests/hook_reads_of_a_drive_letter_key_open_its_wsl_mount.test.ts @@ -138,10 +138,11 @@ describe('a hook reading the file behind a drive-letter key opens it where the h }) it('the Bash pre hook names a sed range over the whole file as the whole file', async () => { - place('x.ts', SOURCE) + // Three copies, past the size a whole-file read passes under. + place('x.ts', SOURCE.repeat(3)) const sid = newSession() - const out = await asHook(sid, () => preBashHandler(bashEvent(sid, 'pre_tool_use', "sed -n '1,300p' x.ts"))) - expect(text(out)).toContain('is the whole file (300 lines)') + const out = await asHook(sid, () => preBashHandler(bashEvent(sid, 'pre_tool_use', "sed -n '1,900p' x.ts"))) + expect(text(out)).toContain('is the whole file (900 lines)') }) it('the Bash post hook folds a whole-file cat of a long document to its heading tree', async () => { diff --git a/tests/hooks_bash.test.ts b/tests/hooks_bash.test.ts index f0655e7a5..ab5c54e65 100644 --- a/tests/hooks_bash.test.ts +++ b/tests/hooks_bash.test.ts @@ -965,7 +965,8 @@ describe('preBashHandler — cat source file recall', () => { const result = preBashHandler(event) expect(result.hookType).toBe('deny') if (result.hookType === 'deny') { - expect(result.message).toContain('token-goat section') + // Not on disk, so no heading can be named: the deny leads with the read of the first lines, which needs none. + expect(result.message).toContain('token-goat read "/c/Projects/report.md::1-80"') } }) @@ -982,7 +983,7 @@ describe('preBashHandler — cat source file recall', () => { it('intercepts python heredoc that reads a markdown file', () => { const event = makeBashEvent("python3 - << 'PYEOF'\npath = 'C:/Projects/yeswehack/report-05/report.md'\nwith open(path, encoding='utf-8') as f: content = f.read()\nPYEOF") - expect(interceptedReadHint(preBashHandler(event))).toContain('token-goat section') + expect(interceptedReadHint(preBashHandler(event))).toContain('token-goat read "C:/Projects/yeswehack/report-05/report.md::1-80"') }) it('does not false-positive on a command that merely contains the substring "python3" elsewhere', () => { @@ -992,25 +993,25 @@ describe('preBashHandler — cat source file recall', () => { expect(result.hookType).toBe('pass') }) + // Neither env file is on disk, so no variable can be named and the deny leads with the read of the first lines, never `config-get "KEY_NAME"`, which exits 1. it('denies cat of a local.env file', () => { const result = preBashHandler(makeBashEvent('cat C:/Projects/myapp/local.env')) expectHookType(result, 'deny') - expect(result.message).toContain('config-get') + expect(result.message).toContain('token-goat read "C:/Projects/myapp/local.env::1-80"') }) it('denies cat of a .env.local file', () => { const result = preBashHandler(makeBashEvent('cat /app/.env.local')) expectHookType(result, 'deny') - expect(result.message).toContain('config-get') + expect(result.message).toContain('token-goat read "/app/.env.local::1-80"') }) it('emits advisory context for cat of a SQL migration file', () => { const result = preBashHandler(makeBashEvent('cat supabase/migrations/0001_init.sql')) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - // `read`, not `section`: `section` finds no headings in a .sql file and exits 1, while `read` returns the CREATE block the SQL adapter indexes by name. - expect(result.context).toContain('token-goat read "supabase/migrations/0001_init.sql::table_name"') - expect(result.context).toContain('CREATE TABLE') + // `read`, not `section`: `section` finds no headings in a .sql file and exits 1. With no table to name (the file is not on disk), `read "file::table_name"` exits 1 too, so the hint reads the first lines. + expect(result.context).toBe('Run `token-goat read "supabase/migrations/0001_init.sql::1-80"` to read its first 80 lines.\n`cat` loads the entire file into context.') } }) @@ -1040,11 +1041,11 @@ describe('preBashHandler — cat source file recall', () => { }, ) - it('tail command on a SQL file suggests table_name, matching head\'s SQL hint', () => { + it('tail command on a SQL file gets the SQL hint, matching head\'s SQL hint', () => { const result = preBashHandler(makeBashEvent('tail -n 30 supabase/migrations/0001_init.sql')) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toContain('table_name') + expect(result.context).toContain('token-goat read "supabase/migrations/0001_init.sql::1-80"') expect(result.context).not.toContain('SectionHeading') } }) @@ -1057,11 +1058,12 @@ describe('preBashHandler — cat source file recall', () => { } }) - it('tail command on a TOML config file names a table through section, which config-get cannot read', () => { + it('tail command on a TOML config file with no table to name leads with the read of its first lines, never a placeholder table', () => { const result = preBashHandler(makeBashEvent('tail -n 30 src/config.toml')) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toContain('token-goat section "src/config.toml::') + expect(result.context).toContain('token-goat read "src/config.toml::1-80"') + expect(result.context).not.toContain('SectionHeading') } }) @@ -1378,7 +1380,7 @@ describe('preBashHandler — cat source file recall', () => { const result = preBashHandler(event) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toContain('token-goat read "migrations/0001_init.sql::table_name"') + expect(result.context).toContain('token-goat read "migrations/0001_init.sql::1-80"') } }) @@ -2520,7 +2522,7 @@ describe('preBashHandler — awk line-range interception', () => { }) }) -/** A line range from line 1 to the last line is `cat` spelled another way. Regression: pricing compared the range against a surgical read of the same lines, which is also the whole file, found no saving and declined, so `awk 'NR>=1 && NR<=324' SKILL.md` over the humanizer skill (324 lines, 53,931 bytes, the ~15.8k-token read named on a feedback card, measured 2026-09-24) passed with no word while `cat SKILL.md` was refused. Provenance: HAND-DERIVED. The fixture is a 40-line markdown file written here; the skill's line and byte counts are read off the file on disk. */ +/** A line range from line 1 to the last line is `cat` spelled another way. Regression: pricing compared the range against a surgical read of the same lines, which is also the whole file, found no saving and declined, so `awk 'NR>=1 && NR<=324' SKILL.md` over the humanizer skill (324 lines, 53,931 bytes, the ~15.8k-token read named on a feedback card, measured 2026-09-24) passed with no word while `cat SKILL.md` was refused. Provenance: HAND-DERIVED. The fixture is a 40-line markdown file written here, its lines padded past the size `cat` passes under; the skill's line and byte counts are read off the file on disk. */ describe('preBashHandler — a line range spanning the whole file', () => { let doc: string const LINES = 40 @@ -2531,7 +2533,7 @@ describe('preBashHandler — a line range spanning the whole file', () => { mkdirSync(dir, { recursive: true }) doc = join(dir, 'whole_range_skill.md').replace(/\\/g, '/') const body: string[] = [] - for (let i = 0; body.length < LINES; i++) body.push(i % 10 === 0 ? `## Part ${i / 10 + 1}` : `Line ${i} of the fixture.`) + for (let i = 0; body.length < LINES; i++) body.push(i % 10 === 0 ? `## Part ${i / 10 + 1}` : `Line ${i} of the fixture. ${'Padding prose for the line. '.repeat(24)}`) writeFileSync(doc, body.join('\n') + '\n') }) @@ -3726,12 +3728,12 @@ describe('preBashHandler — SQL file cat hint', () => { clearModuleCaches() }) + // Neither SQL file is on disk, so no table can be named and the hint reads the first lines; tests/bash_deny_names_a_runnable_target.test.ts covers the real-table form. it('emits context hint (not deny) for cat of a SQL schema file', () => { const result = preBashHandler(makeBashEvent('cat schema.sql')) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toContain('token-goat read "schema.sql::table_name"') - expect(result.context).toContain('CREATE TABLE') + expect(result.context).toBe('Run `token-goat read "schema.sql::1-80"` to read its first 80 lines.\n`cat` loads the entire file into context.') } }) @@ -3739,7 +3741,7 @@ describe('preBashHandler — SQL file cat hint', () => { const result = preBashHandler(makeBashEvent('cat migration.sql')) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toContain('token-goat read "migration.sql::table_name"') + expect(result.context).toContain('token-goat read "migration.sql::1-80"') } }) @@ -3753,8 +3755,7 @@ describe('preBashHandler — SQL file cat hint', () => { const result = preBashHandler(makeBashEvent(`powershell -Command "Get-Content 'schema.sql'"`)) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toContain('token-goat read "schema.sql::table_name"') - expect(result.context).toContain('CREATE TABLE') + expect(result.context).toContain('token-goat read "schema.sql::1-80"') } }) }) @@ -4899,7 +4900,7 @@ describe('preBashHandler — stderr-redirect and cat-piped read spellings (loop- const result = preBashHandler(makeBashEvent('cat docs/loop46_memory.md 2>/dev/null')) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toBe('Run `token-goat section "docs/loop46_memory.md::SectionHeading"` to read one section, or `token-goat outline "docs/loop46_memory.md"` for every heading with line ranges.\n`cat` loads the entire file into context.') + expect(result.context).toBe('Run `token-goat read "docs/loop46_memory.md::1-80"` to read its first 80 lines.\n`cat` loads the entire file into context.') } }) @@ -4923,7 +4924,7 @@ describe('preBashHandler — stderr-redirect and cat-piped read spellings (loop- const result = preBashHandler(makeBashEvent('cat -n README_loop46.md 2>/dev/null | tail -60')) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toBe('Run `token-goat section "README_loop46.md::SectionHeading"` to read one section, or `token-goat outline "README_loop46.md"` for every heading with line ranges.\n`tail` bypasses read hooks.') + expect(result.context).toBe('Run `token-goat read "README_loop46.md::1-80"` to read its first 80 lines.\n`tail` bypasses read hooks.') } }) @@ -5129,7 +5130,7 @@ describe('preBashHandler — PowerShell [IO.File]::ReadAllText interception', () const result = preBashHandler(makeBashEvent(`[IO.File]::ReadAllText('schema.sql')`)) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toContain('token-goat read "schema.sql::table_name"') + expect(result.context).toContain('token-goat read "schema.sql::1-80"') } }) @@ -5141,12 +5142,13 @@ describe('preBashHandler — PowerShell [IO.File]::ReadAllText interception', () } }) - // HAND-DERIVED: surgicalHintFor (src/bash_extractors.ts) points an isEnv read at `config-get ... KEY_NAME`, and every other file-read extractor's handler in preBashHandlerInner calls it with isEnv, so a .env read through this same handler must not fall through to the generic "SymbolName" hint a .env file has no symbols to satisfy. - it('denies a .NET .env file read with a config-get hint, not the generic symbol hint', () => { + // HAND-DERIVED: surgicalHintFor (src/bash_extractors.ts) gives an isEnv read the env ladder (`config-get` on a variable the file holds, else its first lines), and every other file-read extractor's handler in preBashHandlerInner calls it with isEnv, so a .env read through this same handler must not fall through to the generic "SymbolName" hint a .env file has no symbols to satisfy. No .env is on disk here, so there is no variable to name. + it('denies a .NET .env file read with the env hint, not the generic symbol hint', () => { const result = preBashHandler(makeBashEvent(`[System.IO.File]::ReadAllText('.env')`)) expect(interceptedReadHint(result)).not.toBeNull() - expect(interceptedReadHint(result)).toContain('token-goat config-get ".env" "KEY_NAME"') + expect(interceptedReadHint(result)).toContain('token-goat read ".env::1-80"') expect(interceptedReadHint(result)).not.toContain('SymbolName') + expect(interceptedReadHint(result)).not.toContain('KEY_NAME') }) }) @@ -5192,7 +5194,7 @@ describe('extractPythonFileRead — PowerShell here-string and multi-format supp const result = preBashHandler(makeBashEvent(cmd)) expect(result.hookType).toBe('context') if (result.hookType === 'context') { - expect(result.context).toContain('token-goat read "db/schema.sql::table_name"') + expect(result.context).toContain('token-goat read "db/schema.sql::1-80"') } }) }) diff --git a/tests/rewrite_permission_e2e.test.ts b/tests/rewrite_permission_e2e.test.ts index 640607a13..eceaab37d 100644 --- a/tests/rewrite_permission_e2e.test.ts +++ b/tests/rewrite_permission_e2e.test.ts @@ -140,7 +140,8 @@ describe('image Read rewrites honor Claude Code Read rules (built bundle)', () = // HAND-DERIVED from https://code.claude.com/docs/en/permissions: a Read deny or ask rule applies to Claude Code's Read and Grep tools and to its recognized file commands in Bash (cat, head, sed), but never to a token-goat command those calls are pointed at instead. describe('read hints honor Claude Code Read rules (built bundle)', () => { - const lock = JSON.stringify({ name: 'p', lockfileVersion: 3, packages: { '': { name: 'p' } } }, null, 2) + // 400 dependencies, about 54 KB: past the 20 KB under which a whole-file `cat` of a JSON file runs without a refusal, so the cat beside the ruled path still draws its hint. + const lock = JSON.stringify({ name: 'p', lockfileVersion: 3, packages: Object.fromEntries([['', { name: 'p' }], ...Array.from({ length: 400 }, (_, i) => [`node_modules/dep${i}`, { version: '1.0.0', resolved: `https://registry.npmjs.org/dep${i}/-/dep${i}-1.0.0.tgz` }])]) }, null, 2) const source = Array.from({ length: 400 }, (_, i) => `export function f${i}(a: number): number {\n return a + ${i}\n}\n`).join('') let calls = 0 diff --git a/tests/subagent_compaction_epoch.test.ts b/tests/subagent_compaction_epoch.test.ts index f5cfca7f2..c97dc4d80 100644 --- a/tests/subagent_compaction_epoch.test.ts +++ b/tests/subagent_compaction_epoch.test.ts @@ -172,7 +172,8 @@ describe('a compaction resets subagent dedup state (built bundle)', () => { describe('a compaction resets the repeat-deny key of a subagent (in-process relay)', () => { const guide = (): string => { - const filler = 'Section prose that pads this fixture past the size floors the gated sites apply. '.repeat(20) + // 130 repeats, about 22 KB in all: past the 20 KB under which a whole-file `cat` runs without a refusal. + const filler = 'Section prose that pads this fixture past the size floors the gated sites apply. '.repeat(130) return '## Install\n\n' + filler + '\n\n## Usage\n\n' + filler + '\n' } diff --git a/tests/suggestion_guard_quote_scan.test.ts b/tests/suggestion_guard_quote_scan.test.ts index 85d58efc6..2d66646c3 100644 --- a/tests/suggestion_guard_quote_scan.test.ts +++ b/tests/suggestion_guard_quote_scan.test.ts @@ -39,8 +39,9 @@ describe('a single-quoted path holding a double quote', () => { const reason = await relayedCatReason(p, 'dq') expect(reason).not.toContain('command omitted') const commands = fencedCommands(reason) - expect(commands).toEqual([`token-goat section 'docs/a"b.md::SectionHeading'`, `token-goat outline 'docs/a"b.md'`]) - if (SH !== null) expect(shArgv(SH, commands[1]!)).toEqual(['outline', p]) + // The file is not on disk, so no heading can be named and the deny leads with the read of its first lines. + expect(commands).toEqual([`token-goat read 'docs/a"b.md::1-80'`]) + if (SH !== null) expect(shArgv(SH, commands[0]!)).toEqual(['read', p + '::1-80']) }) it('keeps a $ the single quotes hold literal, beside the double quote that made the guard look', () => { From 750e04b99095780dd4063966a6205dc373296add Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 05:58:32 -0500 Subject: [PATCH 12/92] fix(hooks): read a compound Python command one simple command at a time, and let a parsing read run where the cap cannot extractPythonFileRead scanned the whole compound command as one snippet whenever it started with python, so the open() inside a script a quoted heredoc was writing (python a.py && cat > b.py <<'EOF' ... EOF) read as a file read and the command was refused. It now splits the command into simple commands, takes the heredoc bodies in the order their openers appear, scans only a Python command's own snippet, and treats a heredoc as Python only when the python reads its program from stdin. A Python command whose standard output goes to a file prints nothing into the conversation, so it is no longer refused or rewritten; a redirect of another stream, a duplication or a /dev/stdout target still counts as printing. Where the capped run cannot ship (compression off, auto mode, a rule that withholds the rewrite), a read that hands the file to json.load, json.loads, csv.reader, csv.DictReader, yaml.safe_load, load_workbook or pd.read_* and never prints its raw text now gets the advice as a hint instead of a refusal; a read that prints the file keeps the refusal. cappedInterpreterRead takes the fallback output, so the node and PowerShell branches pass their refusal unchanged. The python heredoc branch that only matched a whole command of the form python - < ""` stores one per project; notes print at every session start and in the compaction manifest. Add `--anchor src/auth.ts::login`, or name a `file::symbol` in the finding, and the note is marked "(changed since note)" once that symbol changes | | Large JSON Lines file (.jsonl ≥20 KB) read in full | Record count and first-record schema shown; suggests slicing records with offset/limit instead of loading millions of tokens | | Other Office binary (.odt, .ods, .ott, .odp) opened via Read | Full read denied; redirects to `pandoc` for text extraction (no dedicated reader for these formats yet) | diff --git a/eslint-suppressions.json b/eslint-suppressions.json index 669b73f93..93d87397d 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -1,7 +1,7 @@ { "src/bash_extractors.ts": { "regexp/no-super-linear-backtracking": { - "count": 19 + "count": 18 } }, "src/cli_bootstrap_audit.ts": { diff --git a/src/bash_extractors.ts b/src/bash_extractors.ts index 3bbd781db..8ce1ddec3 100644 --- a/src/bash_extractors.ts +++ b/src/bash_extractors.ts @@ -534,12 +534,56 @@ export interface PythonFileReadResult { isSql: boolean isXml?: boolean isOutputFile: boolean + /** True when the snippet hands the file to a parser and never prints its text ({@link pythonReadFeedsParser}). */ + feedsParser: boolean } export const KNOWN_PYTHON_EXT_STR = 'java|py|ts|tsx|js|jsx|go|rb|rs|cpp|cc|cxx|c|h|hpp|kt|swift|cs|php|scala|clj|css|scss|sass|less|md|mdx|rst|txt|json|yaml|yml|toml|xml|dtsx|ampkg|xaml|html|htm|conf|cfg|ini|properties|sql|ps1|psm1|env' export const OPEN_EXT = new RegExp(`\\.(?:${KNOWN_PYTHON_EXT_STR})$`, 'i') -/** Returns the file path and metadata if the bash command is a Python snippet that reads a known-extension file via open(). Returns null otherwise. */ +/** A Python interpreter starting a simple command. */ +const PYTHON_COMMAND_RE = /^(?:python3?|py)(?:\.exe)?\b/i + +/** A Python command that reads its program from standard input, which is what makes the heredoc it opens a Python snippet: `python - <<'PY'` and `python3 -u < m[1] as string) + return !new RegExp(String.raw`\bfor\s+[\w\s,()]+?\s+in\s+(?:enumerate\s*\(\s*)?(?:open\s*\(${handles.map((h) => '|' + h + String.raw`\b`).join('')})`).test(rest) +} + +/** True when `command`, one simple command, sends its standard output to a file, so nothing it prints reaches the conversation. A redirect of another stream (`2>`), a duplication (`>&2`, `2>&1`) and a target that is the terminal or a standard stream again (`/dev/stdout`, `/dev/stderr`, `/dev/tty`) do not count. */ +function stdoutGoesToFile(command: string): boolean { + // A `>` inside a quoted argument (`-c "print(a > b)"`) is not a redirect. + const masked = maskPythonStrings(command) + for (const m of masked.matchAll(/(?:^|[\s;&|(])(?:1|&)?>>?(?![&>(|])/g)) { + const target = /^(?:"([^"]*)"|'([^']*)'|([^\s;&|<>]+))/.exec(command.slice((m.index ?? 0) + m[0].length).trimStart()) + const word = target?.[1] ?? target?.[2] ?? target?.[3] + if (word !== undefined && !/^\/dev\/(?:stdout|stderr|tty|fd\/[12])$/.test(word)) return true + } + return false +} + +/** Returns the file path and metadata if the bash command is a Python snippet that reads a known-extension file via open(). Returns null otherwise. A compound command is read one simple command at a time, and only a Python one is scanned: `python a.py && cat > b.py <<'EOF' ... EOF && python b.py` writes a script whose `open()` reads nothing until it runs, and scanning the whole line refused it as a read of the file that script names (3 of 16 measured refusals). A heredoc is a snippet only when it is the program a `python` reads from standard input, and a Python command whose standard output goes to a file prints nothing into the conversation, so it is no read at all. */ export function extractPythonFileRead(cmd: string): PythonFileReadResult | null { let inner = cmd.trim() const w = POWERSHELL_WRAP_RE.exec(inner) @@ -551,15 +595,26 @@ export function extractPythonFileRead(cmd: string): PythonFileReadResult | null const psHereMatch = /^@'([\s\S]*?)'@\s*\|\s*(?:python3?|py)(?:\.exe)?(?:\s+-\S*|\s+-)?\s*$/i.exec(inner) ?? /^@"([\s\S]*?)"@\s*\|\s*(?:python3?|py)(?:\.exe)?(?:\s+-\S*|\s+-)?\s*$/i.exec(inner) - let pythonBody: string | null = null - if (psHereMatch) { - pythonBody = (psHereMatch[1] ?? '').trim() - } else if (/^(?:python3?|py)(?:\.exe)?\b/i.test(inner)) { - pythonBody = inner + if (psHereMatch) return pythonSnippetRead((psHereMatch[1] ?? '').trim()) + if (!PYTHON_COMMAND_RE.test(inner)) return null + + const { code, bodies } = splitHeredocBodies(inner) + let nextBody = 0 + for (const segment of splitShellSegments(code)) { + // Bodies arrive in the order their openers were written, so each segment takes as many as it opens, Python or not. + const opened = Array.from(segment.matchAll(HEREDOC_OPENER_RE)).length + const body = opened > 0 ? bodies[nextBody] : undefined + nextBody += opened + const command = stripLeadingAssignments(segment) + if (!PYTHON_COMMAND_RE.test(command) || stdoutGoesToFile(command)) continue + const read = pythonSnippetRead(body !== undefined && PYTHON_STDIN_PROGRAM_RE.test(command) ? body : command) + if (read !== null) return read } + return null +} - if (pythonBody === null) return null - +/** The file a Python snippet reads through open(), as {@link extractPythonFileRead} reports it, or null when it reads none or writes one. */ +function pythonSnippetRead(pythonBody: string): PythonFileReadResult | null { // Return null when the command shows write intent — these are edits, not reads if (pythonOpenWritesAFile(pythonBody) || pythonWritesThroughFileObject(pythonBody)) return null @@ -568,44 +623,18 @@ export function extractPythonFileRead(cmd: string): PythonFileReadResult | null if (outputOpen?.[1]) { const filePath = outputOpen[1] if (isOrchestratorStateFile(filePath)) return null - return { filePath, isDoc: false, isConfig: false, isEnv: false, isSql: false, isOutputFile: true } + return { filePath, isDoc: false, isConfig: false, isEnv: false, isSql: false, isOutputFile: true, feedsParser: false } } + const feedsParser = pythonReadFeedsParser(pythonBody) const classifyResult = (filePath: string): PythonFileReadResult => { const flags = classifyFileExtensions(filePath) if (flags !== null) { - return { filePath, isDoc: flags.isDoc, isConfig: flags.isConfig, isEnv: flags.isEnv, isSql: flags.isSql, isXml: flags.isXml, isOutputFile: false } + return { filePath, isDoc: flags.isDoc, isConfig: flags.isConfig, isEnv: flags.isEnv, isSql: flags.isSql, isXml: flags.isXml, isOutputFile: false, feedsParser } } const { isDoc, isConfig, isSql, isXml } = classifyDocConfig(filePath) const isEnv = /\.env(\.\w+)?$/i.test(filePath) - return { filePath, isDoc, isConfig, isEnv, isSql, isXml, isOutputFile: false } - } - - // Heredoc form: python3 - << 'PYEOF'\n...\nPYEOF - const heredocMatch = /^python3?\s+-\s+<<\s*'?(\w+)'?\s*\n([\s\S]*?)\n\1\s*$/.exec(pythonBody) - if (heredocMatch) { - const body = heredocMatch[2] ?? '' - // Write-mode exclusion in the heredoc body - if (pythonOpenWritesAFile(body) || pythonWritesThroughFileObject(body)) return null - // Direct: open(r'path.ext') or open("path.ext") in body - const heredocOpen = new RegExp(`open\\s*\\(\\s*r?['"]([^'"]+\\.(?:${KNOWN_PYTHON_EXT_STR}))['"]`, 'i').exec(body) - if (heredocOpen?.[1]) { - const filePath = heredocOpen[1] - if (isOrchestratorStateFile(filePath)) return null - return classifyResult(filePath) - } - // Indirect: open(var, ...) where a string literal with known ext appears in the body - if (/open\s*\(/.test(body) && !pythonOpenPathsAreAllLiteral(body)) { - const literal = new RegExp(`['"]([^'"]+\\.(?:${KNOWN_PYTHON_EXT_STR}))['"]`, 'i').exec(body) - if (literal?.[1]) { - const filePath = literal[1] - if (isOrchestratorStateFile(filePath)) return null - if (OPEN_EXT.test(filePath)) { - return classifyResult(filePath) - } - } - } - return null + return { filePath, isDoc, isConfig, isEnv, isSql, isXml, isOutputFile: false, feedsParser } } // Direct: open('path.ext') or open("path.ext") @@ -1174,17 +1203,31 @@ const HEREDOC_OPENER_RE = /(? 0) { - if (line.trim() === pending[0]) pending.shift() + if (line.trim() === pending[0]) { + pending.shift() + bodies.push(body.join('\n')) + body = [] + } else { + body.push(line) + } continue } kept.push(line) for (const m of line.matchAll(HEREDOC_OPENER_RE)) pending.push(m[2] as string) } - return kept.join('\n') + if (pending.length > 0) bodies.push(body.join('\n')) + return { code: kept.join('\n'), bodies } } /** Words that can stand in front of a command's own assignments without being the command: `time X=0 cmd`, `do X=0 cmd; done`, `export X=0`. */ diff --git a/src/hooks_bash.ts b/src/hooks_bash.ts index da34d8a6b..afeca691f 100644 --- a/src/hooks_bash.ts +++ b/src/hooks_bash.ts @@ -109,9 +109,9 @@ function readGateFloorBytes(filePath: string): number { return READ_GATE_FLOOR_BYTES[path.extname(filePath).slice(1).toLowerCase()] ?? FILE_TYPE_THRESHOLDS.txt } -// Runs an inline interpreter file read under a token cap instead of refusing it. Many are projections, `json.load(open('r.json'))['118615']`, that print a few lines no token-goat command beats by a second round trip; refused, each cost the agent a retry and often a second refusal before it found a spelling the hook let through. A projection now passes whole, and a whole-file dump comes back capped with its recall id and the same surgical command the refusal named. The refusal stays wherever the wrapper cannot run: compression off, a pipeline or chain, a background `&`, a script past the command-line ceiling, VS Code. The hint passes the suggestion guard here because the relay only guards a refusal: printed after the capped output, a path that broke its quoting would reach the model unguarded. -function cappedInterpreterRead(event: HookEvent, rawCmd: string, cmd: string, hint: string, denial: string): HookOutput { - return maybeCompressRewrite(event, rawCmd, cmd, { maxTokens: INTERPRETER_READ_TOKEN_CAP, hint: stripUnsafeSuggestions(hint) }) ?? denyOutput(denial) +// Runs an inline interpreter file read under a token cap instead of refusing it. Many are projections, `json.load(open('r.json'))['118615']`, that print a few lines no token-goat command beats by a second round trip; refused, each cost the agent a retry and often a second refusal before it found a spelling the hook let through. A projection now passes whole, and a whole-file dump comes back capped with its recall id and the same surgical command the refusal named. Wherever the wrapper cannot run (compression off, a background `&`, a script past the command-line ceiling, VS Code, or rewrite_permission.ts withholding the rewrite, as it does in auto mode and under a rule that could match) `fallback` answers instead: the refusal for a read that prints the file, advice for one that hands it to a parser. The hint passes the suggestion guard here because the relay only guards a refusal: printed after the capped output, a path that broke its quoting would reach the model unguarded. +function cappedInterpreterRead(event: HookEvent, rawCmd: string, cmd: string, hint: string, fallback: HookOutput): HookOutput { + return maybeCompressRewrite(event, rawCmd, cmd, { maxTokens: INTERPRETER_READ_TOKEN_CAP, hint: stripUnsafeSuggestions(hint) }) ?? fallback } /** A rewrite of this Bash call's command, or null when rewrite_permission.ts finds it could change the permission outcome of the command the model wrote. */ @@ -534,7 +534,7 @@ function preBashHandlerInner(event: HookEvent): HookOutput { const pyRead = extractPythonFileRead(cmd) if (pyRead !== null) { - const { filePath, isDoc, isConfig, isEnv, isSql, isXml, isOutputFile } = pyRead + const { filePath, isDoc, isConfig, isEnv, isSql, isXml, isOutputFile, feedsParser } = pyRead const hintPath = displaySafePath(cdStripped ? resolveCdHintPath(rawCmd, filePath, hintCwd) : filePath) recordStat('session_hint', 0, 0) if (isOutputFile) { @@ -553,7 +553,8 @@ function preBashHandlerInner(event: HookEvent): HookOutput { const target = targetFor(hintPath, isEnv ? 'key' : undefined) const hint = surgicalHintFor(hintPath, isEnv, isConfig, isDoc, isXml ?? false, target) const denial = surgicalHintFor(hintPath, isEnv, isConfig, isDoc, isXml ?? false, target, reason) - return cdStripped ? pathHint(hintPath, denial) : cappedInterpreterRead(event, rawCmd, cmd, hint, denial) + // A script that parses the file computes on it rather than printing it, so where the cap cannot run it gets the advice and runs: refused, every measured one came back as the same script from a file, the file on stdin or a copy elsewhere, a turn later. + return cdStripped ? pathHint(hintPath, denial) : cappedInterpreterRead(event, rawCmd, cmd, hint, feedsParser ? pathHint(hintPath, denial) : denyOutput(denial)) } const tailResult = extractTailFile(cmd) @@ -589,7 +590,7 @@ function preBashHandlerInner(event: HookEvent): HookOutput { const hint = surgicalHintFor(hintPath, false, isConfig, isDoc, false, target) const denial = surgicalHintFor(hintPath, false, isConfig, isDoc, false, target, lead) recordStat('session_hint', 0, 0) - return cdStripped ? pathHint(hintPath, denial) : cappedInterpreterRead(event, rawCmd, cmd, hint, denial) + return cdStripped ? pathHint(hintPath, denial) : cappedInterpreterRead(event, rawCmd, cmd, hint, denyOutput(denial)) } const psMethodRead = extractPowerShellFileMethodRead(cmd, event) @@ -605,7 +606,7 @@ function preBashHandlerInner(event: HookEvent): HookOutput { const hint = surgicalHintFor(hintPath, isEnv, isConfig, isDoc, isXml, target) const denial = surgicalHintFor(hintPath, isEnv, isConfig, isDoc, isXml, target, lead) recordStat('session_hint', 0, 0) - return cdStripped ? pathHint(hintPath, denial) : cappedInterpreterRead(event, rawCmd, cmd, hint, denial) + return cdStripped ? pathHint(hintPath, denial) : cappedInterpreterRead(event, rawCmd, cmd, hint, denyOutput(denial)) } // A plain-enumeration rg/grep structural search (whole-file symbols, headings, imports) has an exact index answer -- rewrite the command to it instead of just hinting, so the model gets the answer in this one tool result. Checked on rawCmd (not the cd-stripped cmd) ahead of the hint-only checks below: detectStructuralIndexRewrite's own detectFromCommand call rejects any `cd DIR &&` prefix as a compound command, which is the correct pass-through for that shape rather than something this call needs to special-case. diff --git a/tests/interpreter_read_shapes.test.ts b/tests/interpreter_read_shapes.test.ts new file mode 100644 index 000000000..8ffecb851 --- /dev/null +++ b/tests/interpreter_read_shapes.test.ts @@ -0,0 +1,125 @@ +// Which inline Python commands the Bash hook treats as a file read into the conversation, and what it does where the capped run cannot ship. CAPTURE (session transcripts 2026-08 to 2026-10): 16 refusals of Python `open()` reads, the suggested command run 0 times and 13 routed around; three were a compound command whose quoted heredoc writes a script (`python other.py && cat > x.py <<'EOF' ... open('data.json') ... EOF` then `python x.py`), and one, in a bypassPermissions session, was `python -I -c "...open(sys.argv[1])..." "" > ""`, whose output went to a file, refused with `token-goat grep "" ...`. The commands below are HAND-DERIVED to those shapes with the paths shortened. +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { extractPythonFileRead, pythonReadFeedsParser } from '../src/bash_extractors.js' +import { preBashHandler } from '../src/hooks_bash.js' +import { clearModuleCaches } from '../src/reset.js' +import type { HookOutput } from '../src/types.js' +import { makeHookEvent } from './helpers/hook-event.js' + +type Mode = 'default' | 'bypassPermissions' | 'auto' +let seq = 0 + +function bash(command: string, mode: Mode): HookOutput { + seq += 1 + return preBashHandler(makeHookEvent({ toolName: 'Bash', toolInput: { command }, sessionId: `interp-shapes-${seq}`, raw: { permission_mode: mode } })) +} + +const SCRIPT_WRITTEN_THEN_RUN = "python a.py && cat > b.py <<'EOF'\nimport json\nd = json.load(open('d.json'))\nprint(len(d['rows']))\nEOF\npython b.py" +const SCRIPT_WRITTEN_AND_RUN_ON_ONE_LINE = "cd tools && python a.py && cat > b.py <<'EOF' && python b.py\nimport json\nprint(json.load(open('d.json'))['rows'][0])\nEOF" +const ARGV_READ_TO_FILE = `python -I -c "import sys; print(open(sys.argv[1], encoding='utf-8').read())" "notes.txt" > "out.md"` +const PARSED = `python -c "import json; d = json.load(open('d.json')); print(len(d['rows']))"` +const PRINTED = `python -c "print(open('d.json').read())"` + +beforeEach(() => { + clearModuleCaches() +}) + +afterEach(() => { + delete process.env['TOKEN_GOAT_BASH_COMPRESS'] + clearModuleCaches() +}) + +describe('a compound command whose heredoc writes a script', () => { + it.each(['default', 'bypassPermissions'] as const)('is neither refused nor rewritten in %s mode', (mode) => { + expect(bash(SCRIPT_WRITTEN_THEN_RUN, mode).hookType).toBe('pass') + expect(bash(SCRIPT_WRITTEN_AND_RUN_ON_ONE_LINE, mode).hookType).toBe('pass') + }) + + it('reads no file out of a heredoc fed to anything but a Python reading its program from stdin', () => { + expect(extractPythonFileRead(SCRIPT_WRITTEN_THEN_RUN)).toBeNull() + expect(extractPythonFileRead("python run.py <<'EOF'\nopen('d.json') is data here\nEOF")).toBeNull() + }) + + it('still reads the heredoc a Python runs, and a later Python segment of the same command', () => { + expect(extractPythonFileRead("python - <<'PY'\nprint(open('d.json').read())\nPY")?.filePath).toBe('d.json') + expect(extractPythonFileRead("python3 -u < { + it.each(['default', 'bypassPermissions'] as const)('is neither refused nor rewritten in %s mode', (mode) => { + expect(bash(ARGV_READ_TO_FILE, mode).hookType).toBe('pass') + expect(bash(`python - <<'PY' >> log.txt\nprint(open('d.json').read())\nPY`, mode).hookType).toBe('pass') + }) + + it('still counts when only another stream is redirected, or the target is a standard stream', () => { + expect(extractPythonFileRead(`python -c "print(open('d.json').read())" 2> err.log`)?.filePath).toBe('d.json') + expect(extractPythonFileRead(`python -c "print(open('d.json').read())" > /dev/stdout`)?.filePath).toBe('d.json') + expect(extractPythonFileRead(`python -c "print(open('d.json').read())" 2>&1 | head -5`)?.filePath).toBe('d.json') + expect(extractPythonFileRead(`python -c "print(open('d.json').read() > '')"`)?.filePath).toBe('d.json') + }) + + it('without the redirect, the same command is a read', () => { + expect(extractPythonFileRead(ARGV_READ_TO_FILE.replace(' > "out.md"', ''))?.filePath).toBe('notes.txt') + }) +}) + +describe('where the capped run cannot ship', () => { + it('a read that parses the file gets advice and runs, in auto mode', () => { + const out = bash(PARSED, 'auto') + expect(out.hookType).toBe('context') + if (out.hookType === 'context') expect(out.context).toContain('token-goat json-outline "d.json"') + expect(bash(`python -c "import yaml; print(yaml.safe_load(open('c.yaml'))['jobs'])"`, 'auto').hookType).toBe('context') + }) + + it('a read that parses the file gets advice and runs when compression is switched off', () => { + process.env['TOKEN_GOAT_BASH_COMPRESS'] = '0' + expect(bash(PARSED, 'default').hookType).toBe('context') + }) + + it('a read that prints the raw file is still refused', () => { + const out = bash(PRINTED, 'auto') + expect(out.hookType).toBe('deny') + if (out.hookType === 'deny') expect(out.message).toContain('Python `open()` file reads bypass read hooks.') + }) + + it('a parsing read still runs under the cap where the capped run can ship', () => { + const out = bash(PARSED, 'default') + expect(out.hookType).toBe('rewriteInput') + if (out.hookType === 'rewriteInput') expect(String(out.updatedInput['command'])).toContain(' -f passthrough ') + }) + + it('names a command that runs as printed for a .txt file, never a grep for ``', () => { + const out = bash(`python -c "print(open('notes.txt').read())"`, 'auto') + expect(out.hookType).toBe('deny') + if (out.hookType === 'deny') { + expect(out.message).toContain('token-goat read "notes.txt::1-80"') + expect(out.message).not.toContain('') + } + }) +}) + +describe('pythonReadFeedsParser', () => { + it.each([ + "import json; d = json.load(open('d.json')); print(d['a'])", + "import json; print(json.loads(open('d.json').read())['a'])", + "import csv\nwith open('t.txt') as f:\n rows = list(csv.DictReader(f))\nprint(len(rows))", + "import yaml; print(yaml.safe_load(open('c.yaml'))['jobs'])", + "from openpyxl import load_workbook; print(load_workbook('b.xlsx').sheetnames)", + "import pandas as pd; print(pd.read_json('d.json').shape)", + ])('is true for a parse: %s', (snippet) => { + expect(pythonReadFeedsParser(snippet)).toBe(true) + }) + + it.each([ + "print(open('d.json').read())", + "import json; print(open('d.json').read()); json.loads('{}')", + "import json\nwith open('d.json') as f:\n for line in f:\n print(line)\njson.loads('{}')", + "import json\nfor line in open('d.json'):\n print(line)\njson.loads('{}')", + "print(len(open('d.json').read()))", + ])('is false for a read of the raw text: %s', (snippet) => { + expect(pythonReadFeedsParser(snippet)).toBe(false) + }) +}) diff --git a/tests/interpreter_read_soft_redirect.test.ts b/tests/interpreter_read_soft_redirect.test.ts index 9ee57f744..3b612dd89 100644 --- a/tests/interpreter_read_soft_redirect.test.ts +++ b/tests/interpreter_read_soft_redirect.test.ts @@ -25,6 +25,8 @@ function capHint(command: string): string { // CAPTURE: two of the 217 refused commands, verbatim from the transcripts (the first a Claude Code session, the second one line of a multi-line one); both project a few fields out of one JSON file. const PROJECTION_ONE_LINE = `python -c "import json;d=json.load(open('C:/Users/zelys/AppData/Local/Temp/tweet.json',encoding='utf-8'));t=d.get('tweet',d);print(t.get('text') or t.get('raw_text',{}).get('text'))"` const PROJECTION_MULTI_LINE = `python -c "\nimport json\nd=json.load(open('.selfimprove/targets.json'))\nfor i,t in enumerate(d['targets']): print(i, t['id'], '|', t['title'][:78])\n"` +// HAND-DERIVED from the first capture: the same file printed as raw text rather than parsed, the read the refusal is kept for where the wrapper cannot run. +const DUMP_ONE_LINE = `python -c "print(open('C:/Users/zelys/AppData/Local/Temp/tweet.json',encoding='utf-8').read())"` describe('inline interpreter file reads run capped instead of refused', () => { const saved = process.env['TOKEN_GOAT_BASH_COMPRESS'] @@ -54,14 +56,17 @@ describe('inline interpreter file reads run capped instead of refused', () => { expect(capHint(wrapped(preBashHandler(bashEvent(`powershell -Command "[IO.File]::ReadAllText('README.md')"`))))).toContain('token-goat section "README.md::') }) - it('refuses with the full message when compression is off, so the wrapper cannot run', () => { + it('refuses a raw dump with the full message when compression is off, so the wrapper cannot run, and advises a parsing read without refusing it', () => { process.env['TOKEN_GOAT_BASH_COMPRESS'] = '0' - const result = preBashHandler(bashEvent(PROJECTION_ONE_LINE)) + const result = preBashHandler(bashEvent(DUMP_ONE_LINE)) expect(result.hookType).toBe('deny') if (result.hookType === 'deny') { expect(result.message).toContain('Python `open()` file reads bypass read hooks.') expect(result.message).toContain('token-goat json-outline "C:/Users/zelys/AppData/Local/Temp/tweet.json"') } + const projection = preBashHandler(bashEvent(PROJECTION_ONE_LINE)) + expect(projection.hookType).toBe('context') + if (projection.hookType === 'context') expect(projection.context).toContain('token-goat json-outline "C:/Users/zelys/AppData/Local/Temp/tweet.json"') }) // PROVENANCE: HAND-DERIVED. The extractors match the command past its leading assignments, but the wrapper runs the command as written, so the read still sees the variable it set. @@ -75,9 +80,10 @@ describe('inline interpreter file reads run capped instead of refused', () => { }) it('refuses a read whose own prefix switches compression off, as the environment opt-out does, since the cap is compression', () => { - const result = preBashHandler(bashEvent(`TOKEN_GOAT_BASH_COMPRESS=0 ${PROJECTION_ONE_LINE}`)) + const result = preBashHandler(bashEvent(`TOKEN_GOAT_BASH_COMPRESS=0 ${DUMP_ONE_LINE}`)) expect(result.hookType).toBe('deny') if (result.hookType === 'deny') expect(result.message).toContain('Python `open()` file reads bypass read hooks.') + expect(preBashHandler(bashEvent(`TOKEN_GOAT_BASH_COMPRESS=0 ${PROJECTION_ONE_LINE}`)).hookType).toBe('context') }) it('wraps a heredoc read too, the shape 81 of the 217 measured refusals arrived in', () => { From 25b4d8f5766aae1cb9822f78a52edfdf61b1c10e Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 06:03:13 -0500 Subject: [PATCH 13/92] fix(hooks): say a file was partially read when only a window of it was, before a token-goat read The Bash hook's cross-reference for a token-goat read said the file "was already fully read via the Read tool this session" whenever wasFileReadThisSession held, which a Read of 30 lines of a 1,069-line file is enough for. It now keeps that note for a whole-file read only; after ranged reads it names the windows on record, and for a line-range read (@A-B, ::A-B) only the windows that overlap it, saying nothing when none do. The two tests that set the note up with a ranged Read and expected the full-read wording now expect the partial one. --- src/hooks_bash.ts | 20 +++++- tests/bash_cd_prefixed_cli_read_key.test.ts | 3 +- ...h_cli_read_note_follows_read_shape.test.ts | 72 +++++++++++++++++++ tests/hook_prose_names_quoted.test.ts | 3 +- 4 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 tests/bash_cli_read_note_follows_read_shape.test.ts diff --git a/src/hooks_bash.ts b/src/hooks_bash.ts index afeca691f..3aed313fd 100644 --- a/src/hooks_bash.ts +++ b/src/hooks_bash.ts @@ -6,7 +6,7 @@ import { leadWithCommand, docSectionHint, fileSubject, nameSubject, quotedArg, q import { contextOutput, denyOutput, passOutput, getCwd } from './hooks_common.js' import { applyHintTracking, classifyBashHint, meetsSavingsFloor, logSuppressedDetection } from './hint_stats.js' import type { HookOutput } from './types.js' -import { getBashOutputId, getCurlDownloadPath, clearCurlDownload, dropFileLineRangesIfChanged, getFileLineRanges, recordBashStartCwd, recordFileLineRange, wasHintShown, markHintShown, wasCliReadThisSession, wasFileReadThisSession } from './session.js' +import { getBashOutputId, getCurlDownloadPath, clearCurlDownload, dropFileLineRangesIfChanged, getFileLineRanges, recordBashStartCwd, recordFileLineRange, wasHintShown, markHintShown, wasCliReadThisSession, wasFileReadThisSession, wasFileFullyReadThisSession } from './session.js' import { resolveIndexPath, displaySafePath, hostPathOfTypedPath, isFileAtIndexKey } from './paths.js' import { shortFingerprint } from './fingerprint.js' import { isBuildCommand, getMonitoringRecallHint, isTestRunnerCommand } from './hints/lang_patterns.js' @@ -211,6 +211,15 @@ function wholeFileRange( } } +/** The windows of a file this session read that bear on a `token-goat read` of `spec`: every one for a symbol or section, whose lines the spec does not say, and for a line range (`@A-B`, `@A`, `::A-B`) only those that overlap it. */ +function partialReadCover(ranges: ReadonlyArray, spec: string): Array { + const lines = /(?:@|::)(\d+)(?:-(\d+))?$/.exec(spec) + if (lines === null) return [...ranges] + const start = parseInt(lines[1] as string, 10) + const end = lines[2] !== undefined ? parseInt(lines[2], 10) : start + return ranges.filter(([a, b]) => a <= end && b >= start) +} + /** pre_tool_use handler for the Bash tool. Emits a recall hint when the command is a known build tool and its output was already captured this session. Passes through for all other commands. */ /** The answers about how a command is written rather than a file it reads, which preBashHandler's Read-rule gate leaves standing. */ const SYNTAX_OUTPUTS = new WeakSet() @@ -771,7 +780,14 @@ function preBashHandlerInner(event: HookEvent): HookOutput { notes.push('You already ran this exact `token-goat ' + tgRead.sub + '` query earlier this session — check your context above before re-running it.') } if (tgRead.filePath !== null && wasFileReadThisSession(tgRead.filePath)) { - notes.push(sentenceStart(fileSubject('', displaySafePath(tgRead.filePath))) + ' was already fully read via the Read tool this session — that content may already cover this.') + const subject = sentenceStart(fileSubject('', displaySafePath(tgRead.filePath))) + if (wasFileFullyReadThisSession(tgRead.filePath)) { + notes.push(subject + ' was already fully read via the Read tool this session — that content may already cover this.') + } else { + // Only windows of it were read, which the note used to call a full read: name the ones on record, and say nothing when none touches the lines this read asks for. + const covered = partialReadCover(getFileLineRanges(tgRead.filePath), tgRead.spec) + if (covered.length > 0) notes.push(subject + ' was partially read earlier this session (lines ' + covered.map(([start, end]) => start + '-' + end).join(', ') + '); check whether those lines already hold this.') + } } if (notes.length > 0) { recordStat('session_hint', 0, 0) diff --git a/tests/bash_cd_prefixed_cli_read_key.test.ts b/tests/bash_cd_prefixed_cli_read_key.test.ts index 1bb5c28b9..9c02b13ae 100644 --- a/tests/bash_cd_prefixed_cli_read_key.test.ts +++ b/tests/bash_cd_prefixed_cli_read_key.test.ts @@ -20,7 +20,8 @@ import type { HookOutput } from '../src/types.js' const LINES = Array.from({ length: 80 }, (_, i) => (i === 0 ? 'export function foo(): number { return 1 }' : `export const value${i} = ${i} * 2`)) const BODY = LINES.join('\n') const REPEAT_NOTE = 'You already ran this exact `token-goat read` query' -const READ_NOTE = 'was already fully read via the Read tool' +// partialRead below covers lines 50-64 only, so the note names that window rather than a full read. +const READ_NOTE = 'was partially read earlier this session (lines 50-64)' const dirs: string[] = [] diff --git a/tests/bash_cli_read_note_follows_read_shape.test.ts b/tests/bash_cli_read_note_follows_read_shape.test.ts new file mode 100644 index 000000000..6608c7dc1 --- /dev/null +++ b/tests/bash_cli_read_note_follows_read_shape.test.ts @@ -0,0 +1,72 @@ +// The note a `token-goat read` gets when its file was already read with the Read tool must describe the read that happened. CAPTURE (a research session, 2026-10): after a Read of lines 225-254 of the 1,069-line src/tool_filters/helpers.ts, `token-goat read` of that file was told it "was already fully read via the Read tool this session". The fixture is HAND-DERIVED to that shape: a 400-line module read through the real Read pre hook with offset 225 and limit 30, then queried through the real Bash pre hook. +import * as fs from 'node:fs' +import * as path from 'node:path' + +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { preBashHandler } from '../src/hooks_bash.js' +import { preReadHandler } from '../src/hooks_read.js' +import { normalizePath } from '../src/paths.js' +import { clearModuleCaches } from '../src/reset.js' +import type { HookOutput } from '../src/types.js' +import { ROOT } from './helpers/bundle.js' +import { makeHookEvent } from './helpers/hook-event.js' + +const FULL_NOTE = 'was already fully read via the Read tool' +let dir: string +let file: string +let session: string +let seq = 0 + +function read(input: Record): HookOutput { + return preReadHandler(makeHookEvent({ toolName: 'Read', toolInput: { file_path: file, ...input }, sessionId: session, raw: { cwd: dir } })) +} + +function cliRead(spec: string): HookOutput { + return preBashHandler(makeHookEvent({ toolName: 'Bash', toolInput: { command: `token-goat read "${spec}"` }, sessionId: session, raw: { cwd: dir } })) +} + +function context(out: HookOutput): string { + return out.hookType === 'context' ? out.context : '' +} + +beforeEach(() => { + clearModuleCaches() + seq += 1 + session = `cli-read-shape-${process.pid}-${seq}` + fs.mkdirSync(path.join(ROOT, '.tmp'), { recursive: true }) + dir = normalizePath(fs.mkdtempSync(path.join(ROOT, '.tmp', 'tg-cli-read-shape-'))) + file = `${dir}/helpers.ts` + const lines = Array.from({ length: 400 }, (_, i) => (i === 230 ? 'export function byteLength(s: string): number { return Buffer.byteLength(s) }' : `export const value${i} = ${i} * 2`)) + fs.writeFileSync(file, lines.join('\n') + '\n') +}) + +afterEach(() => { + fs.rmSync(dir, { recursive: true, force: true }) + clearModuleCaches() +}) + +describe('the Read cross-reference on a token-goat read', () => { + it('names the lines a ranged Read covered instead of claiming a full read', () => { + expect(read({ offset: 225, limit: 30 }).hookType).not.toBe('deny') + const note = context(cliRead(`${file}::byteLength`)) + expect(note).not.toContain(FULL_NOTE) + expect(note).toContain('was partially read earlier this session (lines 225-254)') + }) + + it('stays quiet when the lines asked for are not ones the ranged Read covered', () => { + read({ offset: 225, limit: 30 }) + expect(cliRead(`${file}@300-340`).hookType).toBe('pass') + expect(cliRead(`${file}::360-380`).hookType).toBe('pass') + }) + + it('names the window when the lines asked for overlap it', () => { + read({ offset: 225, limit: 30 }) + expect(context(cliRead(`${file}@240-260`))).toContain('(lines 225-254)') + }) + + it('keeps the full-read note after a whole-file Read', () => { + read({}) + expect(context(cliRead(`${file}::byteLength`))).toContain(FULL_NOTE) + }) +}) diff --git a/tests/hook_prose_names_quoted.test.ts b/tests/hook_prose_names_quoted.test.ts index 8aacd4b27..9fe334884 100644 --- a/tests/hook_prose_names_quoted.test.ts +++ b/tests/hook_prose_names_quoted.test.ts @@ -18,7 +18,8 @@ import type { HookOutput } from '../src/types.js' // HAND-DERIVED: a module whose `foo` the surgical read names; the backtick file name is the case under test, and every shell accepts it in single quotes. const LINES = Array.from({ length: 80 }, (_, i) => (i === 0 ? 'export function foo(): number { return 1 }' : `export const value${i} = ${i} * 2`)) -const READ_NOTE = 'was already fully read via the Read tool' +// The Read below covers lines 50-64 only, so the note names that window rather than a full read. +const READ_NOTE = 'was partially read earlier this session (lines 50-64)' const DOWNLOAD_NOTE = 'was already downloaded earlier this session.' const dirs: string[] = [] From b41f83f48a99f087d71ca1dd689fb0e79eba07b1 Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 06:07:54 -0500 Subject: [PATCH 14/92] fix(hooks): offer no map hint for a directory listing that asks for sizes, times or permissions ls -la dist/ | head, ls -lR, eza --long and tree -h drew "Use token-goat map --compact", and map prints files and symbols, never the metadata a long listing asks for. extractDirectoryListing now declines a listing whose first command carries a metadata flag of its tool (ls -l/-h/-t/-S/-s/-i/--full-time and kin, eza --long/--git/-l and kin, tree -h/-s/-p/-D and kin), bundled short flags included; a listing of names alone still gets the hint. The eza --long pattern that only ever matched a long listing is gone, and eslint-suppressions.json drops the backtracking suppression it carried. The tests that asserted a map hint for ls -la, ls -lR, ls -laR and eza --long now assert none, beside new positive cases for ls -a | head and ls -aR. The hook entry's eager set grows by 788 bytes to 2,596,163 over 60 chunks, 8,651 over the dist_chunks_deduped ceiling, so the ceiling is re-based on the measurement plus the usual 4 KB and its doc comment names what bought the bytes. --- eslint-suppressions.json | 2 +- src/bash_extractors.ts | 19 ++++++++-- tests/guards/dist_chunks_deduped.test.ts | 4 +-- tests/hooks_bash.test.ts | 45 ++++++++++-------------- 4 files changed, 39 insertions(+), 31 deletions(-) diff --git a/eslint-suppressions.json b/eslint-suppressions.json index 93d87397d..8dca05bb2 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -1,7 +1,7 @@ { "src/bash_extractors.ts": { "regexp/no-super-linear-backtracking": { - "count": 18 + "count": 17 } }, "src/cli_bootstrap_audit.ts": { diff --git a/src/bash_extractors.ts b/src/bash_extractors.ts index 8ce1ddec3..819455770 100644 --- a/src/bash_extractors.ts +++ b/src/bash_extractors.ts @@ -1057,10 +1057,25 @@ export function extractToolResultsFile(cmd: string): { path: string } | null { return null } -/** Returns true when the command is a directory listing (eza --long or ls … | head) for which `token-goat map --compact` is a cheaper alternative. */ +/** The flags of each listing command that ask for something about the files besides their names: sizes, times, owners, permissions, inodes, or an order by one of them. Short ones may come bundled (`-la`, `-lhS`). */ +const LISTING_METADATA_FLAGS: Readonly> = { + ls: { short: /[lgnoshtSiuc]/, long: /^--(?:format=(?:long|verbose)|full-time|time|size|human-readable|si|inode|author|numeric-uid-gid|sort=(?:size|time)|block-size)/ }, + eza: { short: /[lghBmuUiS]/, long: /^--(?:long|git|bytes|binary|header|group|modified|accessed|created|changed|inode|blocksize|links|octal-permissions|time|total-size)/ }, + tree: { short: /[hspugD]/, long: /^--(?:du|si|timefmt)/ }, +} + +/** True when a listing asks for metadata. `token-goat map` lists files and symbols, never a size, time or permission, so it cannot stand in for `ls -la dist/`. */ +function listingAsksForMetadata(cmd: string): boolean { + const words = splitCommandWords(splitShellSegments(cmd)[0] ?? '') + const flags = LISTING_METADATA_FLAGS[words[0] ?? ''] + if (flags === undefined) return false + return words.slice(1).some((w) => (w.startsWith('--') ? flags.long.test(w) : /^-[A-Za-z]/.test(w) && flags.short.test(w.slice(1)))) +} + +/** Returns true when the command is a directory listing (eza --tree, tree, a recursive ls, or ls … | head) for which `token-goat map --compact` is a cheaper alternative, and it asks only for names. */ export function extractDirectoryListing(cmd: string): boolean { + if (listingAsksForMetadata(cmd)) return false return ( - /^eza\s+.*--long\s+\S+/.test(cmd) || /^eza\s+.*--tree/.test(cmd) || /^tree(\s|$)/.test(cmd) || /^ls\s+(?:\S+\s+)*-[a-zA-Z]*R[a-zA-Z]*(?:\s|$)/.test(cmd) || diff --git a/tests/guards/dist_chunks_deduped.test.ts b/tests/guards/dist_chunks_deduped.test.ts index 4a7950569..8012462f4 100644 --- a/tests/guards/dist_chunks_deduped.test.ts +++ b/tests/guards/dist_chunks_deduped.test.ts @@ -33,8 +33,8 @@ function modulesIn(files: readonly string[]): Set { return out } -/** Ceiling on what the hook entry pulls in statically, mirroring the core's ceiling in core_bundle_stays_split.test.ts. A bridge `import()`s this bundle on nearly every tool call and V8 parses every byte of the eager set before running any of it. Measured at 1.837 MB across 7 chunks both before and after the two builds were merged; the headroom is a regression trip-wire, not a budget to re-tune on every dependency change. It went 2.4 MB -> 2.6 MB when the Fortran, Pascal, MATLAB and CMake adapters (41 KB, eager like every other adapter) grew the set, and back to 2.25 MB once the adapters moved behind parser.ts's dynamic import of languages/registry.ts. Measured on one machine across that move: 2.430 MB over 7 chunks before, 2.167 MB over 8 after, and 47-53 ms to import the hook bundle before against 42-43 ms after. The 4 KB above the round 2.25 MB was bought by EXIF orientation correction in image_engine.ts, which has to be eager: the pre-read hook is what shrinks images, so the rotation runs on the hook path or not at all. Measured 2,360,603 bytes over 18 chunks with it, against a 2,359,296-byte line it missed by 1,307. A further 2 KB was bought by the structural-index rewrite in bash_structural_index.ts (batch Q): it decides whether to rewrite an `rg`/`grep` structural search into a `token-goat outline`/`imports` call, and that decision has to run inside the synchronous pre_tool_use Bash hook before the tool executes, so it cannot be deferred behind a dynamic import the way a language adapter can. Every module it imports (paths.js, parser_types.js, language_specs.js, tool_filters/helpers.js, bash_extractors.js, fingerprint.js, index_reader.js) was already eager via an existing hook path; the added bytes are its own ~130 lines of recognizer logic. Measured 2,365,172 bytes over 19 chunks with it, against a 2,363,392-byte line it missed by 1,780. A further 3 KB was bought by Batch S's hook wall-clock timing: relayInProcess (relay.ts) now imports recordStat from stats.ts to measure and persist its own duration on every hook invocation, which is unavoidably eager for the same reason as the structural-index rewrite above -- the timer has to close before relayInProcess returns, on every hook, not behind a dynamic import. stats.ts itself was already eager via other hook paths (session_hint, image_shrink, ...); the added bytes are recordStat's new duration_ms plumbing and pruneHookStats. The CLI/doctor-only read side (hookLatencyBreakdown, renderHookLatencyStats) lives in the separate hook_latency.ts on purpose, so it never joins this eager set at all. Measured 2,367,105 bytes over 19 chunks with it, against a 2,365,440-byte line it missed by 1,665. Raise this only for something equally unavoidable, and say here what bought it. A further 1 KB was bought by the fix that stops recordStat holding the hook path for db.ts's full 15s busy_timeout under contention: recordStat now opens its own short-budget connection instead of reusing the shared, patient one getDb() caches for indexing and the worker, and closes it before returning. That connection has to be opened and torn down synchronously inside recordStat, on every hook, same reason as the timing plumbing above -- it cannot be deferred behind a dynamic import. Measured 2,368,665 bytes over 19 chunks with it, against a 2,368,349-byte line it missed by 316. A further 1 KB was bought by `hint_stats.defiance_threshold_pct` (batch b02 J4): the same five config rows as the entry below, plus the `isSuppressionCategory` branch in shouldSuppress, which unlike checkHookLatency really is eager -- hint_stats.ts runs inside the pre/post tool-use hooks. Measured 2,370,518 bytes over 19 chunks with it, against the 2,370,560-byte line the entry below left, which it cleared by 42. Two config keys in one batch consumed 1,115 of this ceiling's bytes between them, which is the number to plan against. A further 1 KB was bought by `hooks.latency_budget_ms` (batch b02 J3), and this entry is worth reading before adding any config key at all, because the cost is not obvious from the diff. config.ts is eager on the hook path -- every hook calls loadConfig -- so a key's five rows (its NUMERIC_FIELD_BOUNDS entry, its two _buildConfig lines, its CONFIG_KEY_ENV_OVERRIDES entry and its saveConfig serialize arm) all land in this set even when the only code that reads the key is CLI-only, as checkHookLatency is. Measured on one machine by building both trees: 2,369,403 bytes over 19 chunks at the parent commit against 2,369,870 with the key, so one key costs 467 bytes here and the parent commit had 133 bytes of headroom left. Unavoidable in the sense that matters: the rows are what make the key settable, and there is no deferred-import shape for a table the eager loader indexes. Budget roughly 0.5 KB of this ceiling per future config key rather than discovering it as a red guard. A further 4 KB was bought by making the parser fingerprint per-language: src/parser_fingerprint.ts now carries a 49-entry language-to-digest map beside the shared digest, and parser_stamp.ts's parserFingerprintForLanguage resolves a row's stamp through it. Eager because the three freshness gates that consult it (worker.ts's drain, reconcile.ts's sweep, fold_delivery.ts's body-fold gate) are all on the hook path, and a digest map is data with no deferred-import shape. Measured by building both trees: 2,372,896 bytes over 19 chunks at the parent commit against 2,374,896 with the map, so the map costs 2,000 bytes here and the parent commit had 1,760 bytes of headroom left. 4 KB rather than 1 leaves 3,856 bytes of headroom, deliberately more than the 1,766 the entry below left, since a language adapter added later grows this map by another ~30 bytes and should not be the change that discovers a red guard. A further 3 KB was bought by merging origin/main's doctor work (findTopIndexedProjects, checkUnmappedTools' age window, checkVscodeProjectMcp and cleanupDeprecatedVscodeProjectMcp) into the local oversized-db category breakdown. cli_doctor.ts is not itself hook-eager, but the merge also pulled origin's changes to stats.ts and util.ts, both of which are. Measured 2,372,890 bytes over 19 chunks after the merge, against the 2,371,584-byte line it missed by 1,306; 3 KB rather than 2 leaves 1,766 bytes of headroom, since the previous entry's 42-byte margin meant the next unrelated change would have discovered this red rather than planned for it. A further 1 KB was bought by resolving Claude Code's transcript root through `claudeConfigDir()` in waste.ts, which honours `CLAUDE_CONFIG_DIR` instead of hardcoding `os.homedir()`. Eager because hooks_bash_post.ts imports projectTranscriptsDir to find the per-session tool-results directory on the Bash hook path, and a path accessor read on every such call has no deferred-import shape. Measured by building three trees on one machine: 2,378,447 bytes over 19 chunks at the parent commit, 2,378,618 with the sessions-directory fix that precedes this one, and 2,378,804 with this one, so the accessor costs 186 bytes here and the parent commit had 305 bytes of headroom left. 1 KB leaves 972 bytes of headroom. A further 6 KB was bought by the two embedding gates. src/asset_extensions.ts is hook-eager because image_shrink.ts, which runs on the pre-read hook path, now takes its image-extension list from there instead of holding a private copy -- parser.ts cannot import image_shrink.ts (it pulls in the hook registry, the harness bridges and the stats ledger), so the one list moved down to a leaf both can reach. src/embed_backfill.ts arrives through worker.ts. Three trees measured on one machine, rebuilt between each: 2,378,804 bytes over 19 chunks at the parent commit (the figure the paragraph above records); 2,382,618 over 19 with the gates and the shared extension module (+3,814); 2,384,032 over 19 with the backfill sweep wired in as well (+1,414 more, +5,228 over the parent), which missed the 2,379,776-byte line by 4,256. Rebuilt four times at that final state, byte-identical each time. The chunk count never moved, so none of this is a new chunk being dragged eager. 6 KB leaves 1,888 bytes of headroom. A further 16 KB was bought by the hint-efficacy measurement batch. hint_stats.ts, hooks_bash.ts, hooks_common.ts and db.ts are all hook-eager, and the batch touches every one of them: hint builders now hand their own file path to the efficacy ledger instead of it being regex-scraped back out of the rendered hint (a `correlators` field on the context HookOutput, a `pathHint` wrapper and 20 converted call sites in hooks_bash.ts); a range hint must now price its own proposal against the window the command asked for before it may emit, which pulls bash_range_savings.ts and line_regions.ts onto the hook path; and two `hint_emissions` columns, with their migrations, separate an emission that was shown but unscoreable from a detection that was never shown at all. Measured by building both trees on one machine, ceiling temporarily forced to 1 to read the figure at the parent: 2,385,404 bytes over 19 chunks at c6968099 -- which had 516 bytes of headroom, so the first step of the batch alone was going to find this red -- against 2,397,441 with all of it, +12,037 over the batch. The last 1,543 of that is the batch's final step, which gives the whole-file deny a real indexed name to print instead of a `SectionHeading`/`KEY_NAME` placeholder that does not run: bash_surgical_target.ts is new, but index_reader.ts and fingerprint.ts were already pulled eager by the range gate above, so it costs its own compiled bytes and nothing else. Rebuilt twice at that final state, byte-identical each time. The chunk count never moved off 19, so none of this is a new chunk being dragged eager: it is compiled bytes on modules the hook path already loaded. 12 KB would have left 767 bytes of headroom, well under the slack every paragraph above keeps, so this takes 16 KB and leaves 4,863 bytes. A further 1 KB was bought by the batch that made the hook path read its own argv and its own printed output. hooks_bash.ts and hooks_bash_commands.ts are hook-eager and carry most of it: pipelineShapeFilter now returns the first stage's argv alongside the filter instead of dropping it, because the caller was passing an empty array and every argv-reading behaviour in every filter was therefore dead on a piped command. manifest.ts's manifestPrintedPaths now samples the rendered manifest rather than re-walking the selection, session_store.ts's file cap ranks edited entries above read ones, and pending_context.ts gained an import of hint_suggestion_guard.ts -- a zero-import leaf already reached through relay.ts, which is why the chunk count did not move. Measured after a fresh build: 2,403,355 bytes over 22 chunks, against the 2,403,328-byte line it missed by 27, and against the 2,402,399 bytes over 22 chunks the entry below records at the parent commit, so the batch costs 956 bytes. Rebuilt and re-measured byte-identical. 1 KB leaves 997 bytes of headroom, the same shape as the entries below, so the next small change is not the one that discovers a red guard. A further 1 KB was bought by the deferred-hint delivery fix and the fold it shares. relay.ts now clears the pending-context queue only once the emitted output is confirmed to carry the text, which puts pending_context.ts on the hook entry's eager graph where it previously arrived only through hooks_session.ts; src/tool_name_fold.ts is the leaf that replaced three copies of foldToolName, one of which had lost its type guard; and overflow_guard.ts's trim now classifies the content once and spends the same divisor the entry check measured with. Measured after the batch: 2,402,399 bytes over 22 chunks, against the 2,402,304-byte line it missed by 95. 1 KB rather than the 95 bytes it costs leaves 929 bytes of headroom, in the same shape as the entries above, so the next small change is not the one that discovers a red guard. A further 1 KB was bought by the batch that gave the Copilot CLI shim own-property lookups and the compaction manifest its subagents' web fetches. copilot_cli.ts is hook-eager through the bridge registry and manifest.ts through the pre_compact hook, and most of the cost is the shim's own comment text rather than its code: the shim is a template literal shipped verbatim, so a comment inside it is bytes on every install instead of being stripped the way a source comment is. Trimming that comment to its essentials, with the full rationale left where it already lived beside the event map, took the batch from +1,191 bytes to +913. No new module and no new chunk -- 22 across both measurements. Measured with the working tree restored to HEAD for the parent build rather than stashed, ceiling temporarily forced to 1 to read the figure: 3d1a799e 2,403,312 bytes; the batch 2,404,225, which cleared the previous 2,404,352-byte line by 127. Rebuilt at that final state, byte-identical. 127 bytes is under every margin the paragraphs above keep and would make the next unrelated one-liner the change that discovers a red guard, so this takes 1 KB and leaves 1,151 bytes. A further 2 KB was bought by the same launcher-options and yarn-built-ins batch the startup paragraph in tests/guards/core_bundle_stays_split.test.ts describes. helpers.ts is hook-eager through the bash filter registry, and the cost is the two string-literal lookup sets it gained: LAUNCHER_VALUE_FLAGS and the 47-name YARN_BUILTIN_SUBCOMMANDS. No new module and no new chunk -- 22 across both measurements -- so this is compiled bytes on a module the hook path already loaded. Measured with the working tree restored to HEAD for the parent build rather than stashed, ceiling temporarily forced to 1 to read the figure: 558b1c96 2,404,169 bytes; the batch 2,405,669 (+1,500), which overran the previous 2,405,376-byte line by 293. Rebuilt twice at each state, byte-identical both times. A 1 KB bump would clear it by 731 bytes, under every margin the paragraphs above keep, so this takes 2 KB and leaves 1,755 bytes. A further 2 KB was bought by moving the worker's transient-read-failure counters out of `files` into their own `index_retries` table, and by the sweep that reclaims a counter whose file is gone. db.ts is hook-eager and carries most of it: the table, its expression index, and -- crucially -- the rationale, which first landed as an SQL `--` comment inside the SCHEMA_SQL template literal. A comment there is a string, not a comment: the bundler ships it verbatim on every install instead of stripping it, which is the same trap the Copilot CLI shim paragraph above records. Moving that prose to the docblock above SCHEMA_SQL took the batch from +785 bytes to +172. index_prune.ts and config_commands.ts are also hook-eager and carry the rest. No new module and no new chunk: 25 across both measurements. Measured with the working tree restored to HEAD for the parent build rather than stashed, ceiling temporarily forced to 1 to read the figure: c7558c84 2,415,615 bytes, which had exactly 1 byte of headroom; the batch 2,415,787, overrunning the previous 2,415,616-byte line by 171. Rebuilt at that final state, byte-identical. A 1 KB bump would clear it by 853 bytes, under every margin the paragraphs above keep, so this takes 2 KB and leaves 1,877 bytes. A further 2 KB was bought by the `refs.context` storage cap. parser.ts is hook-eager, and the cost is boundRefContext plus the docblock explaining why a ref's context is windowed around its column rather than sliced from the head of its line -- on a document written as one long line, every reference stored the whole file. No new module and no new chunk: 25 across both measurements. Measured with the working tree restored to HEAD for the parent build rather than stashed, ceiling temporarily forced to 1 to read the figure: HEAD 2,413,046 bytes; the fix 2,413,859 (+813), which overran the previous 2,413,568-byte line by 291. A 1 KB bump would clear it by 733 bytes, under every margin the paragraphs above keep, so this takes 2 KB and leaves 1,757 bytes. A further 2 KB was bought by teaching KubectlFilter to leave a structured `-o json`/`-o yaml`/template listing alone instead of truncating it to a head and a tail. tool_filters/containers.ts is hook-eager through the bash filter registry, the same path the launcher-options paragraph above describes for helpers.ts, so its compiled bytes land here whether or not a hook ever runs kubectl. The cost is the flag reader -- `-o` has four spellings, separate, `=`-joined and glued to the short flag -- plus the format regex, which is a string literal the bundler ships verbatim rather than a comment it strips. That regex first listed all nine format names kubectl accepts and now matches four prefixes instead, every suffixed spelling being an extension of one of them, which took the batch from +755 bytes to +683. No new module and no new chunk: 25 across both measurements. Measured with the working tree restored to HEAD for the parent build rather than stashed: HEAD 2,417,277 bytes, which had 387 bytes of headroom; the batch 2,417,960, overrunning the previous 2,417,664-byte line by 296. Rebuilt twice at that final state, byte-identical each time. A 1 KB bump would clear it by 728 bytes, under every margin the paragraphs above keep, so this takes 2 KB and leaves 1,752 bytes. A further 2 KB was bought by PowerShell runner resolution and Windows PowerShell harness command wrapping with base64 round-trip validation. src/shell.ts and src/hooks_bash.ts are hook-eager, adding resolvePowerShell and canRunPowerShell helpers plus base64 rewrite logic so Copilot CLI and Codex on Windows execute wrapped commands natively under PowerShell rather than bash without quoting hazards. A further 7 KB was bought by project notes reaching session start and the compaction manifest, plus the passthrough filter and cap hint an inline interpreter read now runs through. hooks_session_start.ts and manifest.ts are hook-eager, so project_memory.ts joined the eager set whole, writer included: 4,828 of the batch's bytes. A deferred import does not fit: preCompactHandler is synchronous, and manifestPrintedPaths must rebuild the manifest under the same character budget the hook printed it with, which the notes are part of, so they would have to be threaded through both. No new chunk: 25 across both measurements. Measured with HEAD built in a separate worktree rather than by restoring this one: HEAD 2,421,826 bytes, already 66 over the previous 2,421,760-byte line on this machine though not on the Linux runner; the batch 2,427,394 (+5,568), built twice, byte-identical. 6 KB would clear it by 510 bytes, under every margin above, so this takes 7 KB and leaves 1,534. A further 2 KB was bought by honoring an inline `TOKEN_GOAT_BASH_COMPRESS=0` prefix across a whole command, pipes and `$env:` assignments included, and by not recording a result the harness persisted to disk as already served. bash_extractors.ts and hooks_bash.ts are hook-eager; the cost is the assignment scanner, whose two regexes are string literals the bundler ships verbatim. No new module and no new chunk: 25 across both measurements. Measured in the checkout with HEAD's three changed sources swapped in and then the fix's, since a scratch copy's path length alone moved the figure by about 1.6 KB: HEAD 2,428,011 bytes, the fix 2,431,008 (+2,997), overrunning the previous 2,428,928-byte line by 2,080. 294 of those bytes are the heredoc scan, which walks lines instead of using a regex that the super-linear-backtracking lint rule rejects. This takes 3 KB and leaves 992. A further 15 KB was bought by wiring Claude Code's PostToolUseFailure event and classifying every failure the brake sees: src/tool_error_class.ts is hook-eager because the brake classifies before deciding whether to speak, and it reaches splitShellSegments and stripLeadingAssignments in bash_extractors.ts to credit a test runner or a grep only at command position. The report behind `session-audit --tool-errors` loads the same classifier through a dynamic import so the core startup set does not carry the Bash parser, and that split is what added chunks here: 25 before, 31 after, the extra boundaries costing about 5 KB of the total. Measured by swapping the HEAD versions of the changed sources into this checkout and building: HEAD 2,431,314 bytes, the batch 2,446,552 (+15,238), overrunning the previous 2,432,000-byte line by 14,552. The static-import alternative measured 2,441,460 here but put 66,290 bytes on the core's eager set instead of 8,236. This takes 15 KB and leaves 808. A further 10 KB is src/call_streak.ts, the batch hint and zero-hit search brake relay.ts applies to every pre/post tool answer, with no new chunk: 31 before and after. Measured by swapping the HEAD versions of the changed sources into this checkout and building: HEAD 2,446,988 bytes, the change 2,456,959 (+9,971), overrunning the previous 2,447,360-byte line by 9,599. This takes 10 KB and leaves 641. A further 11 KB is src/hint_target.ts on the hook path, the real-name resolver every read and bash hook now calls and the repeated-deny shortening relay.ts applies to every answer, with no new chunk (31 before and after): HEAD e5ca3a21 2,457,016 bytes, the change 2,468,245 (+11,229), overrunning the previous 2,457,600-byte line by 10,645, so this takes 11 KB and leaves 619. A further 1 KB is the Claude Code structured Grep/Glob count the dedup note now reads, the transcript-head check on a background task's output, and the content-delivering deny moved to src/delivering_deny.ts, with no new chunk (31 before and after): HEAD ec98d8c8 2,468,245 bytes, the change 2,469,202 (+957), overrunning the previous 2,468,864-byte line by 338, so this takes 1 KB and leaves 686. A further 2 KB is the bounded wait between in-process relay calls, relay.ts re-seeding the session id per event, the subshell-group peel every Bash read gate now matches through, and resolving a relative task `.output` path against the event cwd, with no new chunk (31 before and after): HEAD 43b59523 2,469,776 bytes, the change 2,471,433 (+1,657), overrunning the previous 2,469,888-byte line by 1,545, so this takes 2 KB and leaves 503. A further 5 KB is harness detection discounting the session id relay seeded (registry.ts) and the read-only index fallback on db.ts's getDb (a read-only or immutable open, the write-access error classifier, the schema check), with no new chunk (31 before and after): from the same HEAD, the earlier changes plus the harness fix 2,471,821 and the read-only fallback on top 2,475,649 (+3,828), the first figure from a worktree build less its constant 2,720-byte path offset, which puts HEAD at the 2,469,776 measured above, overrunning the previous 2,471,936-byte line by 3,713, so this takes 5 KB and leaves 1,407. A further 5 KB is worker.ts refusing to spawn against a data directory that refuses writes and sweeping the embed backlog a stopped worker dropped, the damaged-database fail-fast in db.ts, the bounded Read of a tool-results file in hooks_read.ts, the whole-file line-range refusal in hooks_bash.ts, and the outline pointer for the headings a large skill's list leaves out, with no new chunk (31 before and after): HEAD 561feedb 2,475,674 bytes, measured by swapping the HEAD versions of the 14 changed sources into this checkout and building, the change 2,481,175 (+5,501), overrunning the previous 2,477,056-byte line by 4,119, so this takes 5 KB and leaves 1,001. The read commands pricing shell-shaped results through delivery_cap.ts then made that module shared, so it moved into its own 1,338-byte chunk (32 after) and the hook set grew 375 bytes to 2,481,592, inside this line with 584 to spare, so it takes nothing. The Copilot CLI hook shim moving to a .cjs file with a .js forwarder, and doctor inspecting project-scope Copilot hooks, grew the set 609 bytes to 2,482,201 across the same 32 chunks, overrunning the 2,482,176-byte line by 25, so this takes 1 KB and leaves 999. The resident hook server grew it 3,464 bytes to 2,486,650 across 35 chunks: 1,687 of that is code the hook path needs to take part (the per-request reset registrations, the shim fragment that tries a server, Copilot CLI's shim interpolating it), and the rest is the import and export lines of three new chunks, which the thin client entry needs so that it shares modules with the hook library instead of copying them. The shim fragment was first imported from shim_common.ts, which pulled every other shim template into this set for 5,655 bytes; it lives in its own module now. That overran the 2,483,200-byte line by 3,450, so this takes 4 KB and leaves 646. The search-pattern guard accepting a pattern with nothing that repeats or branches without timing it and re-timing a slow one before refusing (regex_guard.ts), note commands refusing to save over a line they cannot read or without the notes lock (project_memory.ts), the session-state merge retrying a briefly locked file (session_store.ts), the dirty-queue append writing again an edit that landed in a queue the worker had just claimed (hooks_index.ts), and the worker skipping absent files when requeueing embeddings grew it 2,923 bytes, with no new chunk (36 before and after): HEAD 008bcb2c 2,485,715 bytes, measured by building HEAD and the change side by side from a scratch copy less its constant 255-byte path offset, the change 2,488,638, overrunning the previous 2,487,296-byte line by 1,342, so this takes 2 KB and leaves 706. A further 2 KB was bought by doctor process health recognizing resident hook-servers and copilot_mcp_tools filtering disabled MCP servers from waste calculations. Measured 2,491,073 bytes across 35 chunks, overrunning the 2,490,368-byte line by 705 bytes, so this takes 2 KB and leaves 1,343. Wiring the native hook client into every installer grew it 11,082 bytes with no new chunk (35 before and after), measured by building HEAD 24799e86 and the change in the same scratch directory: 2,492,439 against 2,503,521 there, 2,500,291 at the repository's own path. 7,645 of it is src/native_hook.ts itself, which the hook path does not call but reaches because it already loads bridges/copilot_cli_install.ts for readCopilotHooksOwners, copilotCliMcpToolsDir and copilotCliUserRoot, and that installer now builds the native command lines; the rest is Copilot's own native command builder, the fallback-reason detail relay.ts records per call, the POSIX quoting helper, and the config key with its environment reading. That overran the 2,489,344-byte line by 10,947, so this takes 12 KB and leaves 1,341. Keeping every line of a rewritten Read on its real line number (the `aligned` fold layout for the Claude Code Read envelope, which the harness numbers by position, with the markdown outline laid over the document rather than printed beside it) grew it 7,021 bytes with no new chunk (35 before and after): 2,503,675 on main 633c6bfc against 2,510,696 with the change, both built in sibling worktrees one directory apart. fold_structure.ts is 2,953 of it, hooks_read_post.ts 1,904, hooks_read_slice.ts 593, fold_delivery.ts 545, and nothing else over 300. It has to be eager because it runs inside the PostToolUse Read hook itself. That overran the 2,504,704-byte line by 5,992, so this takes 7 KB and leaves 1,176. Reading each drained file's own project `.token-goat.toml` in the worker grew it 1,506 bytes with no new chunk (35 before and after): 2,511,025 with the HEAD versions of config_project.ts and worker.ts swapped into this checkout against 2,512,531 with the change, both built at the repository's own path. worker.ts is 736 of it (configRootOf and the per-root reads in makeIndexer, embedFileSerialized and requeueStaleEmbeddings), config_project.ts 672 (withConfigProjectRoot, configProjectRootFor and the override resolveConfigProjectRoot consults), and the chunk import and export lists carrying those two names the other 98. The hook path never drains, but esbuild splits by file, and the hook loads worker.ts for ensureWorkerAlive and the dirty-queue helpers and config_project.ts for every loadConfig, so both arrive whole, as native_hook.ts does above. Splitting the four cloud provider filters out of cloud.ts into cloud_providers.ts added 101 bytes of module boundary to the same set. 96c1ff19 measured 2,510,924 and the two changes together 2,512,531, overrunning the 2,511,872-byte line by 659, so this takes 1 KB and leaves 365. Dating project notes and ordering them by recency (project_memory.ts, which the hook loads for the session-start notes block) grew the set from 2,512,531 bytes at 4691ad28 to 2,514,354, 1,458 over the line, so this takes 2 KB and leaves 590. Moving the worker's process control (pid file, start, stop, heartbeat, error log) into worker_lifecycle.ts and the dirty-queue helpers into dirty_queue.ts took worker.ts off the hook path, which only ever needed those, and with it the four modules only the drain uses: embed_backfill.ts, index_prune.ts, index_reclaim.ts and process_priority.ts. The set went from 2,514,354 bytes at 1321984c to 2,478,170, 35 chunks before and after: 36,184 smaller, so this gives back 35 KB and leaves 934. Reporting what reached the dirty queue rather than what the sweep offered it (reconcile.ts, which the hook loads for the session-start sweep, and the session-start note that says so) and resolving a relative edit path before queueing it (hooks_edit.ts) grew the set from 2,478,443 bytes at b5d140d8 to 2,479,411, 307 over the line; moving the install commands out of cli.ts took 55 off it. So this takes 1 KB and leaves 717. Keying a cached Bash result on the directory its `cd` prefix lands in (commandRunDir and bashRecallKey in hooks_bash_commands.ts, which both Bash hooks now call where each used to build the key inline), and holding the directory each Bash call starts in from its pre hook to its post hook (recordBashStartCwd and takeBashStartCwd in session.ts, mergeBashStartCwds in session_store.ts), and letting a string shaped like an untrusted-content fence through unescaped only when its notice is one token-goat writes, with its body escaped again (neutralizeOutsideFences in injection_scan.ts), grew the set from 2,479,988 bytes at dbb03d4c to 2,483,120, 35 chunks before and after: session.ts is +1,122 of it, injection_scan.ts +638, session_store.ts +611, hooks_bash_commands.ts +312, hooks_bash.ts +133, hooks_bash_post.ts +84, and the import and export lists between chunks, which no module's own bytes include, the other 232. That overran the 2,480,128-byte line by 2,992, so this takes 3 KB and leaves 80. Looking a package-manager script up in the directory a `cd` prefix runs the command in (hooks_bash.ts, hooks_bash_post.ts), and resolving a reported Bash call to the command and directory its output is cached under in one function the post hook and the waste report both call (resolveBashCall in hooks_bash_commands.ts), cost 185 bytes: 2,483,120 over 35 chunks at 7d570d04, 2,483,305 with the waste report still on its own lookup. Its call made waste.ts import hooks_bash_commands.ts, and since the post hook imported projectTranscriptsDir from waste.ts, that edge redrew the chunk boundaries around the whole analyser: 2,486,077 over 36 chunks. Moving projectTranscriptsDir into claude_config_dir.ts, whose own header says a path accessor must not drag the waste analyser onto hook-eager paths, took waste.ts and its 9,224 bytes off the hook path, and the set went to 2,476,728 over 36 chunks, 3,034 of it the import and export lists the new edge still draws between chunks. That is 6,392 smaller than 7d570d04. Writing the Copilot CLI adapter's shrunk image copies with the materializer VS Code's image path uses (materializeShrunkImageFile in bridges/vscode_hooks.ts) rather than a second typed copy gave that function the adapter's hourly sweep throttle, and VS Code's image path loads that function eagerly while the copy it replaced was never on the hook path, which adds the throttle's 143 bytes: 2,476,871 over the same 36 chunks. That is 6,249 smaller than 7d570d04, so this gives back 6 KB and leaves 185. Moving the rendering of the `token-goat stats` report out of stats.ts into stats_report.ts took render/stats_renderer.ts off the hook path, which imports stats.ts to record a stat and never renders a report: the set went from 2,476,871 bytes at ab921ee7 to 2,444,539, 36 chunks before and after: 32,332 smaller. Recording a Bash read only of a file that is there after a command that succeeded, trying a drive-letter index key again at its WSL mount before taking the file for absent, keeping the `cd` in the large-output compress hint when its directory is there, named at its WSL mount when that is where it is, and storing the compound and elided outputs under the directory the command ran in (hooks_bash.ts, hooks_bash_post.ts, and the file and directory checks in paths.ts) add 1,043 bytes: 2,445,582 over the same 36 chunks. That is 31,289 smaller than ab921ee7, so this gives back 30 KB and leaves 754. Opening a drive-letter index key at its WSL mount everywhere a hook opens a file, not only in the existence checks (hostPathOfIndexKey and wslMountOfIndexKey in paths.ts and their callers across the read, Bash and edit hooks), asking the pre-tool path gate about that mount as a second path a hook can open for one typed path (vscode_path_gate.ts), putting the structural-index rewrite's stat behind the same gate, and anchoring the Get-Content parameter patterns to the start of a token grew the set 1,899 bytes with no new chunk (36 before and after): vscode_path_gate.ts is +496 of it, paths.ts +381, hooks_read_slice.ts +147, hooks_bash.ts +136, hooks_read_post.ts +90, bash_structural_index.ts +89, eleven other modules +386 between them, and the import and export lists between chunks the other 174. The three changes landed before it had taken the set from 2,445,582 bytes at caf442b9 to 2,445,991, inside the line: moving the database-client filters into tool_filters/db_clients.ts added 95 bytes of module boundary, the one MCP config read-back the four editor bridges now share (managedServerEntry in bridges/mcp_servers_json.ts) 64, and reading an unquoted leading `~` in a `cd` prefix as the home directory (hooks_bash_commands.ts) 250. The set went to 2,447,890, overrunning the 2,446,336-byte line by 1,554, so this takes 2 KB and leaves 494. Running embeddings on the bundled WebAssembly build of ONNX Runtime when onnxruntime-node is absent first put all of its loader on the hook path with the rest of embeddings.ts's imports, taking the set from 2,447,890 bytes to 2,460,701 over the same 36 chunks. Hooks never embed, so the placing, downloading, tar reading and starting moved behind dynamic imports, into embed_runtime_web.ts and pinned_fetch.ts: 2,453,894 over 37 chunks with that alone, because the lazy module's static imports split embed_runtime.ts and pinned_file.ts into a chunk of their own; 2,450,655 once pinned_file.ts's hashing, copying and downloading followed; 2,449,917 over 36 once the lazy module took what it needs as an argument instead of importing it, which folded the two back into embed_model.ts's chunk. That is 10,784 fewer than the first version and 2,027 more than before it, and the remainder cannot move without moving EMBED_FINGERPRINT: embed_runtime.ts (3,942 bytes: which runtime runs, the memory of a failed start, where the binary lives and whether it is there) and pinned_file.ts (1,079: synchronous publishing and three one-line wrappers) are what embed_model.ts and embeddings.ts, both hashed, import and read synchronously. The first version took 13 KB here; this takes 2 KB, overrunning the old 2,448,384-byte line by 1,533 and leaving 515. A further 2 KB was bought by keeping a failure of that runtime to start on disk, so that `semantic --preflight` and `doctor` in a later process report it instead of calling the runtime ready: they ask isRuntimeAvailable and runtimeLoadError, which hashed embed_model.ts and embeddings.ts read synchronously, so reading the record and building the key it is held to (startFailureKey, recordedStartFailure in embed_runtime.ts) has to be static, while writing and clearing it went into the lazy embed_runtime_web.ts. Measured 2,451,440 bytes over 36 chunks with it, against a 2,450,432-byte line it missed by 1,008, leaving 1,040. Mapping VS Code 1.137's multi_replace_string_in_file, apply_patch, fetch_webpage, runSubagent and get_terminal_output onto the hook path (the two map entries each in hooks_cli.ts, the patch-header reader and derived-key table beside them, getFilePaths in hooks_common.ts, the per-file loop in hooks_edit.ts and the per-URL policy loop in hooks_fetch.ts) added about 2 KB with no new chunk. Wiring the subagent-start hook for Copilot CLI and VS Code (subagentStartHandler and subagentStartWired in hooks_agent_spawn.ts, the subagent_start event name, the agent-id exception in relay.ts's buildEvent, and displaySafeText on the folder name and recent files in baseline.ts's formatProjectMap) took the set from 2,462,891 to 2,466,239 with no new chunk (36 before and after), overrunning the 2,463,744-byte line by 2,495, so this takes 3 KB and leaves 577. Rewriting a long subagent report through Copilot CLI's subagentStop modifiedResponse (subagentReportRewriteHandler, reportNotice, planReportCompaction and recordReportCompaction in hooks_agent_spawn.ts, and the response forward and modifiedResponse reply in the Copilot shim) took the set from 2,466,239 to 2,469,217 with no new chunk (36 before and after), overrunning the 2,466,816-byte line by 2,401, so this takes 3 KB and leaves 671. Reading a Copilot CLI skill's body from the SKILL.md Copilot loaded, since its skill tool result is only a confirmation line (copilotSkillPath in copilot_skill_path.ts, copilotCliUserRoot moved into copilot_home.ts, and the harness gate in hooks_skill.ts), took the set from 2,469,611 to 2,471,159 with no new chunk (36 before and after), overrunning the 2,469,888-byte line by 1,271, so this takes 2 KB and leaves 777. It has to be eager: the skill hooks are where the body is cached and the reload is denied. Loading copilot_skill_path.ts through a dynamic import instead measured 2,471,624 over 39 chunks, larger, because the split added three shared chunks. The per-change sum above had drifted to 2,462,720 bytes, 80 KB clear of the build it guarded, which is room for a whole regression to land unseen. The ceiling is therefore re-based on a measurement rather than summed: HAND-DERIVED, 2,382,498 bytes over 36 chunks, measured by this file's own static closure on the feat/copilot-vscode-integration build rebased onto 4770a39c, plus a 4 KB margin, the size of a typical single increment recorded above. Phase 5 re-measured it on its own branch rebased onto that build: the refused re-read identity row (rereadIdentity in hooks_read_slice.ts, booked from hooks_read.ts), the edit-after-partial-read count (read_shape.ts), and the list of files a compaction summary left out (compact_dropped.ts, written from hooks_compact.ts) took the set to 2,388,942 bytes over the same 36 chunks, 6,444 above the measured figure and 2,348 over the line. All three run inside a hook, so they cannot sit behind a dynamic import; the measured figure is replaced by that one. A change that needs more re-measures and replaces the measured figure here, naming what bought the bytes. The one-time savings receipt (claimFirstSavingsReceipt and firstReceiptShownAt in stats.ts, withSavingsReceipt and savingsReceiptText in hooks_session_start.ts, noticeField in hook_registry.ts) measured 2,395,393 bytes over 36 chunks, 2,355 over the line: it runs inside the session_start hook and rides stats.ts, which the hook path already loads, so it is eager like the rest of that module. Re-based on that measurement plus the same 4 KB margin. The Antigravity CLI bridge (antigravityToCanonicalWire and its tool-name and argument-key maps in hooks_cli.ts, serializeAntigravityOutput in bridges/antigravity_hooks.ts, dispatched from hook_registry.ts) measured 2,403,142 bytes over 40 chunks, 3,653 over the line: every agy hook call is translated in on the way in and shaped on the way out, so both halves run inside the hook and are eager like the Gemini and Qwen serializers beside them. Re-based on that measurement plus the same 4 KB margin. smol-toml 1.8.0 -> 1.9.0, the release that fixes the denial of service in GHSA-r4xh-jqrq-34v2, took the set from 2,406,552 to 2,413,530 bytes over the same 40 chunks, 6,292 over the line, measured on one tree with only that package swapped. It is inlined into the chunk that holds src/config.ts, which every hook call loads to read config.toml, so it is eager for the same reason config is. Re-based on that measurement plus the same 4 KB margin. Reading Gemini, Qwen and Grok read_file results as raw text (rawTextReadHarness, splitRawReadHeader, rawReadCutShort and wholeNumberedReadResult in hooks_read_slice.ts) took the set from 2,414,265 to 2,418,127 bytes over the same 40 chunks, 501 over the line, measured on one tree with only that file swapped: the post_tool_use Read hook is what decides whether a delivery was cut short, so the parser runs inside the hook and is eager like the Claude Code parser beside it. Re-based on that measurement plus the same 4 KB margin. Anchored project notes took the set from 2,421,706 bytes over 40 chunks to 2,424,764 over 43, 2,541 over the line, measured on one tree with only this change swapped. project_memory.ts grew 1,650 bytes: session_start loads it to inject the notes, and it has to read and rewrite the `# anchor:` line or a save would drop it. The rest is the three chunk boundaries esbuild cut for the dynamic import that keeps note_anchor.ts, notes.ts and the index reads they make off this set; loading them eagerly instead measured 2,427,842. Re-based on that measurement plus the same 4 KB margin. The next round grew the set 6,893 bytes on one worktree measured before and after, all of it code the hooks run: the compaction reset in session_store.ts and hooks_compact.ts, the launcher spellings and line-range checks in hooks_bash_commands.ts and hooks_bash_post.ts, the ranged-Read overlap check in hooks_read.ts, the expired-id filter in manifest.ts and web_cache.ts, the clip floor in tool_filters/helpers.ts, and the sparse save in config.ts. Re-based on the main checkout's measurement of 2,442,785 plus the same 4 KB margin. Later rounds spent that margin, and the 2 KB it was widened by, until the set stood at 2,448,844, 85 bytes under the line. The next 93 bytes are withTelemetryWriteDb in stats.ts: the short-budget connection recordStat already opened, pulled out so the semantic-query ledger writes through it too instead of waiting out the shared connection's 15s busy_timeout. It runs inside recordStat on every hook, so it is eager for the same reason that connection was. Re-based on the main checkout's measurement of 2,448,937 plus the same 4 KB margin. Later redaction and parser fixes spent that margin, and the line was crossed by the PowerShell wrapper in bash_runner.ts, which now resets LASTEXITCODE and captures the script's own `$?` so a wrapped run exits as a direct `pwsh -Command` would; it is the hook-side runner, so it is eager. Re-based on the main checkout's measurement of 2,453,145 plus the same 4 KB margin. The single-pass banner, hint and skill-name scanners in line_matchers.ts spent that margin to 2,457,124, and the line was crossed by 517 bytes in secret_redact.ts: url_credentials now accepts an empty or `@`-bearing username, and hasPreciseSecret skips a custom pattern's empty match, which matters here because the Read-hook fold gates call it. Re-based on the main checkout's measurement of 2,457,641 plus the same 4 KB margin. The PowerShell wrapper in bash_runner.ts, which the Bash hook loads eagerly, then grew twice to match direct pwsh exit codes: once for statement errors and `return`, measured 2,462,612 in a clean checkout of 822d121b, and once for named blocks, quoted, dotted and computed break labels and Windows PowerShell 5.1, measured 2,464,017. That is measured with nothing uncommitted in the tree, because an unrelated uncommitted edit hid the first crossing by shrinking the set. Re-based on 2,464,017 plus the same 4 KB margin. A further 5 KB was bought by the indexer fixes for C++ destructors and operators, CommonJS exports, C tag definitions and whole YAML/TOML block spans in parser_treesitter.ts and parser_structured.ts: the read hook's source-skeleton fold calls the tree-sitter extractors synchronously through parseSourceSymbolsTreeSitterOnly, so they cannot sit behind a dynamic import the way the regex adapters do. Measured in a clean worktree: 2,470,165 at HEAD and 2,475,197 with the fixes. Re-based on 2,475,197 plus the same 4 KB margin. A clean Windows worktree at ff780313 already measured 2,480,769 after the round of graph and index fixes that followed, while the Linux CI build stayed under the line, and a further 4.3 KB was bought by Ruby calls without parentheses recorded as references in parser_refs.ts, go test -v log lines kept with their own test in tool_filters/go_test.ts, setext heading levels in hint_target.ts and the record of install-created configs in bridges/created_configs.ts, all four reached synchronously from the edit and read hooks. Measured 2,485,071 with nothing else uncommitted. Re-based on 2,485,071 plus the same 4 KB margin. A further 7.1 KB was bought by four fixes that all run on a hook path: the cargo failing-build collapse in tool_filters/build.ts runs inside the Bash hook's output compression, the AVIF/HEIC probe in image_shrink.ts adds only its ftyp check and a dynamic-import stub because image_isobmff.ts itself loads lazily, the hint fixes in hint_target.ts and hooks_read.ts run in the synchronous Read and Grep hooks, and the owner-checked symlink resolution in util.ts sits under every atomic write. Measured 2,492,170 over 50 chunks with nothing else uncommitted. Re-based on 2,492,170 plus the same 4 KB margin. A further 6.4 KB came from the interpolation-hole scanner in languages/common.ts (4.2 KB), which the hook loads eagerly because a post-edit hook can reindex a file inline and the brace walk is synchronous, so it cannot sit behind a dynamic import; the rest is the heading-body re-slice in read_commands.ts, the CRLF-preserving block edit in util_config.ts and the CLAUDE_CONFIG_DIR lookup in claude_config_dir.ts. Measured 2,498,557 over 50 chunks with nothing else uncommitted. Re-based on 2,498,557 plus the same 4 KB margin. A further 1.4 KB came from whole-document JSON handling for aws, az and npm audit in tool_filters/cloud_providers.ts, package_managers.ts and helpers.ts, which run inside the Bash hook's output compression, and the byte-order-mark keep in util_config.ts, which sits under writers that module already carries onto the hook path. Measured 2,504,004 over 50 chunks with nothing else uncommitted (a worktree whose node_modules is a junction reads about 2.2 KB higher, because esbuild writes the junction's real path into every module marker comment). Re-based on 2,504,004 plus the same 4 KB margin. The PowerShell heredoc adapter powershell_compat.ts, which bash_runner.ts imports on the compress path, took it to 2,510,432 on a branch the next five measurements were taken without. Dropping a file's recorded line ranges once it changed on disk before a `sed -n` or `head` read is measured against them (dropFileLineRangesIfChanged and the range-identity map in session.ts, its merge in session_store.ts, the calls in hooks_bash.ts and bash_extractors.ts) runs inside the Bash pre hook, so it is eager; with the 58-byte snippet_window.ts that recall_index.ts brings onto this set it took the set from 2,506,477 at 504aa3b8 to 2,508,739 over the same 51 chunks, measured with nothing else uncommitted, 639 over the line. Re-based on 2,508,739 plus the same 4 KB margin. Telling a project indexed on purpose from one where the edit hook indexed a single file (indexed_roots.ts) is read by the session-start hook, and its table's v16 -> v17 backfill sits in db.ts, which every hook opens; the predicate is loaded behind a dynamic import, which costs one more shared chunk. It took the set from 2,512,086 over 52 chunks at f8ba8c4d to 2,513,395 over 53, measured with nothing else uncommitted, 560 over the line. Re-based on 2,513,395 plus the same 4 KB margin. rewrite_permission.ts then added about 17 KB: every rewrite the pre_tool_use Bash, Agent and Read hooks make must first be checked against the user's Claude Code permission rules, and the Bash and Agent handlers are synchronous, so the check cannot sit behind a dynamic import; it took the set to 2,532,536 over 53 chunks, measured with nothing else uncommitted. With the PowerShell adapter merged in, the set measured 2,536,695 over the same 53 chunks, less than the two increments summed because they share chunks. The first-read symbol policy (hooks_read_policy.ts and its index_reader.ts and db.ts lookups) runs inside the Read pre hook, so it is eager too; it added 9,063 bytes on its own branch (2,510,432 to 2,519,495). Merged with everything above, and with the permission checks that follow links and 8.3 short names, treat settings.local.json as untracked only when git says so, read settings from the session's project and read Codex rules files, the set measured 2,548,632 over 52 chunks with nothing else uncommitted. Holding back read hints on paths a Read deny or ask rule covers took that to 2,552,512. The policy's later revision (a whole read after slices, the heading-tree intercept, the stale-evidence fail-open) added 2,568 bytes on its own branch (2,521,517 to 2,524,085); merged with everything above the set measured 2,552,844 over 52 chunks with nothing else uncommitted. Re-based on that plus the same 4 KB margin. At fb08050a it measured 2,554,811 over 53 chunks. Counting only the reads and edits since the last compaction (epochReadCounts, mergeEpochBase and wasEditedSinceCompaction in session.ts, their persistence in session_store.ts) is read by the Read pre hook's re-read denies and by the pre-compact manifest, so it is eager; it took the set to 2,557,331 over the same 53 chunks, measured with nothing else uncommitted, 391 over the line. Re-based on 2,557,331 plus the same 4 KB margin. The zero epoch base an entry created after a compaction carries (session.ts newEntryEpochBase) and the one-read cap epochReadCounts applies to a hook process that missed the compaction are on that same eager Read pre hook path; they took the set to 2,561,526 over the same 53 chunks, measured with nothing else uncommitted, 99 over the line. Re-based on 2,561,526 plus the same 4 KB margin. Quoting every argument of a suggested command with one mark (hint_suggestion_guard.ts quotedArgs, used by the Read pre hook's edit-anyway and truncated-read denies) is on that same path; with the first-read policy fixes before it, it took the set to 2,565,785 over the same 53 chunks, measured with nothing else uncommitted, 163 over the line. Re-based on 2,565,785 plus the same 4 KB margin. The PowerShell lexer that keeps the heredoc and python -c rewrites to top-level code (powershell_compat.ts layoutPowerShell) is reached through bash_runner.ts, which hooks_bash_post.ts imports statically for MAX_CAPTURE_BYTES; it took the set to 2,570,838 over the same 53 chunks, measured with nothing else uncommitted, 957 over the line. Re-based on 2,570,838 plus the same 4 KB margin. Cutting an unsafe suggested command at its own span (hint_suggestion_guard.ts fencedCommandEnd and the backslash-aware scanQuotes) and naming the file in read-hook prose only where no command carries it (hooks_read.ts fileSubject) are on the same Read pre hook path; they took the set to 2,574,986 over the same 53 chunks, measured with nothing else uncommitted, 52 over the line. Re-based on 2,574,986 plus the same 4 KB margin. Naming a skill, a symbol, a tool or a downloaded file in hook prose through nameSubject and fileSubject (hint_suggestion_guard.ts) is on the same hook path; it took the set to 2,579,113 over the same 53 chunks, measured with nothing else uncommitted, 31 over the line. Re-based on 2,579,113 plus the same 4 KB margin. Reading a heredoc body the way bash does, the `-c` loader that imports only sys, and naming a `cat >` file as a PowerShell string literal (powershell_compat.ts, reached through bash_runner.ts as above) took the set to 2,583,416 over 54 chunks, measured with nothing else uncommitted, 207 over the line. Re-based on 2,583,416 plus the same 4 KB margin. */ -const MAX_HOOK_EAGER_BYTES = 2_583_416 + 4 * 1024 +/** Ceiling on what the hook entry pulls in statically, mirroring the core's ceiling in core_bundle_stays_split.test.ts. A bridge `import()`s this bundle on nearly every tool call and V8 parses every byte of the eager set before running any of it. Measured at 1.837 MB across 7 chunks both before and after the two builds were merged; the headroom is a regression trip-wire, not a budget to re-tune on every dependency change. It went 2.4 MB -> 2.6 MB when the Fortran, Pascal, MATLAB and CMake adapters (41 KB, eager like every other adapter) grew the set, and back to 2.25 MB once the adapters moved behind parser.ts's dynamic import of languages/registry.ts. Measured on one machine across that move: 2.430 MB over 7 chunks before, 2.167 MB over 8 after, and 47-53 ms to import the hook bundle before against 42-43 ms after. The 4 KB above the round 2.25 MB was bought by EXIF orientation correction in image_engine.ts, which has to be eager: the pre-read hook is what shrinks images, so the rotation runs on the hook path or not at all. Measured 2,360,603 bytes over 18 chunks with it, against a 2,359,296-byte line it missed by 1,307. A further 2 KB was bought by the structural-index rewrite in bash_structural_index.ts (batch Q): it decides whether to rewrite an `rg`/`grep` structural search into a `token-goat outline`/`imports` call, and that decision has to run inside the synchronous pre_tool_use Bash hook before the tool executes, so it cannot be deferred behind a dynamic import the way a language adapter can. Every module it imports (paths.js, parser_types.js, language_specs.js, tool_filters/helpers.js, bash_extractors.js, fingerprint.js, index_reader.js) was already eager via an existing hook path; the added bytes are its own ~130 lines of recognizer logic. Measured 2,365,172 bytes over 19 chunks with it, against a 2,363,392-byte line it missed by 1,780. A further 3 KB was bought by Batch S's hook wall-clock timing: relayInProcess (relay.ts) now imports recordStat from stats.ts to measure and persist its own duration on every hook invocation, which is unavoidably eager for the same reason as the structural-index rewrite above -- the timer has to close before relayInProcess returns, on every hook, not behind a dynamic import. stats.ts itself was already eager via other hook paths (session_hint, image_shrink, ...); the added bytes are recordStat's new duration_ms plumbing and pruneHookStats. The CLI/doctor-only read side (hookLatencyBreakdown, renderHookLatencyStats) lives in the separate hook_latency.ts on purpose, so it never joins this eager set at all. Measured 2,367,105 bytes over 19 chunks with it, against a 2,365,440-byte line it missed by 1,665. Raise this only for something equally unavoidable, and say here what bought it. A further 1 KB was bought by the fix that stops recordStat holding the hook path for db.ts's full 15s busy_timeout under contention: recordStat now opens its own short-budget connection instead of reusing the shared, patient one getDb() caches for indexing and the worker, and closes it before returning. That connection has to be opened and torn down synchronously inside recordStat, on every hook, same reason as the timing plumbing above -- it cannot be deferred behind a dynamic import. Measured 2,368,665 bytes over 19 chunks with it, against a 2,368,349-byte line it missed by 316. A further 1 KB was bought by `hint_stats.defiance_threshold_pct` (batch b02 J4): the same five config rows as the entry below, plus the `isSuppressionCategory` branch in shouldSuppress, which unlike checkHookLatency really is eager -- hint_stats.ts runs inside the pre/post tool-use hooks. Measured 2,370,518 bytes over 19 chunks with it, against the 2,370,560-byte line the entry below left, which it cleared by 42. Two config keys in one batch consumed 1,115 of this ceiling's bytes between them, which is the number to plan against. A further 1 KB was bought by `hooks.latency_budget_ms` (batch b02 J3), and this entry is worth reading before adding any config key at all, because the cost is not obvious from the diff. config.ts is eager on the hook path -- every hook calls loadConfig -- so a key's five rows (its NUMERIC_FIELD_BOUNDS entry, its two _buildConfig lines, its CONFIG_KEY_ENV_OVERRIDES entry and its saveConfig serialize arm) all land in this set even when the only code that reads the key is CLI-only, as checkHookLatency is. Measured on one machine by building both trees: 2,369,403 bytes over 19 chunks at the parent commit against 2,369,870 with the key, so one key costs 467 bytes here and the parent commit had 133 bytes of headroom left. Unavoidable in the sense that matters: the rows are what make the key settable, and there is no deferred-import shape for a table the eager loader indexes. Budget roughly 0.5 KB of this ceiling per future config key rather than discovering it as a red guard. A further 4 KB was bought by making the parser fingerprint per-language: src/parser_fingerprint.ts now carries a 49-entry language-to-digest map beside the shared digest, and parser_stamp.ts's parserFingerprintForLanguage resolves a row's stamp through it. Eager because the three freshness gates that consult it (worker.ts's drain, reconcile.ts's sweep, fold_delivery.ts's body-fold gate) are all on the hook path, and a digest map is data with no deferred-import shape. Measured by building both trees: 2,372,896 bytes over 19 chunks at the parent commit against 2,374,896 with the map, so the map costs 2,000 bytes here and the parent commit had 1,760 bytes of headroom left. 4 KB rather than 1 leaves 3,856 bytes of headroom, deliberately more than the 1,766 the entry below left, since a language adapter added later grows this map by another ~30 bytes and should not be the change that discovers a red guard. A further 3 KB was bought by merging origin/main's doctor work (findTopIndexedProjects, checkUnmappedTools' age window, checkVscodeProjectMcp and cleanupDeprecatedVscodeProjectMcp) into the local oversized-db category breakdown. cli_doctor.ts is not itself hook-eager, but the merge also pulled origin's changes to stats.ts and util.ts, both of which are. Measured 2,372,890 bytes over 19 chunks after the merge, against the 2,371,584-byte line it missed by 1,306; 3 KB rather than 2 leaves 1,766 bytes of headroom, since the previous entry's 42-byte margin meant the next unrelated change would have discovered this red rather than planned for it. A further 1 KB was bought by resolving Claude Code's transcript root through `claudeConfigDir()` in waste.ts, which honours `CLAUDE_CONFIG_DIR` instead of hardcoding `os.homedir()`. Eager because hooks_bash_post.ts imports projectTranscriptsDir to find the per-session tool-results directory on the Bash hook path, and a path accessor read on every such call has no deferred-import shape. Measured by building three trees on one machine: 2,378,447 bytes over 19 chunks at the parent commit, 2,378,618 with the sessions-directory fix that precedes this one, and 2,378,804 with this one, so the accessor costs 186 bytes here and the parent commit had 305 bytes of headroom left. 1 KB leaves 972 bytes of headroom. A further 6 KB was bought by the two embedding gates. src/asset_extensions.ts is hook-eager because image_shrink.ts, which runs on the pre-read hook path, now takes its image-extension list from there instead of holding a private copy -- parser.ts cannot import image_shrink.ts (it pulls in the hook registry, the harness bridges and the stats ledger), so the one list moved down to a leaf both can reach. src/embed_backfill.ts arrives through worker.ts. Three trees measured on one machine, rebuilt between each: 2,378,804 bytes over 19 chunks at the parent commit (the figure the paragraph above records); 2,382,618 over 19 with the gates and the shared extension module (+3,814); 2,384,032 over 19 with the backfill sweep wired in as well (+1,414 more, +5,228 over the parent), which missed the 2,379,776-byte line by 4,256. Rebuilt four times at that final state, byte-identical each time. The chunk count never moved, so none of this is a new chunk being dragged eager. 6 KB leaves 1,888 bytes of headroom. A further 16 KB was bought by the hint-efficacy measurement batch. hint_stats.ts, hooks_bash.ts, hooks_common.ts and db.ts are all hook-eager, and the batch touches every one of them: hint builders now hand their own file path to the efficacy ledger instead of it being regex-scraped back out of the rendered hint (a `correlators` field on the context HookOutput, a `pathHint` wrapper and 20 converted call sites in hooks_bash.ts); a range hint must now price its own proposal against the window the command asked for before it may emit, which pulls bash_range_savings.ts and line_regions.ts onto the hook path; and two `hint_emissions` columns, with their migrations, separate an emission that was shown but unscoreable from a detection that was never shown at all. Measured by building both trees on one machine, ceiling temporarily forced to 1 to read the figure at the parent: 2,385,404 bytes over 19 chunks at c6968099 -- which had 516 bytes of headroom, so the first step of the batch alone was going to find this red -- against 2,397,441 with all of it, +12,037 over the batch. The last 1,543 of that is the batch's final step, which gives the whole-file deny a real indexed name to print instead of a `SectionHeading`/`KEY_NAME` placeholder that does not run: bash_surgical_target.ts is new, but index_reader.ts and fingerprint.ts were already pulled eager by the range gate above, so it costs its own compiled bytes and nothing else. Rebuilt twice at that final state, byte-identical each time. The chunk count never moved off 19, so none of this is a new chunk being dragged eager: it is compiled bytes on modules the hook path already loaded. 12 KB would have left 767 bytes of headroom, well under the slack every paragraph above keeps, so this takes 16 KB and leaves 4,863 bytes. A further 1 KB was bought by the batch that made the hook path read its own argv and its own printed output. hooks_bash.ts and hooks_bash_commands.ts are hook-eager and carry most of it: pipelineShapeFilter now returns the first stage's argv alongside the filter instead of dropping it, because the caller was passing an empty array and every argv-reading behaviour in every filter was therefore dead on a piped command. manifest.ts's manifestPrintedPaths now samples the rendered manifest rather than re-walking the selection, session_store.ts's file cap ranks edited entries above read ones, and pending_context.ts gained an import of hint_suggestion_guard.ts -- a zero-import leaf already reached through relay.ts, which is why the chunk count did not move. Measured after a fresh build: 2,403,355 bytes over 22 chunks, against the 2,403,328-byte line it missed by 27, and against the 2,402,399 bytes over 22 chunks the entry below records at the parent commit, so the batch costs 956 bytes. Rebuilt and re-measured byte-identical. 1 KB leaves 997 bytes of headroom, the same shape as the entries below, so the next small change is not the one that discovers a red guard. A further 1 KB was bought by the deferred-hint delivery fix and the fold it shares. relay.ts now clears the pending-context queue only once the emitted output is confirmed to carry the text, which puts pending_context.ts on the hook entry's eager graph where it previously arrived only through hooks_session.ts; src/tool_name_fold.ts is the leaf that replaced three copies of foldToolName, one of which had lost its type guard; and overflow_guard.ts's trim now classifies the content once and spends the same divisor the entry check measured with. Measured after the batch: 2,402,399 bytes over 22 chunks, against the 2,402,304-byte line it missed by 95. 1 KB rather than the 95 bytes it costs leaves 929 bytes of headroom, in the same shape as the entries above, so the next small change is not the one that discovers a red guard. A further 1 KB was bought by the batch that gave the Copilot CLI shim own-property lookups and the compaction manifest its subagents' web fetches. copilot_cli.ts is hook-eager through the bridge registry and manifest.ts through the pre_compact hook, and most of the cost is the shim's own comment text rather than its code: the shim is a template literal shipped verbatim, so a comment inside it is bytes on every install instead of being stripped the way a source comment is. Trimming that comment to its essentials, with the full rationale left where it already lived beside the event map, took the batch from +1,191 bytes to +913. No new module and no new chunk -- 22 across both measurements. Measured with the working tree restored to HEAD for the parent build rather than stashed, ceiling temporarily forced to 1 to read the figure: 3d1a799e 2,403,312 bytes; the batch 2,404,225, which cleared the previous 2,404,352-byte line by 127. Rebuilt at that final state, byte-identical. 127 bytes is under every margin the paragraphs above keep and would make the next unrelated one-liner the change that discovers a red guard, so this takes 1 KB and leaves 1,151 bytes. A further 2 KB was bought by the same launcher-options and yarn-built-ins batch the startup paragraph in tests/guards/core_bundle_stays_split.test.ts describes. helpers.ts is hook-eager through the bash filter registry, and the cost is the two string-literal lookup sets it gained: LAUNCHER_VALUE_FLAGS and the 47-name YARN_BUILTIN_SUBCOMMANDS. No new module and no new chunk -- 22 across both measurements -- so this is compiled bytes on a module the hook path already loaded. Measured with the working tree restored to HEAD for the parent build rather than stashed, ceiling temporarily forced to 1 to read the figure: 558b1c96 2,404,169 bytes; the batch 2,405,669 (+1,500), which overran the previous 2,405,376-byte line by 293. Rebuilt twice at each state, byte-identical both times. A 1 KB bump would clear it by 731 bytes, under every margin the paragraphs above keep, so this takes 2 KB and leaves 1,755 bytes. A further 2 KB was bought by moving the worker's transient-read-failure counters out of `files` into their own `index_retries` table, and by the sweep that reclaims a counter whose file is gone. db.ts is hook-eager and carries most of it: the table, its expression index, and -- crucially -- the rationale, which first landed as an SQL `--` comment inside the SCHEMA_SQL template literal. A comment there is a string, not a comment: the bundler ships it verbatim on every install instead of stripping it, which is the same trap the Copilot CLI shim paragraph above records. Moving that prose to the docblock above SCHEMA_SQL took the batch from +785 bytes to +172. index_prune.ts and config_commands.ts are also hook-eager and carry the rest. No new module and no new chunk: 25 across both measurements. Measured with the working tree restored to HEAD for the parent build rather than stashed, ceiling temporarily forced to 1 to read the figure: c7558c84 2,415,615 bytes, which had exactly 1 byte of headroom; the batch 2,415,787, overrunning the previous 2,415,616-byte line by 171. Rebuilt at that final state, byte-identical. A 1 KB bump would clear it by 853 bytes, under every margin the paragraphs above keep, so this takes 2 KB and leaves 1,877 bytes. A further 2 KB was bought by the `refs.context` storage cap. parser.ts is hook-eager, and the cost is boundRefContext plus the docblock explaining why a ref's context is windowed around its column rather than sliced from the head of its line -- on a document written as one long line, every reference stored the whole file. No new module and no new chunk: 25 across both measurements. Measured with the working tree restored to HEAD for the parent build rather than stashed, ceiling temporarily forced to 1 to read the figure: HEAD 2,413,046 bytes; the fix 2,413,859 (+813), which overran the previous 2,413,568-byte line by 291. A 1 KB bump would clear it by 733 bytes, under every margin the paragraphs above keep, so this takes 2 KB and leaves 1,757 bytes. A further 2 KB was bought by teaching KubectlFilter to leave a structured `-o json`/`-o yaml`/template listing alone instead of truncating it to a head and a tail. tool_filters/containers.ts is hook-eager through the bash filter registry, the same path the launcher-options paragraph above describes for helpers.ts, so its compiled bytes land here whether or not a hook ever runs kubectl. The cost is the flag reader -- `-o` has four spellings, separate, `=`-joined and glued to the short flag -- plus the format regex, which is a string literal the bundler ships verbatim rather than a comment it strips. That regex first listed all nine format names kubectl accepts and now matches four prefixes instead, every suffixed spelling being an extension of one of them, which took the batch from +755 bytes to +683. No new module and no new chunk: 25 across both measurements. Measured with the working tree restored to HEAD for the parent build rather than stashed: HEAD 2,417,277 bytes, which had 387 bytes of headroom; the batch 2,417,960, overrunning the previous 2,417,664-byte line by 296. Rebuilt twice at that final state, byte-identical each time. A 1 KB bump would clear it by 728 bytes, under every margin the paragraphs above keep, so this takes 2 KB and leaves 1,752 bytes. A further 2 KB was bought by PowerShell runner resolution and Windows PowerShell harness command wrapping with base64 round-trip validation. src/shell.ts and src/hooks_bash.ts are hook-eager, adding resolvePowerShell and canRunPowerShell helpers plus base64 rewrite logic so Copilot CLI and Codex on Windows execute wrapped commands natively under PowerShell rather than bash without quoting hazards. A further 7 KB was bought by project notes reaching session start and the compaction manifest, plus the passthrough filter and cap hint an inline interpreter read now runs through. hooks_session_start.ts and manifest.ts are hook-eager, so project_memory.ts joined the eager set whole, writer included: 4,828 of the batch's bytes. A deferred import does not fit: preCompactHandler is synchronous, and manifestPrintedPaths must rebuild the manifest under the same character budget the hook printed it with, which the notes are part of, so they would have to be threaded through both. No new chunk: 25 across both measurements. Measured with HEAD built in a separate worktree rather than by restoring this one: HEAD 2,421,826 bytes, already 66 over the previous 2,421,760-byte line on this machine though not on the Linux runner; the batch 2,427,394 (+5,568), built twice, byte-identical. 6 KB would clear it by 510 bytes, under every margin above, so this takes 7 KB and leaves 1,534. A further 2 KB was bought by honoring an inline `TOKEN_GOAT_BASH_COMPRESS=0` prefix across a whole command, pipes and `$env:` assignments included, and by not recording a result the harness persisted to disk as already served. bash_extractors.ts and hooks_bash.ts are hook-eager; the cost is the assignment scanner, whose two regexes are string literals the bundler ships verbatim. No new module and no new chunk: 25 across both measurements. Measured in the checkout with HEAD's three changed sources swapped in and then the fix's, since a scratch copy's path length alone moved the figure by about 1.6 KB: HEAD 2,428,011 bytes, the fix 2,431,008 (+2,997), overrunning the previous 2,428,928-byte line by 2,080. 294 of those bytes are the heredoc scan, which walks lines instead of using a regex that the super-linear-backtracking lint rule rejects. This takes 3 KB and leaves 992. A further 15 KB was bought by wiring Claude Code's PostToolUseFailure event and classifying every failure the brake sees: src/tool_error_class.ts is hook-eager because the brake classifies before deciding whether to speak, and it reaches splitShellSegments and stripLeadingAssignments in bash_extractors.ts to credit a test runner or a grep only at command position. The report behind `session-audit --tool-errors` loads the same classifier through a dynamic import so the core startup set does not carry the Bash parser, and that split is what added chunks here: 25 before, 31 after, the extra boundaries costing about 5 KB of the total. Measured by swapping the HEAD versions of the changed sources into this checkout and building: HEAD 2,431,314 bytes, the batch 2,446,552 (+15,238), overrunning the previous 2,432,000-byte line by 14,552. The static-import alternative measured 2,441,460 here but put 66,290 bytes on the core's eager set instead of 8,236. This takes 15 KB and leaves 808. A further 10 KB is src/call_streak.ts, the batch hint and zero-hit search brake relay.ts applies to every pre/post tool answer, with no new chunk: 31 before and after. Measured by swapping the HEAD versions of the changed sources into this checkout and building: HEAD 2,446,988 bytes, the change 2,456,959 (+9,971), overrunning the previous 2,447,360-byte line by 9,599. This takes 10 KB and leaves 641. A further 11 KB is src/hint_target.ts on the hook path, the real-name resolver every read and bash hook now calls and the repeated-deny shortening relay.ts applies to every answer, with no new chunk (31 before and after): HEAD e5ca3a21 2,457,016 bytes, the change 2,468,245 (+11,229), overrunning the previous 2,457,600-byte line by 10,645, so this takes 11 KB and leaves 619. A further 1 KB is the Claude Code structured Grep/Glob count the dedup note now reads, the transcript-head check on a background task's output, and the content-delivering deny moved to src/delivering_deny.ts, with no new chunk (31 before and after): HEAD ec98d8c8 2,468,245 bytes, the change 2,469,202 (+957), overrunning the previous 2,468,864-byte line by 338, so this takes 1 KB and leaves 686. A further 2 KB is the bounded wait between in-process relay calls, relay.ts re-seeding the session id per event, the subshell-group peel every Bash read gate now matches through, and resolving a relative task `.output` path against the event cwd, with no new chunk (31 before and after): HEAD 43b59523 2,469,776 bytes, the change 2,471,433 (+1,657), overrunning the previous 2,469,888-byte line by 1,545, so this takes 2 KB and leaves 503. A further 5 KB is harness detection discounting the session id relay seeded (registry.ts) and the read-only index fallback on db.ts's getDb (a read-only or immutable open, the write-access error classifier, the schema check), with no new chunk (31 before and after): from the same HEAD, the earlier changes plus the harness fix 2,471,821 and the read-only fallback on top 2,475,649 (+3,828), the first figure from a worktree build less its constant 2,720-byte path offset, which puts HEAD at the 2,469,776 measured above, overrunning the previous 2,471,936-byte line by 3,713, so this takes 5 KB and leaves 1,407. A further 5 KB is worker.ts refusing to spawn against a data directory that refuses writes and sweeping the embed backlog a stopped worker dropped, the damaged-database fail-fast in db.ts, the bounded Read of a tool-results file in hooks_read.ts, the whole-file line-range refusal in hooks_bash.ts, and the outline pointer for the headings a large skill's list leaves out, with no new chunk (31 before and after): HEAD 561feedb 2,475,674 bytes, measured by swapping the HEAD versions of the 14 changed sources into this checkout and building, the change 2,481,175 (+5,501), overrunning the previous 2,477,056-byte line by 4,119, so this takes 5 KB and leaves 1,001. The read commands pricing shell-shaped results through delivery_cap.ts then made that module shared, so it moved into its own 1,338-byte chunk (32 after) and the hook set grew 375 bytes to 2,481,592, inside this line with 584 to spare, so it takes nothing. The Copilot CLI hook shim moving to a .cjs file with a .js forwarder, and doctor inspecting project-scope Copilot hooks, grew the set 609 bytes to 2,482,201 across the same 32 chunks, overrunning the 2,482,176-byte line by 25, so this takes 1 KB and leaves 999. The resident hook server grew it 3,464 bytes to 2,486,650 across 35 chunks: 1,687 of that is code the hook path needs to take part (the per-request reset registrations, the shim fragment that tries a server, Copilot CLI's shim interpolating it), and the rest is the import and export lines of three new chunks, which the thin client entry needs so that it shares modules with the hook library instead of copying them. The shim fragment was first imported from shim_common.ts, which pulled every other shim template into this set for 5,655 bytes; it lives in its own module now. That overran the 2,483,200-byte line by 3,450, so this takes 4 KB and leaves 646. The search-pattern guard accepting a pattern with nothing that repeats or branches without timing it and re-timing a slow one before refusing (regex_guard.ts), note commands refusing to save over a line they cannot read or without the notes lock (project_memory.ts), the session-state merge retrying a briefly locked file (session_store.ts), the dirty-queue append writing again an edit that landed in a queue the worker had just claimed (hooks_index.ts), and the worker skipping absent files when requeueing embeddings grew it 2,923 bytes, with no new chunk (36 before and after): HEAD 008bcb2c 2,485,715 bytes, measured by building HEAD and the change side by side from a scratch copy less its constant 255-byte path offset, the change 2,488,638, overrunning the previous 2,487,296-byte line by 1,342, so this takes 2 KB and leaves 706. A further 2 KB was bought by doctor process health recognizing resident hook-servers and copilot_mcp_tools filtering disabled MCP servers from waste calculations. Measured 2,491,073 bytes across 35 chunks, overrunning the 2,490,368-byte line by 705 bytes, so this takes 2 KB and leaves 1,343. Wiring the native hook client into every installer grew it 11,082 bytes with no new chunk (35 before and after), measured by building HEAD 24799e86 and the change in the same scratch directory: 2,492,439 against 2,503,521 there, 2,500,291 at the repository's own path. 7,645 of it is src/native_hook.ts itself, which the hook path does not call but reaches because it already loads bridges/copilot_cli_install.ts for readCopilotHooksOwners, copilotCliMcpToolsDir and copilotCliUserRoot, and that installer now builds the native command lines; the rest is Copilot's own native command builder, the fallback-reason detail relay.ts records per call, the POSIX quoting helper, and the config key with its environment reading. That overran the 2,489,344-byte line by 10,947, so this takes 12 KB and leaves 1,341. Keeping every line of a rewritten Read on its real line number (the `aligned` fold layout for the Claude Code Read envelope, which the harness numbers by position, with the markdown outline laid over the document rather than printed beside it) grew it 7,021 bytes with no new chunk (35 before and after): 2,503,675 on main 633c6bfc against 2,510,696 with the change, both built in sibling worktrees one directory apart. fold_structure.ts is 2,953 of it, hooks_read_post.ts 1,904, hooks_read_slice.ts 593, fold_delivery.ts 545, and nothing else over 300. It has to be eager because it runs inside the PostToolUse Read hook itself. That overran the 2,504,704-byte line by 5,992, so this takes 7 KB and leaves 1,176. Reading each drained file's own project `.token-goat.toml` in the worker grew it 1,506 bytes with no new chunk (35 before and after): 2,511,025 with the HEAD versions of config_project.ts and worker.ts swapped into this checkout against 2,512,531 with the change, both built at the repository's own path. worker.ts is 736 of it (configRootOf and the per-root reads in makeIndexer, embedFileSerialized and requeueStaleEmbeddings), config_project.ts 672 (withConfigProjectRoot, configProjectRootFor and the override resolveConfigProjectRoot consults), and the chunk import and export lists carrying those two names the other 98. The hook path never drains, but esbuild splits by file, and the hook loads worker.ts for ensureWorkerAlive and the dirty-queue helpers and config_project.ts for every loadConfig, so both arrive whole, as native_hook.ts does above. Splitting the four cloud provider filters out of cloud.ts into cloud_providers.ts added 101 bytes of module boundary to the same set. 96c1ff19 measured 2,510,924 and the two changes together 2,512,531, overrunning the 2,511,872-byte line by 659, so this takes 1 KB and leaves 365. Dating project notes and ordering them by recency (project_memory.ts, which the hook loads for the session-start notes block) grew the set from 2,512,531 bytes at 4691ad28 to 2,514,354, 1,458 over the line, so this takes 2 KB and leaves 590. Moving the worker's process control (pid file, start, stop, heartbeat, error log) into worker_lifecycle.ts and the dirty-queue helpers into dirty_queue.ts took worker.ts off the hook path, which only ever needed those, and with it the four modules only the drain uses: embed_backfill.ts, index_prune.ts, index_reclaim.ts and process_priority.ts. The set went from 2,514,354 bytes at 1321984c to 2,478,170, 35 chunks before and after: 36,184 smaller, so this gives back 35 KB and leaves 934. Reporting what reached the dirty queue rather than what the sweep offered it (reconcile.ts, which the hook loads for the session-start sweep, and the session-start note that says so) and resolving a relative edit path before queueing it (hooks_edit.ts) grew the set from 2,478,443 bytes at b5d140d8 to 2,479,411, 307 over the line; moving the install commands out of cli.ts took 55 off it. So this takes 1 KB and leaves 717. Keying a cached Bash result on the directory its `cd` prefix lands in (commandRunDir and bashRecallKey in hooks_bash_commands.ts, which both Bash hooks now call where each used to build the key inline), and holding the directory each Bash call starts in from its pre hook to its post hook (recordBashStartCwd and takeBashStartCwd in session.ts, mergeBashStartCwds in session_store.ts), and letting a string shaped like an untrusted-content fence through unescaped only when its notice is one token-goat writes, with its body escaped again (neutralizeOutsideFences in injection_scan.ts), grew the set from 2,479,988 bytes at dbb03d4c to 2,483,120, 35 chunks before and after: session.ts is +1,122 of it, injection_scan.ts +638, session_store.ts +611, hooks_bash_commands.ts +312, hooks_bash.ts +133, hooks_bash_post.ts +84, and the import and export lists between chunks, which no module's own bytes include, the other 232. That overran the 2,480,128-byte line by 2,992, so this takes 3 KB and leaves 80. Looking a package-manager script up in the directory a `cd` prefix runs the command in (hooks_bash.ts, hooks_bash_post.ts), and resolving a reported Bash call to the command and directory its output is cached under in one function the post hook and the waste report both call (resolveBashCall in hooks_bash_commands.ts), cost 185 bytes: 2,483,120 over 35 chunks at 7d570d04, 2,483,305 with the waste report still on its own lookup. Its call made waste.ts import hooks_bash_commands.ts, and since the post hook imported projectTranscriptsDir from waste.ts, that edge redrew the chunk boundaries around the whole analyser: 2,486,077 over 36 chunks. Moving projectTranscriptsDir into claude_config_dir.ts, whose own header says a path accessor must not drag the waste analyser onto hook-eager paths, took waste.ts and its 9,224 bytes off the hook path, and the set went to 2,476,728 over 36 chunks, 3,034 of it the import and export lists the new edge still draws between chunks. That is 6,392 smaller than 7d570d04. Writing the Copilot CLI adapter's shrunk image copies with the materializer VS Code's image path uses (materializeShrunkImageFile in bridges/vscode_hooks.ts) rather than a second typed copy gave that function the adapter's hourly sweep throttle, and VS Code's image path loads that function eagerly while the copy it replaced was never on the hook path, which adds the throttle's 143 bytes: 2,476,871 over the same 36 chunks. That is 6,249 smaller than 7d570d04, so this gives back 6 KB and leaves 185. Moving the rendering of the `token-goat stats` report out of stats.ts into stats_report.ts took render/stats_renderer.ts off the hook path, which imports stats.ts to record a stat and never renders a report: the set went from 2,476,871 bytes at ab921ee7 to 2,444,539, 36 chunks before and after: 32,332 smaller. Recording a Bash read only of a file that is there after a command that succeeded, trying a drive-letter index key again at its WSL mount before taking the file for absent, keeping the `cd` in the large-output compress hint when its directory is there, named at its WSL mount when that is where it is, and storing the compound and elided outputs under the directory the command ran in (hooks_bash.ts, hooks_bash_post.ts, and the file and directory checks in paths.ts) add 1,043 bytes: 2,445,582 over the same 36 chunks. That is 31,289 smaller than ab921ee7, so this gives back 30 KB and leaves 754. Opening a drive-letter index key at its WSL mount everywhere a hook opens a file, not only in the existence checks (hostPathOfIndexKey and wslMountOfIndexKey in paths.ts and their callers across the read, Bash and edit hooks), asking the pre-tool path gate about that mount as a second path a hook can open for one typed path (vscode_path_gate.ts), putting the structural-index rewrite's stat behind the same gate, and anchoring the Get-Content parameter patterns to the start of a token grew the set 1,899 bytes with no new chunk (36 before and after): vscode_path_gate.ts is +496 of it, paths.ts +381, hooks_read_slice.ts +147, hooks_bash.ts +136, hooks_read_post.ts +90, bash_structural_index.ts +89, eleven other modules +386 between them, and the import and export lists between chunks the other 174. The three changes landed before it had taken the set from 2,445,582 bytes at caf442b9 to 2,445,991, inside the line: moving the database-client filters into tool_filters/db_clients.ts added 95 bytes of module boundary, the one MCP config read-back the four editor bridges now share (managedServerEntry in bridges/mcp_servers_json.ts) 64, and reading an unquoted leading `~` in a `cd` prefix as the home directory (hooks_bash_commands.ts) 250. The set went to 2,447,890, overrunning the 2,446,336-byte line by 1,554, so this takes 2 KB and leaves 494. Running embeddings on the bundled WebAssembly build of ONNX Runtime when onnxruntime-node is absent first put all of its loader on the hook path with the rest of embeddings.ts's imports, taking the set from 2,447,890 bytes to 2,460,701 over the same 36 chunks. Hooks never embed, so the placing, downloading, tar reading and starting moved behind dynamic imports, into embed_runtime_web.ts and pinned_fetch.ts: 2,453,894 over 37 chunks with that alone, because the lazy module's static imports split embed_runtime.ts and pinned_file.ts into a chunk of their own; 2,450,655 once pinned_file.ts's hashing, copying and downloading followed; 2,449,917 over 36 once the lazy module took what it needs as an argument instead of importing it, which folded the two back into embed_model.ts's chunk. That is 10,784 fewer than the first version and 2,027 more than before it, and the remainder cannot move without moving EMBED_FINGERPRINT: embed_runtime.ts (3,942 bytes: which runtime runs, the memory of a failed start, where the binary lives and whether it is there) and pinned_file.ts (1,079: synchronous publishing and three one-line wrappers) are what embed_model.ts and embeddings.ts, both hashed, import and read synchronously. The first version took 13 KB here; this takes 2 KB, overrunning the old 2,448,384-byte line by 1,533 and leaving 515. A further 2 KB was bought by keeping a failure of that runtime to start on disk, so that `semantic --preflight` and `doctor` in a later process report it instead of calling the runtime ready: they ask isRuntimeAvailable and runtimeLoadError, which hashed embed_model.ts and embeddings.ts read synchronously, so reading the record and building the key it is held to (startFailureKey, recordedStartFailure in embed_runtime.ts) has to be static, while writing and clearing it went into the lazy embed_runtime_web.ts. Measured 2,451,440 bytes over 36 chunks with it, against a 2,450,432-byte line it missed by 1,008, leaving 1,040. Mapping VS Code 1.137's multi_replace_string_in_file, apply_patch, fetch_webpage, runSubagent and get_terminal_output onto the hook path (the two map entries each in hooks_cli.ts, the patch-header reader and derived-key table beside them, getFilePaths in hooks_common.ts, the per-file loop in hooks_edit.ts and the per-URL policy loop in hooks_fetch.ts) added about 2 KB with no new chunk. Wiring the subagent-start hook for Copilot CLI and VS Code (subagentStartHandler and subagentStartWired in hooks_agent_spawn.ts, the subagent_start event name, the agent-id exception in relay.ts's buildEvent, and displaySafeText on the folder name and recent files in baseline.ts's formatProjectMap) took the set from 2,462,891 to 2,466,239 with no new chunk (36 before and after), overrunning the 2,463,744-byte line by 2,495, so this takes 3 KB and leaves 577. Rewriting a long subagent report through Copilot CLI's subagentStop modifiedResponse (subagentReportRewriteHandler, reportNotice, planReportCompaction and recordReportCompaction in hooks_agent_spawn.ts, and the response forward and modifiedResponse reply in the Copilot shim) took the set from 2,466,239 to 2,469,217 with no new chunk (36 before and after), overrunning the 2,466,816-byte line by 2,401, so this takes 3 KB and leaves 671. Reading a Copilot CLI skill's body from the SKILL.md Copilot loaded, since its skill tool result is only a confirmation line (copilotSkillPath in copilot_skill_path.ts, copilotCliUserRoot moved into copilot_home.ts, and the harness gate in hooks_skill.ts), took the set from 2,469,611 to 2,471,159 with no new chunk (36 before and after), overrunning the 2,469,888-byte line by 1,271, so this takes 2 KB and leaves 777. It has to be eager: the skill hooks are where the body is cached and the reload is denied. Loading copilot_skill_path.ts through a dynamic import instead measured 2,471,624 over 39 chunks, larger, because the split added three shared chunks. The per-change sum above had drifted to 2,462,720 bytes, 80 KB clear of the build it guarded, which is room for a whole regression to land unseen. The ceiling is therefore re-based on a measurement rather than summed: HAND-DERIVED, 2,382,498 bytes over 36 chunks, measured by this file's own static closure on the feat/copilot-vscode-integration build rebased onto 4770a39c, plus a 4 KB margin, the size of a typical single increment recorded above. Phase 5 re-measured it on its own branch rebased onto that build: the refused re-read identity row (rereadIdentity in hooks_read_slice.ts, booked from hooks_read.ts), the edit-after-partial-read count (read_shape.ts), and the list of files a compaction summary left out (compact_dropped.ts, written from hooks_compact.ts) took the set to 2,388,942 bytes over the same 36 chunks, 6,444 above the measured figure and 2,348 over the line. All three run inside a hook, so they cannot sit behind a dynamic import; the measured figure is replaced by that one. A change that needs more re-measures and replaces the measured figure here, naming what bought the bytes. The one-time savings receipt (claimFirstSavingsReceipt and firstReceiptShownAt in stats.ts, withSavingsReceipt and savingsReceiptText in hooks_session_start.ts, noticeField in hook_registry.ts) measured 2,395,393 bytes over 36 chunks, 2,355 over the line: it runs inside the session_start hook and rides stats.ts, which the hook path already loads, so it is eager like the rest of that module. Re-based on that measurement plus the same 4 KB margin. The Antigravity CLI bridge (antigravityToCanonicalWire and its tool-name and argument-key maps in hooks_cli.ts, serializeAntigravityOutput in bridges/antigravity_hooks.ts, dispatched from hook_registry.ts) measured 2,403,142 bytes over 40 chunks, 3,653 over the line: every agy hook call is translated in on the way in and shaped on the way out, so both halves run inside the hook and are eager like the Gemini and Qwen serializers beside them. Re-based on that measurement plus the same 4 KB margin. smol-toml 1.8.0 -> 1.9.0, the release that fixes the denial of service in GHSA-r4xh-jqrq-34v2, took the set from 2,406,552 to 2,413,530 bytes over the same 40 chunks, 6,292 over the line, measured on one tree with only that package swapped. It is inlined into the chunk that holds src/config.ts, which every hook call loads to read config.toml, so it is eager for the same reason config is. Re-based on that measurement plus the same 4 KB margin. Reading Gemini, Qwen and Grok read_file results as raw text (rawTextReadHarness, splitRawReadHeader, rawReadCutShort and wholeNumberedReadResult in hooks_read_slice.ts) took the set from 2,414,265 to 2,418,127 bytes over the same 40 chunks, 501 over the line, measured on one tree with only that file swapped: the post_tool_use Read hook is what decides whether a delivery was cut short, so the parser runs inside the hook and is eager like the Claude Code parser beside it. Re-based on that measurement plus the same 4 KB margin. Anchored project notes took the set from 2,421,706 bytes over 40 chunks to 2,424,764 over 43, 2,541 over the line, measured on one tree with only this change swapped. project_memory.ts grew 1,650 bytes: session_start loads it to inject the notes, and it has to read and rewrite the `# anchor:` line or a save would drop it. The rest is the three chunk boundaries esbuild cut for the dynamic import that keeps note_anchor.ts, notes.ts and the index reads they make off this set; loading them eagerly instead measured 2,427,842. Re-based on that measurement plus the same 4 KB margin. The next round grew the set 6,893 bytes on one worktree measured before and after, all of it code the hooks run: the compaction reset in session_store.ts and hooks_compact.ts, the launcher spellings and line-range checks in hooks_bash_commands.ts and hooks_bash_post.ts, the ranged-Read overlap check in hooks_read.ts, the expired-id filter in manifest.ts and web_cache.ts, the clip floor in tool_filters/helpers.ts, and the sparse save in config.ts. Re-based on the main checkout's measurement of 2,442,785 plus the same 4 KB margin. Later rounds spent that margin, and the 2 KB it was widened by, until the set stood at 2,448,844, 85 bytes under the line. The next 93 bytes are withTelemetryWriteDb in stats.ts: the short-budget connection recordStat already opened, pulled out so the semantic-query ledger writes through it too instead of waiting out the shared connection's 15s busy_timeout. It runs inside recordStat on every hook, so it is eager for the same reason that connection was. Re-based on the main checkout's measurement of 2,448,937 plus the same 4 KB margin. Later redaction and parser fixes spent that margin, and the line was crossed by the PowerShell wrapper in bash_runner.ts, which now resets LASTEXITCODE and captures the script's own `$?` so a wrapped run exits as a direct `pwsh -Command` would; it is the hook-side runner, so it is eager. Re-based on the main checkout's measurement of 2,453,145 plus the same 4 KB margin. The single-pass banner, hint and skill-name scanners in line_matchers.ts spent that margin to 2,457,124, and the line was crossed by 517 bytes in secret_redact.ts: url_credentials now accepts an empty or `@`-bearing username, and hasPreciseSecret skips a custom pattern's empty match, which matters here because the Read-hook fold gates call it. Re-based on the main checkout's measurement of 2,457,641 plus the same 4 KB margin. The PowerShell wrapper in bash_runner.ts, which the Bash hook loads eagerly, then grew twice to match direct pwsh exit codes: once for statement errors and `return`, measured 2,462,612 in a clean checkout of 822d121b, and once for named blocks, quoted, dotted and computed break labels and Windows PowerShell 5.1, measured 2,464,017. That is measured with nothing uncommitted in the tree, because an unrelated uncommitted edit hid the first crossing by shrinking the set. Re-based on 2,464,017 plus the same 4 KB margin. A further 5 KB was bought by the indexer fixes for C++ destructors and operators, CommonJS exports, C tag definitions and whole YAML/TOML block spans in parser_treesitter.ts and parser_structured.ts: the read hook's source-skeleton fold calls the tree-sitter extractors synchronously through parseSourceSymbolsTreeSitterOnly, so they cannot sit behind a dynamic import the way the regex adapters do. Measured in a clean worktree: 2,470,165 at HEAD and 2,475,197 with the fixes. Re-based on 2,475,197 plus the same 4 KB margin. A clean Windows worktree at ff780313 already measured 2,480,769 after the round of graph and index fixes that followed, while the Linux CI build stayed under the line, and a further 4.3 KB was bought by Ruby calls without parentheses recorded as references in parser_refs.ts, go test -v log lines kept with their own test in tool_filters/go_test.ts, setext heading levels in hint_target.ts and the record of install-created configs in bridges/created_configs.ts, all four reached synchronously from the edit and read hooks. Measured 2,485,071 with nothing else uncommitted. Re-based on 2,485,071 plus the same 4 KB margin. A further 7.1 KB was bought by four fixes that all run on a hook path: the cargo failing-build collapse in tool_filters/build.ts runs inside the Bash hook's output compression, the AVIF/HEIC probe in image_shrink.ts adds only its ftyp check and a dynamic-import stub because image_isobmff.ts itself loads lazily, the hint fixes in hint_target.ts and hooks_read.ts run in the synchronous Read and Grep hooks, and the owner-checked symlink resolution in util.ts sits under every atomic write. Measured 2,492,170 over 50 chunks with nothing else uncommitted. Re-based on 2,492,170 plus the same 4 KB margin. A further 6.4 KB came from the interpolation-hole scanner in languages/common.ts (4.2 KB), which the hook loads eagerly because a post-edit hook can reindex a file inline and the brace walk is synchronous, so it cannot sit behind a dynamic import; the rest is the heading-body re-slice in read_commands.ts, the CRLF-preserving block edit in util_config.ts and the CLAUDE_CONFIG_DIR lookup in claude_config_dir.ts. Measured 2,498,557 over 50 chunks with nothing else uncommitted. Re-based on 2,498,557 plus the same 4 KB margin. A further 1.4 KB came from whole-document JSON handling for aws, az and npm audit in tool_filters/cloud_providers.ts, package_managers.ts and helpers.ts, which run inside the Bash hook's output compression, and the byte-order-mark keep in util_config.ts, which sits under writers that module already carries onto the hook path. Measured 2,504,004 over 50 chunks with nothing else uncommitted (a worktree whose node_modules is a junction reads about 2.2 KB higher, because esbuild writes the junction's real path into every module marker comment). Re-based on 2,504,004 plus the same 4 KB margin. The PowerShell heredoc adapter powershell_compat.ts, which bash_runner.ts imports on the compress path, took it to 2,510,432 on a branch the next five measurements were taken without. Dropping a file's recorded line ranges once it changed on disk before a `sed -n` or `head` read is measured against them (dropFileLineRangesIfChanged and the range-identity map in session.ts, its merge in session_store.ts, the calls in hooks_bash.ts and bash_extractors.ts) runs inside the Bash pre hook, so it is eager; with the 58-byte snippet_window.ts that recall_index.ts brings onto this set it took the set from 2,506,477 at 504aa3b8 to 2,508,739 over the same 51 chunks, measured with nothing else uncommitted, 639 over the line. Re-based on 2,508,739 plus the same 4 KB margin. Telling a project indexed on purpose from one where the edit hook indexed a single file (indexed_roots.ts) is read by the session-start hook, and its table's v16 -> v17 backfill sits in db.ts, which every hook opens; the predicate is loaded behind a dynamic import, which costs one more shared chunk. It took the set from 2,512,086 over 52 chunks at f8ba8c4d to 2,513,395 over 53, measured with nothing else uncommitted, 560 over the line. Re-based on 2,513,395 plus the same 4 KB margin. rewrite_permission.ts then added about 17 KB: every rewrite the pre_tool_use Bash, Agent and Read hooks make must first be checked against the user's Claude Code permission rules, and the Bash and Agent handlers are synchronous, so the check cannot sit behind a dynamic import; it took the set to 2,532,536 over 53 chunks, measured with nothing else uncommitted. With the PowerShell adapter merged in, the set measured 2,536,695 over the same 53 chunks, less than the two increments summed because they share chunks. The first-read symbol policy (hooks_read_policy.ts and its index_reader.ts and db.ts lookups) runs inside the Read pre hook, so it is eager too; it added 9,063 bytes on its own branch (2,510,432 to 2,519,495). Merged with everything above, and with the permission checks that follow links and 8.3 short names, treat settings.local.json as untracked only when git says so, read settings from the session's project and read Codex rules files, the set measured 2,548,632 over 52 chunks with nothing else uncommitted. Holding back read hints on paths a Read deny or ask rule covers took that to 2,552,512. The policy's later revision (a whole read after slices, the heading-tree intercept, the stale-evidence fail-open) added 2,568 bytes on its own branch (2,521,517 to 2,524,085); merged with everything above the set measured 2,552,844 over 52 chunks with nothing else uncommitted. Re-based on that plus the same 4 KB margin. At fb08050a it measured 2,554,811 over 53 chunks. Counting only the reads and edits since the last compaction (epochReadCounts, mergeEpochBase and wasEditedSinceCompaction in session.ts, their persistence in session_store.ts) is read by the Read pre hook's re-read denies and by the pre-compact manifest, so it is eager; it took the set to 2,557,331 over the same 53 chunks, measured with nothing else uncommitted, 391 over the line. Re-based on 2,557,331 plus the same 4 KB margin. The zero epoch base an entry created after a compaction carries (session.ts newEntryEpochBase) and the one-read cap epochReadCounts applies to a hook process that missed the compaction are on that same eager Read pre hook path; they took the set to 2,561,526 over the same 53 chunks, measured with nothing else uncommitted, 99 over the line. Re-based on 2,561,526 plus the same 4 KB margin. Quoting every argument of a suggested command with one mark (hint_suggestion_guard.ts quotedArgs, used by the Read pre hook's edit-anyway and truncated-read denies) is on that same path; with the first-read policy fixes before it, it took the set to 2,565,785 over the same 53 chunks, measured with nothing else uncommitted, 163 over the line. Re-based on 2,565,785 plus the same 4 KB margin. The PowerShell lexer that keeps the heredoc and python -c rewrites to top-level code (powershell_compat.ts layoutPowerShell) is reached through bash_runner.ts, which hooks_bash_post.ts imports statically for MAX_CAPTURE_BYTES; it took the set to 2,570,838 over the same 53 chunks, measured with nothing else uncommitted, 957 over the line. Re-based on 2,570,838 plus the same 4 KB margin. Cutting an unsafe suggested command at its own span (hint_suggestion_guard.ts fencedCommandEnd and the backslash-aware scanQuotes) and naming the file in read-hook prose only where no command carries it (hooks_read.ts fileSubject) are on the same Read pre hook path; they took the set to 2,574,986 over the same 53 chunks, measured with nothing else uncommitted, 52 over the line. Re-based on 2,574,986 plus the same 4 KB margin. Naming a skill, a symbol, a tool or a downloaded file in hook prose through nameSubject and fileSubject (hint_suggestion_guard.ts) is on the same hook path; it took the set to 2,579,113 over the same 53 chunks, measured with nothing else uncommitted, 31 over the line. Re-based on 2,579,113 plus the same 4 KB margin. Reading a heredoc body the way bash does, the `-c` loader that imports only sys, and naming a `cat >` file as a PowerShell string literal (powershell_compat.ts, reached through bash_runner.ts as above) took the set to 2,583,416 over 54 chunks, measured with nothing else uncommitted, 207 over the line. Re-based on 2,583,416 plus the same 4 KB margin. Declining the map hint for a directory listing that asks for sizes, times or permissions (the metadata-flag check in bash_extractors.ts, +788, run by the Bash pre hook) took the set to 2,596,163 over 60 chunks, measured on top of the line-range read lock in read_range_evasion.ts, 8,651 over the line. Re-based on 2,596,163 plus the same 4 KB margin. */ +const MAX_HOOK_EAGER_BYTES = 2_596_163 + 4 * 1024 /** Chunk filenames `file` imports with a static `import ... from "./..."`, not a deferred one. */ function staticChunkImports(file: string): string[] { diff --git a/tests/hooks_bash.test.ts b/tests/hooks_bash.test.ts index ab5c54e65..d03b8dc8a 100644 --- a/tests/hooks_bash.test.ts +++ b/tests/hooks_bash.test.ts @@ -2612,16 +2612,25 @@ describe('preBashHandler — directory listing map hint', () => { clearModuleCaches() }) - it('emits map hint for eza --long on a path', () => { - const result = preBashHandler(makeBashEvent('eza --git --long src/')) - expect(result.hookType).toBe('context') - if (result.hookType === 'context') { - expect(result.context).toContain('token-goat map') - } + // CAPTURE (a research session, 2026-10): `ls -la dist/` got "Use `token-goat map --compact`", and map lists files and symbols, not the sizes, times and permissions a long listing asks for. The other spellings are HAND-DERIVED from each tool's metadata flags. + it.each([ + 'ls -la dist/', + 'ls -la dist/ | head -20', + 'ls -lh dist/ | wc -l', + 'ls -lt src/ | head', + 'ls --full-time src/ | grep .ts', + 'ls -lR src/', + 'ls -laR', + 'eza --git --long src/', + 'eza -l --tree src/', + 'tree -h src/', + ])('names no map for a listing that asks for metadata: %s', (command) => { + const result = preBashHandler(makeBashEvent(command)) + expect(result.hookType === 'context' ? result.context : '').not.toContain('token-goat map') }) - it('emits map hint for ls -la piped to head', () => { - const result = preBashHandler(makeBashEvent('ls -la src/ | head -20')) + it('emits map hint for ls piped to head', () => { + const result = preBashHandler(makeBashEvent('ls -a src/ | head -20')) expect(result.hookType).toBe('context') if (result.hookType === 'context') { expect(result.context).toContain('token-goat map') @@ -2658,24 +2667,8 @@ describe('preBashHandler — directory listing map hint', () => { } }) - it('emits map hint for ls -lR', () => { - const result = preBashHandler(makeBashEvent('ls -lR src/')) - expect(result.hookType).toBe('context') - if (result.hookType === 'context') { - expect(result.context).toContain('token-goat map') - } - }) - - it('emits map hint for ls -laR', () => { - const result = preBashHandler(makeBashEvent('ls -laR')) - expect(result.hookType).toBe('context') - if (result.hookType === 'context') { - expect(result.context).toContain('token-goat map') - } - }) - - it('still fires for eza --long (existing path)', () => { - const result = preBashHandler(makeBashEvent('eza --git --long src/')) + it('emits map hint for ls -aR', () => { + const result = preBashHandler(makeBashEvent('ls -aR')) expect(result.hookType).toBe('context') if (result.hookType === 'context') { expect(result.context).toContain('token-goat map') From 609487be5f058909b22722ee694cb4965a7328d4 Mon Sep 17 00:00:00 2001 From: Zelys Date: Tue, 13 Oct 2026 23:39:03 -0500 Subject: [PATCH 15/92] feat(hooks): move a long Bash command's quoted heredoc bodies into files on Claude Code for Windows Claude Code's Bash tool on Windows fails a command of about 7,300 characters or more before any of it runs ("unexpected EOF while looking for matching" a quote, exit 2), and token-goat's PreToolUse answered {} to every one: across the measured sessions every heredoc command of 7,963 characters or more failed. Git Bash itself, spawned with bash -c, cuts the argument near 8,190 characters and writes the heredoc truncated with exit 0. On Claude Code for Windows, a command that opens a heredoc and whose length, counting three more for each apostrophe and one more for each double quote or backslash, reaches 6,500 has each quoted heredoc's body moved into a file, and the operator, body and closing line become < '' with the rest of the command as written. The heredocs come from the PowerShell adapter's lexer, now exported as topLevelHeredocs and taught the <<\TAG form; text outside the bodies where that lexer and bash could disagree, and a delimiter bash would read as a longer word (<<'EOF'x), leave the command alone. The rewrite goes through shellRewrite and permissionNeutralRewrite like every other shell rewrite, and ships only when that decision approves it, which in practice is bypassPermissions: a rewrite left to a prompt would ask the user to approve a command whose body the prompt cannot show. A body holding anything redactSecrets would redact is never written. Each body goes to a new file under ~/.token-goat/heredoc_spill, the directory created 0700 and the file opened exclusively with mode 0600, so an existing path is never written through and two spills of one body never share a file; spills older than an hour are removed whenever another is written. Where the rewrite cannot ship, a command of 8,000 characters or more is refused with the reason and what to do; below that it runs as written. README.md, docs/security.md and CLAUDE.arch.md name the new rewrite and what it writes to disk. src/bash_heredoc_spill.ts is the 502nd TypeScript module under src/, past the 501 that parser_stamp_resolver_is_unhashed pinned, so that population is re-pinned at floor 470 and ceiling 520. The hook entry's eager set grows to 2,602,003 bytes over 60 chunks, past the dist_chunks_deduped ceiling, so the ceiling is re-based on the measurement plus the usual 4 KB and its doc comment names what bought the bytes. --- CLAUDE.arch.md | 3 +- README.md | 1 + docs/security.md | 4 +- src/bash_heredoc_spill.ts | 134 +++++++++ src/hooks_bash.ts | 22 ++ src/powershell_compat.ts | 27 +- tests/bash_heredoc_spill.test.ts | 282 ++++++++++++++++++ tests/bash_heredoc_spill_e2e.test.ts | 153 ++++++++++ tests/guards/dist_chunks_deduped.test.ts | 4 +- .../parser_stamp_resolver_is_unhashed.test.ts | 2 +- tests/powershell_heredoc_compat.test.ts | 6 + 11 files changed, 624 insertions(+), 14 deletions(-) create mode 100644 src/bash_heredoc_spill.ts create mode 100644 tests/bash_heredoc_spill.test.ts create mode 100644 tests/bash_heredoc_spill_e2e.test.ts diff --git a/CLAUDE.arch.md b/CLAUDE.arch.md index 763abe7d7..81b10f232 100644 --- a/CLAUDE.arch.md +++ b/CLAUDE.arch.md @@ -285,6 +285,7 @@ token-goat is a TypeScript CLI bundled to `dist/token-goat.mjs` via esbuild. The | Module | Role | |--------|------| | [`src/bash_extractors.ts`](src/bash_extractors.ts) | Command line extractors, classification, and surgical hint builders for bash hook handlers. | +| [`src/bash_heredoc_spill.ts`](src/bash_heredoc_spill.ts) | Moves the bodies of a long Bash command's quoted heredocs into temp files, for Claude Code's Bash tool on Windows, which fails a command of about 7,300 characters or more before an | | [`src/bash_output_cache.ts`](src/bash_output_cache.ts) | Bash stdout/stderr disk store (byte cap plus 4096 file-count cap, oldest-first eviction) | | [`src/bash_range_savings.ts`](src/bash_range_savings.ts) | Prices a line-range read's proposed surgical replacement against the read itself, so a hint that redirects a line-range read can be required to prove it saves something before it i | | [`src/bash_runner.ts`](src/bash_runner.ts) | Exports: `DEFAULT_TIMEOUT_SECONDS`, `MAX_CAPTURE_BYTES`, `RunOptions`, `run` | @@ -807,7 +808,7 @@ That alternative was built and then withdrawn, which is the part worth keeping. **Where the fence is available, and where it is not.** [`tests/guards/substituted_output_reaches_fence.test.ts`](tests/guards/substituted_output_reaches_fence.test.ts) walks every function reaching `emitRewrite`, and the eight it found split cleanly in two. Where token-goat's text sits at one end of the body -- the `BashOutput` and `TaskOutput` poll deltas, and the compound-Bash rewrite before them -- the notice goes above the opening tag and the fence covers the rest, which is the shape to copy. Where our markers are spliced *between* the bytes they replaced, there is no such cut: fencing the whole body would run the marker neutraliser over token-goat's own markers and hand the model `[token-goat: 40 lines elided]`. The Bash elisions are no longer in that position. [`fenceUntrustedSpans`](src/untrusted_fence.ts) takes the body already split into spans, each marked with whether token-goat wrote it, and runs the neutraliser on the others and on nothing else. Authorship is positional, declared by the producer that emitted the span, rather than recognised from the text: a rule that spotted our markers by their spelling would exempt a forged one just as readily. Both `hooks_bash_post.ts` elision sites fence through it, and the fence is priced before the net-benefit gate rather than after, so the gate cannot decline a rewrite for the cost of its own protection. Three remain open: the agent-report compaction, the already-served Read elision, and the browser block join. What keeps them open is the round-trip risk rather than the missing mechanism, since an untrusted span containing the literal `[token-goat` comes back escaped and those three feed the surfaces most likely to be copied into a file. Reopen them when there is a way to escape a span that a model cannot round-trip into an edit, or when a marker-shaped line is measured arriving on one of the three rather than assumed. -**A rewrite must leave the user's own permission rules deciding the original call.** Claude Code matches its permission rules against a PreToolUse hook's `updatedInput`, and `permissionDecision: "allow"` skips its prompt, so when `serializeOutput` answered every rewrite with allow, a compressible `curl` wrapped in `token-goat compress` ran with no prompt, a `Bash(curl:*)` deny never saw `curl` at the head of the command, and a shrunk image's temp path walked past a `Read(./private/**)` deny. Every rewriteInput producer (Bash compression, the structural rg/grep query, the Agent briefing, the image Read) goes through `permissionNeutralRewrite` in [`src/rewrite_permission.ts`](src/rewrite_permission.ts), which reads every settings source Claude Code could apply (managed file, registry and plist policy, the user files, every `.claude` directory from the cwd up) and declines the rewrite when a deny or ask rule could match the original or any source cannot be read. A registry policy key counts as absent only when `reg` gives its own not-found answer and gives the identical answer for a sentinel key under HKLM that no user can create; any other failure leaves the policy unreadable, because a user can deny read access to an HKCU key and so fake its absence. `reg` and PowerShell start only by absolute path from a validated Windows folder ([`src/windows_system32.ts`](src/windows_system32.ts)), never by bare name, since a bare name or an unvalidated `SystemRoot` lets a `reg.cmd` planted in the working directory answer in their place. Claude Code checks a path rule against both the requested path and the file it resolves to, and the rewrite shows it neither, so the image Read's path and every word of a shell command are also matched under `realpathSync.native` (symlinks, junctions, 8.3 short names, a not-yet-created file through its nearest existing ancestor); a junction `pics -> secrets` once carried a Read past `Read(./secrets/**)`. Its `approve` verdict is the only thing that puts allow on the Claude Code wire, and it is set only when the original would have run with no prompt anyway: a trusted allow rule matching a simple command, a built-in read-only command, an image Read inside the working directory. A `.claude/settings.local.json` allow is trusted only when `git ls-files --error-unmatch` exits 1 (untracked inside a repository); exit 128 once counted too, but it also means no repository at all, as in an unpacked archive that ships the file, or one git refuses under `safe.directory`. `loadPermissionSnapshot` anchors on `CLAUDE_PROJECT_DIR` (where the session started) as well as the hook's cwd: deny and ask rules count from every `.claude` folder above the cwd, the project and its main checkout (a linked worktree resolves through `commondir`), a cwd outside the project returns null, and a local allow counts only at the starting directory while the cwd is there and, off Windows, that directory is the checkout root, the one place old and new Claude Code versions agree they read it from. A rewrite is never approved in plan mode. In auto and bypassPermissions the user is never prompted, so a rewrite there is approved or skipped, never deferred: `decideRewrite` skips every kind in auto (auto drops broad allow rules and has its classifier review even read-only commands, so an allow skipped that review, and a plain rewrite could draw a prompt the original would not), and in bypassPermissions approves a non-skip verdict only when the snapshot has no `PermissionRequest` hook and [`src/claude_hidden_rules.ts`](src/claude_hidden_rules.ts) finds no source the settings files do not show: an entry point other than `cli`/`sdk-cli` (an SDK host's `canUseTool`, an editor), a claude command line carrying any flag outside its harmless allowlist (`--disallowedTools`, `--settings`), read from `CLAUDE_PID` through CIM, `/proc` or `ps`, a skill, command or agent `.md` whose `disallowed-tools` names a pattern, and a plugin cache `hooks.json` with `PermissionRequest`, scanning nested `.claude` folders through `git ls-files` because Claude Code skips gitignored ones. That module is a dynamic import primed by `loadingHiddenRuleCheck` on the bypass event, so it stays off the eager chunk, and an unprimed check counts as hidden. Every other mode withholds an `approve` the same way: a `PermissionRequest` hook in the snapshot, or the same module asked for the files only (skills, commands, agents and plugin hooks, not the claude process), on Claude Code and VS Code alike, and `lineRules` (the claude command line, where `--setting-sources` is harmless only when it names user, project and local) on Claude Code only; with the check unloaded no mode approves. A tracked skill file that is gone from disk is read from the git index, an untracked gone one adds no rule, and the nested scan is stamped like the fixed folders so a clean answer is dropped when a file it read changes, when the checkout's index or HEAD changes (the git path stamps both, and a linked worktree's or separate git directory's), or when a folder that git listed within three levels of the project, or on the path to a listed `.claude` at any depth, gains an entry; a scan whose stamp is under 2 s old (`RACY_MS`, git's racy-index rule, which also covers a `.claude` made between the listing and the stamping), cannot be read (only ENOENT and ENOTDIR are `absent`) or needs 2000 folders or more is `volatile` and its clean answer is rescanned on the next call; a `.claude` first created untracked more than three levels down in a folder with no listed file keeps the minute. [`tests/guards/bypass_and_auto_never_prompt.test.ts`](tests/guards/bypass_and_auto_never_prompt.test.ts) sweeps the decision and forbids any `ask` decision or a `rewriteInput` built outside this module. Every doubt resolves to no rewrite, because losing a compression costs tokens and skipping a rule costs the user's control. Codex is the one wire that still carries allow, since it applies `updatedInput` only beside it. Codex matches its execpolicy `prefix_rule`s against the command's words, which the wrapper hides, so a Codex shell rewrite goes through `decideCodex` in [`src/codex_rules.ts`](src/codex_rules.ts), a dynamic import that `loadCodexRules` primes from the Codex Bash hook so its Starlark reader stays off every other hook's eager path (unprimed, the rewrite is skipped): it ships only when no rule of any decision in any rules layer (the system folder, `~/.codex`, `CODEX_HOME`, every `.codex` from the cwd up) could match the original, because even an allow rule runs the original outside the sandbox, which the wrapper would lose. A rule headed by a shell needs only its script tokens, since Codex matches a script it cannot split as `[shell, flag, script]` (the TUI saves Windows approvals that way, as `pwsh.exe -Command