diff --git a/CHANGELOG.md b/CHANGELOG.md index edea5bc2..cdf8c511 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **MCP shutdown persistence**: Treat late completion of an already-interrupted operation as a no-op. Prevent cancelled startup diagnostics from scheduling writes after ordered shutdown has drained, including recreation of removed runtime storage. - **Startup watcher catch-up**: Request one ordinary coordinated refresh after file/Git subscriptions become ready when automatic indexing is enabled. Recover edits made during initial watcher startup or background-leader promotion without changing watcher-only startup or queuing work after stop. - **Consumer-visible evidence correctness**: Count only retained source and graph rows in pre-edit details, close truncated source fences at whole lines, and reserve outer-heading/final-truncation overhead before source fitting. Credit source only when its complete fitted block survives final composition. Allow structural-mode architecture maps to read branch files from SQLite without a vector store. Add native-backed citation, Unicode/CRLF, scope, and token-budget acceptance. +- **Conceptual source overlap**: Preserve the longest uncovered source interval from overlapping same-file retrieval hits, slice aligned source bodies to emitted lines, and omit declaration columns on fragments. Exclude terminal LF delimiters from expanded source coordinates so citations do not claim a phantom trailing source line. Keep definition lookup ranges and PDF page evidence unchanged. - **Pre-edit source identity and cancellation**: Resolve canonical native symbol IDs before fitting target source, including large fragmented declarations, instead of accepting ranked body slices, suffix paths, or enclosing declarations. Cap source blocks at 512 tokens, withhold missing source, preserve default host roots, and share the exact lookup with evaluation. Keep selection-hook test references metadata-only and honor cancellation after empty lookups. - **Working-tree snapshot integrity**: Detect concurrent database publication even when source hashes and catalog counts are unchanged. Reject lossy UTF-8 Git paths and retain only project-local endpoints for cross-boundary renames. diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 947df284..3fc98100 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -3422,10 +3422,18 @@ export class Indexer { return { content, startLine: metadata.startLine, endLine: metadata.endLine }; } - const lines = (await fsPromises.readFile(filePath, "utf-8")).split("\n"); + const source = await fsPromises.readFile(filePath, "utf-8"); + const hasTerminalNewline = source.endsWith("\n"); + const lines = source.split("\n"); + if (hasTerminalNewline) lines.pop(); const startLine = Math.max(1, metadata.startLine - contextLines); const endLine = Math.min(lines.length, metadata.endLine + contextLines); - return { content: lines.slice(startLine - 1, endLine).join("\n"), startLine, endLine }; + const content = lines.slice(startLine - 1, endLine).join("\n"); + return { + content: hasTerminalNewline && endLine === lines.length ? `${content}\n` : content, + startLine, + endLine, + }; } private async createRerankerDocumentText( diff --git a/src/tools/context-pack.ts b/src/tools/context-pack.ts index 42476c23..ce664c01 100644 --- a/src/tools/context-pack.ts +++ b/src/tools/context-pack.ts @@ -152,7 +152,9 @@ export function fitTextToContextBudget(text: string, tokenBudget?: number): Budg }; } -function normalizedLineRange(result: SearchResult): { start: number; end: number } { +type LineRange = { start: number; end: number }; + +function normalizedLineRange(result: SearchResult): LineRange { return result.startLine <= result.endLine ? { start: result.startLine, end: result.endLine } : { start: result.endLine, end: result.startLine }; @@ -181,8 +183,55 @@ function rankContextCandidates( }); } -function deduplicateContextCandidates(candidates: RankedSearchResult[]): SearchResult[] { - const acceptedByFile = new Map>(); +function longestUncoveredRange(range: LineRange, accepted: LineRange[]): LineRange | undefined { + let cursor = range.start; + let bestStart = cursor; + let bestEnd = cursor - 1; + // Accepted intervals are sorted. Keep one contiguous fragment per candidate; + // equal-length gaps retain the earliest source lines. + for (const item of accepted) { + if (item.end < cursor) continue; + if (item.start > range.end) break; + const end = Math.min(range.end, item.start - 1); + if (end - cursor > bestEnd - bestStart) { + bestStart = cursor; + bestEnd = end; + } + cursor = Math.max(cursor, item.end + 1); + if (cursor > range.end) break; + } + if (range.end - cursor > bestEnd - bestStart) { + bestStart = cursor; + bestEnd = range.end; + } + if (bestEnd < bestStart) return undefined; + return bestStart === range.start && bestEnd === range.end + ? range + : { start: bestStart, end: bestEnd }; +} + +function fragmentContent(content: string, original: LineRange, selected: LineRange): string { + if (!content) return ""; + let line = original.start; + let offset = 0; + let startOffset = 0; + let endOffset = content.length; + for (;;) { + const newline = content.indexOf("\n", offset); + if (line === selected.start) startOffset = offset; + if (line === selected.end) endOffset = newline < 0 ? content.length : newline; + if (newline < 0) break; + line += 1; + offset = newline + 1; + } + // Search results may carry unavailable-source markers or transformed markup. + // Do not attribute an unaligned body to a narrower source range. + const alignedLineCount = line === original.end || (line === original.end + 1 && content.endsWith("\n")); + return alignedLineCount ? content.slice(startOffset, endOffset) : ""; +} + +function deduplicateContextCandidates(candidates: RankedSearchResult[], preserveWholeRanges: boolean): SearchResult[] { + const acceptedByFile = new Map(); const deduplicated: SearchResult[] = []; for (const { result } of candidates) { @@ -191,12 +240,30 @@ function deduplicateContextCandidates(candidates: RankedSearchResult[]): SearchR ? `${result.filePath}#pdf-page-${result.documentLocation.pageStart}-${result.documentLocation.pageEnd}` : result.filePath; const accepted = acceptedByFile.get(locationKey) ?? []; - if (accepted.some((item) => item.start <= range.end && range.start <= item.end)) { - continue; + let selected = range; + if (preserveWholeRanges || result.documentLocation?.kind === "pdf") { + if (accepted.some((item) => item.start <= range.end && range.start <= item.end)) continue; + } else { + const uncovered = longestUncoveredRange(range, accepted); + if (!uncovered) continue; + selected = uncovered; + } + let emitted = result; + if (selected.start !== range.start || selected.end !== range.end) { + emitted = { + ...result, + startLine: selected.start, + endLine: selected.end, + content: fragmentContent(result.content, range, selected), + }; + delete emitted.startCol; + delete emitted.endCol; } - accepted.push(range); + let insertionIndex = 0; + while (insertionIndex < accepted.length && accepted[insertionIndex].start < selected.start) insertionIndex += 1; + accepted.splice(insertionIndex, 0, selected); acceptedByFile.set(locationKey, accepted); - deduplicated.push(result); + deduplicated.push(emitted); } return deduplicated; @@ -406,7 +473,7 @@ export function buildContextPack(results: SearchResult[], options: ContextPackOp ? results.map((result, originalIndex) => ({ result, originalIndex })) : rankContextCandidates(results, options.preferImplementationPaths ?? false, options.projectRoot); const rankedCandidates = ranked.map((entry) => toContextPackTraceCandidate(entry.result)); - const deduplicated = deduplicateContextCandidates(ranked); + const deduplicated = deduplicateContextCandidates(ranked, preserveInputOrder); const deduplicatedCandidates = deduplicated.map((result) => toContextPackTraceCandidate(result)); const configuredOrigins = options.origins ? canonicalOrigins(options.origins) : []; const observedOrigins = new Set(deduplicated.map((result) => resultOrigin(result, configuredOrigins))); diff --git a/tests/context-evidence-acceptance.test.ts b/tests/context-evidence-acceptance.test.ts index 1a393c2d..72c43168 100644 --- a/tests/context-evidence-acceptance.test.ts +++ b/tests/context-evidence-acceptance.test.ts @@ -11,7 +11,7 @@ import { resolveCodebaseContext } from "../src/tools/context.js"; import { resolveCodebaseEditContext } from "../src/tools/edit-context.js"; import { getArchitectureContext } from "../src/tools/operations.js"; import { configCache, getIndexerCacheKey, indexerCache } from "../src/tools/operation-runtime.js"; -import { countContextTokens } from "../src/tools/context-pack.js"; +import { buildContextPack, countContextTokens } from "../src/tools/context-pack.js"; import { estimateTokens } from "../src/utils/cost.js"; const TARGET = "signalGateForUnicodeEnvelopeValidationWithPreservedSourceCoordinatesAndExplicitEvidenceIdentity"; @@ -244,6 +244,75 @@ afterEach(async () => { }); describe("native-backed consumer evidence acceptance", () => { + it("maps fragments of context-expanded search bodies to the expanded source coordinates", async () => { + const relative = "src/expanded-fragment.ts"; + const symbol = "expandedFragmentCoordinateProof"; + const content = [ + "export const first = 0;", + "export const second = 1;", + "export const beforeExpanded = 'é雪🙂';", + "", + `export function ${symbol}(value: string): string {`, + " const normalized = value.trim();", + " if (!normalized) return 'empty';", + " return normalized;", + "}", + "", + "export const afterExpanded = 3;", + "export const last = 4;", + ].join("\r\n"); + await writeSource(relative, content); + await indexer!.index(); + const narrow = (await indexer!.search(symbol, 10, { contextLines: 0 })) + .find((result) => result.filePath === path.join(root, relative) && result.name === symbol); + const expanded = (await indexer!.search(symbol, 10, { contextLines: 2 })) + .find((result) => result.filePath === path.join(root, relative) && result.name === symbol); + expect(narrow).toMatchObject({ startLine: 5, endLine: 9 }); + expect(expanded).toMatchObject({ startLine: 3, endLine: 11 }); + expect(expanded!.content).toBe(content.split("\n").slice(2, 11).join("\n")); + + const packed = buildContextPack([{ ...narrow!, score: 1 }, { ...expanded!, score: 0.9 }], { tokenBudget: 1200 }); + expect(packed.results.map(({ startLine, endLine }) => [startLine, endLine])).toEqual([[5, 9], [3, 4]]); + expect(packed.results[1].filePath).toBe(path.join(root, relative)); + expect(packed.text).toContain(`${path.basename(relative)}:3-4`); + }); + it("keeps terminal LF delimiters outside expanded source coordinates", async () => { + const relative = "src/terminal-delimiter.ts"; + const firstSymbol = "terminalDelimiterFirst"; + const lastSymbol = "terminalDelimiterLast"; + const content = [ + `export function ${firstSymbol}() {`, + " return 1;", + "}", + "// gap line 4", + "// gap line 5", + "// gap line 6", + "// gap line 7", + "// gap line 8", + "// gap line 9", + `export function ${lastSymbol}() {`, + " return 2;", + "}", + "", + ].join("\n"); + await writeSource(relative, content); + await indexer!.index(); + + const firstResults = await indexer!.search(firstSymbol, 10, { contextLines: 6 }); + const lastResults = await indexer!.search(lastSymbol, 10, { contextLines: 6 }); + const first = firstResults.find((result) => result.filePath === path.join(root, relative) && result.name === firstSymbol); + const last = lastResults.find((result) => result.filePath === path.join(root, relative) && result.name === lastSymbol); + expect(first).toMatchObject({ startLine: 1, endLine: 9 }); + expect(last).toMatchObject({ startLine: 1, endLine: 12 }); + expect(first!.content).toBe(content.split("\n").slice(0, 9).join("\n")); + expect(last!.content).toBe(content); + + const packed = buildContextPack([{ ...first!, score: 1 }, { ...last!, score: 0.9 }], { tokenBudget: 1200 }); + expect(packed.results.map(({ startLine, endLine }) => [startLine, endLine])).toEqual([[1, 9], [10, 12]]); + expect(packed.results[1]!.content).toBe("export function terminalDelimiterLast() {\n return 2;\n}"); + expect(packed.text).toContain(`${path.basename(relative)}:10-12`); + }); + it.each(BUDGETS)("retains only fixture citations in %i-token metadata context", async (tokenBudget) => { const response = await resolveCodebaseContext(root, "opencode", { query: TARGET, symbol: TARGET, directory: TARGET_DIRECTORY, tokenBudget, diff --git a/tests/tools-utils.test.ts b/tests/tools-utils.test.ts index e77bc640..40bf789d 100644 --- a/tests/tools-utils.test.ts +++ b/tests/tools-utils.test.ts @@ -717,55 +717,108 @@ describe("tools utils", () => { expect(tooSmall.tokenEstimate).toBeLessThanOrEqual(MIN_CONTEXT_PACK_TOKEN_BUDGET); }); - it("deduplicates exact and overlapping same-file hits", () => { - const results: SearchResult[] = [ - { - filePath: "src/foo.ts", - startLine: 10, - endLine: 30, - content: "a", - score: 0.90, - chunkType: "function", - name: "outer", - }, - { - filePath: "src/foo.ts", - startLine: 12, - endLine: 18, - content: "b", - score: 0.95, - chunkType: "function", - name: "inner", - }, - { - filePath: "src/foo.ts", - startLine: 40, - endLine: 45, - content: "c", - score: 0.85, - chunkType: "function", - name: "later", - }, - { - filePath: "src/foo.ts", - startLine: 10, - endLine: 30, - content: "d", - score: 0.40, - chunkType: "function", - name: "exactDup", - }, - ]; + function rangeHit(startLine: number, endLine: number, score: number): SearchResult { + return { + filePath: "src/flow.ts", + startLine, + endLine, + content: Array.from({ length: endLine - startLine + 1 }, (_, index) => `line ${startLine + index}`).join("\n"), + score, + chunkType: "function", + name: "flow", + }; + } - const packed = buildContextPack(results, { tokenBudget: 2048 }); - expect(packed.results).toHaveLength(2); - expect(packed.candidateCount).toBe(4); - expect(packed.deduplicatedCount).toBe(2); - expect(packed.duplicateCount).toBe(2); - expect(packed.omittedCount).toBe(2); - const names = packed.results.map((r) => r.name); - expect(names).toEqual(["inner", "later"]); - expect(names).not.toContain("overlap"); + it("retains novel source lines from a partially overlapping hit without claiming a full declaration", () => { + const first = rangeHit(2, 5, 1); + const overlapping = { + ...rangeHit(4, 9, 0.9), + startCol: 2, + endCol: 7, + content: "line 4\r\nline 5\r\né雪🙂\r\nline 7\r\nline 8\r\nline 9", + }; + const packed = buildContextPack([first, overlapping, rangeHit(3, 4, 0.8)], { tokenBudget: 2048 }); + + expect(packed.results.map(({ startLine, endLine, content }) => ({ startLine, endLine, content }))).toEqual([ + { startLine: 2, endLine: 5, content: first.content }, + { startLine: 6, endLine: 9, content: "é雪🙂\r\nline 7\r\nline 8\r\nline 9" }, + ]); + expect(packed.results[1].startCol).toBeUndefined(); + expect(packed.results[1].endCol).toBeUndefined(); + expect(packed.text).toContain("src/flow.ts:6-9"); + expect(packed.duplicateCount).toBe(1); + expect(packed.tokenEstimate).toBe(countContextTokens(packed.text)); + expect(packed.tokenEstimate).toBeLessThanOrEqual(packed.tokenBudget); + expect(overlapping).toMatchObject({ startLine: 4, endLine: 9, startCol: 2, endCol: 7 }); + expect(overlapping.content).toBe("line 4\r\nline 5\r\né雪🙂\r\nline 7\r\nline 8\r\nline 9"); + }); + it("keeps aligned fragments when the source body ends with a newline", () => { + const packed = buildContextPack([ + { ...rangeHit(1, 2, 1), content: "line 1\nline 2\n" }, + { ...rangeHit(2, 3, 0.9), content: "line 2\nline 3\n" }, + ], { tokenBudget: 2048 }); + + expect(packed.results.map(({ startLine, endLine, content }) => [startLine, endLine, content])) + .toEqual([[1, 2, "line 1\nline 2\n"], [3, 3, "line 3"]]); + }); + + + it("keeps the longest uncovered segment and resolves equal-length gaps by earliest source line", () => { + const packed = buildContextPack([ + rangeHit(5, 7, 1), + rangeHit(11, 13, 0.9), + rangeHit(2, 16, 0.8), + rangeHit(5, 13, 0.7), + rangeHit(2, 13, 0.6), + ], { tokenBudget: 2048 }); + + expect(packed.results.map(({ startLine, endLine, content }) => [startLine, endLine, content])).toEqual([ + [5, 7, "line 5\nline 6\nline 7"], + [11, 13, "line 11\nline 12\nline 13"], + [2, 4, "line 2\nline 3\nline 4"], + [8, 10, "line 8\nline 9\nline 10"], + ]); + expect(packed.duplicateCount).toBe(1); + }); + + it("retains a longer suffix rather than a shorter uncovered prefix", () => { + const packed = buildContextPack([rangeHit(5, 7, 1), rangeHit(2, 12, 0.9)], { tokenBudget: 2048 }); + expect(packed.results.map(({ startLine, endLine }) => [startLine, endLine])).toEqual([[5, 7], [8, 12]]); + expect(packed.results[1].content).toBe("line 8\nline 9\nline 10\nline 11\nline 12"); + }); + + it("keeps metadata-only fragments and withholds content that cannot be mapped to the narrowed range", () => { + for (const content of ["", "unavailable source"]) { + const packed = buildContextPack([ + rangeHit(10, 15, 1), + { ...rangeHit(13, 20, 0.9), content }, + ], { tokenBudget: 2048 }); + expect(packed.results[1]).toMatchObject({ startLine: 16, endLine: 20, content: "" }); + } + }); + + it("preserves exact definition ranges, columns and lookup order when input order is authoritative", () => { + const exact = { ...rangeHit(5, 7, 0.1), startCol: 3, endCol: 8 }; + const packed = buildContextPack([exact, rangeHit(2, 12, 1)], { + tokenBudget: 2048, + preserveInputOrder: true, + }); + expect(packed.results).toEqual([exact]); + expect(packed.results[0]).toMatchObject({ startLine: 5, endLine: 7, startCol: 3, endCol: 8 }); + }); + + it("does not narrow page-local PDF ranges or deduplicate evidence from different pages", () => { + const page = (startLine: number, endLine: number, score: number, pageNumber: number): SearchResult => ({ + ...rangeHit(startLine, endLine, score), + filePath: "knowledge/guide.pdf", + documentLocation: { kind: "pdf", pageStart: pageNumber, pageEnd: pageNumber }, + }); + const packed = buildContextPack([page(1, 5, 1, 1), page(3, 8, 0.9, 1), page(1, 5, 0.8, 2)], { + tokenBudget: 2048, + }); + expect(packed.results.map(({ startLine, endLine, documentLocation }) => [documentLocation?.pageStart, startLine, endLine])) + .toEqual([[1, 1, 5], [2, 1, 5]]); + expect(packed.results[0].content).toBe("line 1\nline 2\nline 3\nline 4\nline 5"); }); it("keeps identical local ranges on different PDF pages", () => {