From 2c2219289bbe549adb973471d8ba9c6c9cc87799 Mon Sep 17 00:00:00 2001 From: Helweg Date: Thu, 29 Oct 2026 13:44:49 +0100 Subject: [PATCH 1/6] fix: restore identifier retrieval and safely refresh keyword caches --- CHANGELOG.md | 4 + docs/configuration.md | 2 + docs/tools.md | 10 + native/src/inverted_index.rs | 171 +++++++++++++----- native/src/lib.rs | 27 ++- src/indexer/definition-ranking.ts | 15 +- src/indexer/index.ts | 43 ++++- src/indexer/intent-aware-ranking.ts | 5 +- src/native/inverted-index.ts | 8 + src/watcher/file-watcher.ts | 15 +- tests/definition-ranking.test.ts | 41 +++-- tests/include-excluded-paths-indexing.test.ts | 23 +-- tests/intent-aware-source-paths.test.ts | 17 ++ tests/inverted-index.test.ts | 69 +++++++ tests/keyword-tokenizer-upgrade.test.ts | 152 ++++++++++++++++ ...tcher-selected-config-dependencies.test.ts | 62 ++++++- 16 files changed, 549 insertions(+), 115 deletions(-) create mode 100644 tests/keyword-tokenizer-upgrade.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9877f34f..2e1948d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Identifier-word retrieval**: Match natural-language terms inside camelCase, PascalCase, and acronym-bearing identifiers in BM25, and preserve word boundaries in lexical definition fallback without colliding with lowercased cache entries. Keep whole-identifier queries distinct from searches for shared identifier words. +- **Keyword cache upgrades**: Keep older keyword snapshots readable and expose `keyword-index-outdated` through explicit freshness checks. Ordinary indexing refreshes affected active-branch tokenization without regenerating unchanged embeddings; structural caches rebuild from persisted source. Track partially refreshed branch chunks across snapshot reloads. Reject unsupported newer keyword formats without deleting their snapshots or resetting catalog/vector data. +- **Code-request source preference**: Prefer implementation paths when a search explicitly asks for code, while preserving explicit test, documentation, and configuration intent. +- **Initial module-config watching**: Register startup-discovered local module configuration dependencies as explicit Chokidar targets, preserving transitive alias-graph updates when an initial recursive directory notification is omitted, including when an explicit host configuration path is supplied. - **Consumer-visible evidence correctness**: Count only retained source and graph rows in pre-edit details, close truncated source fences at whole lines, and reserve outer-heading/final-truncation overhead before source fitting. Credit source only when its complete fitted block survives final composition. Allow structural-mode architecture maps to read branch files from SQLite without a vector store. Add native-backed citation, Unicode/CRLF, scope, and token-budget acceptance. - **Pre-edit source identity and cancellation**: Resolve canonical native symbol IDs before fitting target source, including large fragmented declarations, instead of accepting ranked body slices, suffix paths, or enclosing declarations. Cap source blocks at 512 tokens, withhold missing source, preserve default host roots, and share the exact lookup with evaluation. Keep selection-hook test references metadata-only and honor cancellation after empty lookups. - **Working-tree snapshot integrity**: Detect concurrent database publication even when source hashes and catalog counts are unchanged. Reject lossy UTF-8 Git paths and retain only project-local endpoints for cross-boundary renames. diff --git a/docs/configuration.md b/docs/configuration.md index 41c4f8c7..d6d8a87a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -311,6 +311,8 @@ Unlimited depth retains hidden/build/ignored-path exclusions, does not follow sy When file watching uses Chokidar, an identified Node `watch` registration failure (`EPERM` or `EACCES`) for a readable, selected source file can trigger the existing one-shot polling recovery, just like open-file exhaustion. The error must name an absolute path inside the project, and the file and all its ancestors must remain eligible under the watcher traversal, exclusion, Git-ignore, and protected-path policy. Readiness follows the replacement watcher, with policy-filtered catch-up for changes during replacement. Polling is not the default; unreadable, restricted, unselected, configuration-only, directory, and unidentified or relative error targets do not enable this recovery. +Chokidar also receives explicit targets for local module configuration dependencies discovered at startup, including transitive `tsconfig` dependencies. This retains its independent first-change coverage when macOS omits an initial recursive directory notification. Existing selection, exclusion, and protected-path policies still apply; configuration notifications do not broaden which files are indexed. + For the disabled-by-default compiler-index pilot, see [Optional SCIP TypeScript enrichment](scip-typescript.md). Python call graphs support conservative local relative imports such as `from .formatting import format_payment` and explicit aliases. The importer and target must be indexed, package paths must have indexed `__init__.py` files, and the target must be a unique top-level function. Absolute/namespace imports, multiline imports, decorated targets, ambiguous modules, shadowing and detected dynamic rebinding are not resolved by this extension. Complex string interpolation also causes conservative abstention. This is static source analysis, not execution of Python's runtime import machinery. Normal indexing updates older graph metadata and refreshes affected Python graph sources in both indexing modes. diff --git a/docs/tools.md b/docs/tools.md index 586e318c..a5f5a721 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -170,6 +170,12 @@ Do not repeat context calls or broad reads when existing evidence already answer ## Core retrieval tools +Keyword retrieval recognizes identifier word boundaries: for example, `response` and `payload` can match `readResponsePayload`, while the complete identifier remains searchable. Lexical definition fallback also recognizes camelCase and snake_case name components. Older keyword snapshots need the ordinary refresh described under `index_status` before they gain these matches. + +Identifier-word expansion applies to indexed code, not query terms: a whole camelCase query does not match an unrelated identifier merely because they share one component. + +Searches that explicitly ask for code prefer implementation paths. Explicit requests for tests, documentation, or configuration still take precedence. + ### `codebase_edit_context` Use for a known or suspected edit target. `query`, optional `symbol`/`filePath`, caller/callee limits, and a 128–4000-token budget produce target source and direct graph evidence, or a risk-marked conceptual fallback. `filePath` disambiguates the target; it does not restrict related test locations to the implementation directory. Details count only source and graph evidence retained in the response. @@ -262,6 +268,10 @@ Pass `checkFreshness: true` to compare the selected source/configuration hashes Catalog presence and provider/model compatibility remain separate from pending storage migrations. An active catalog can be `ready` and compatible while parser, symbol, or call-graph migration markers are outdated. `migrationRequired` and a noncurrent freshness reason recommend ordinary `index_codebase` with `force: false`, not index deletion. Resolve unreadable source/index access before indexing. The CLI equivalents are `ocbi status --check-freshness` and `ocbi status --check-freshness --json` (`cbi` is equivalent). JSON contains the shared status object; MCP adds the diagnostics below, while Pi/omp return the same facts as details. Workspace status remains a bounded catalog inspection with freshness explicitly not checked. +An older keyword snapshot remains readable. With freshness checking, `keyword-index-outdated` means the active branch needs the current identifier-word tokenizer. Run ordinary `index_codebase` or `ocbi index`; unchanged embeddings are reused, without deleting the index or forcing re-embedding. Read-only status checks never rewrite the cache. Other hybrid branches' pending tokenizer state is not counted against the refreshed active branch. + +A cache written with a newer, unsupported tokenizer version is not a corrupt cache to reset. Ordinary indexing rejects it without deleting or rewriting its keyword snapshot, catalog, or vectors. Use a compatible package version. + MCP responses add `structuredContent.mcpDiagnostics` with schema version `1`. `activeOperations` reports each operation, its current phase, start and last-activity timestamps, and `active` or `suspected_stall` status. The current `index_status` call is excluded. `latestInterruptedOperation` is included only after an ordered shutdown marks an active call or a later local process confirms that the recorded PID is absent. A stale heartbeat alone is never classified as an interruption. ### `index_codebase` diff --git a/native/src/inverted_index.rs b/native/src/inverted_index.rs index f79e0742..c24cef47 100644 --- a/native/src/inverted_index.rs +++ b/native/src/inverted_index.rs @@ -5,11 +5,32 @@ use std::fs; use std::io::{BufWriter, Write}; use std::path::{Path, PathBuf}; +const TOKENIZER_VERSION: u32 = 1; + +#[derive(Debug)] +pub struct UnsupportedTokenizerVersionError(u32); + +impl std::fmt::Display for UnsupportedTokenizerVersionError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "Keyword index uses an unsupported tokenizer version ({}, supported: {})", + self.0, TOKENIZER_VERSION + ) + } +} + +impl std::error::Error for UnsupportedTokenizerVersionError {} + #[derive(Serialize, Deserialize, Default)] struct InvertedIndexData { term_to_chunks: HashMap>, chunk_tokens: HashMap>, avg_doc_length: f64, + #[serde(default)] + tokenizer_version: u32, + #[serde(default)] + legacy_chunks: HashSet, } #[derive(Serialize)] @@ -17,6 +38,8 @@ struct InvertedIndexSnapshot<'a> { term_to_chunks: &'a HashMap>, chunk_tokens: &'a HashMap>, avg_doc_length: f64, + tokenizer_version: u32, + legacy_chunks: &'a HashSet, } pub struct InvertedIndexInner { @@ -24,6 +47,7 @@ pub struct InvertedIndexInner { term_to_chunks: HashMap>, chunk_tokens: HashMap>, total_token_count: u64, + legacy_chunks: HashSet, } impl InvertedIndexInner { @@ -33,6 +57,7 @@ impl InvertedIndexInner { term_to_chunks: HashMap::new(), chunk_tokens: HashMap::new(), total_token_count: 0, + legacy_chunks: HashSet::new(), } } @@ -42,24 +67,7 @@ impl InvertedIndexInner { } let content = fs::read_to_string(&self.index_path)?; - let data: InvertedIndexData = serde_json::from_str(&content)?; - - self.term_to_chunks.clear(); - for (term, chunk_ids) in data.term_to_chunks { - self.term_to_chunks - .insert(term, chunk_ids.into_iter().collect()); - } - - self.chunk_tokens.clear(); - self.total_token_count = 0; - for (chunk_id, tokens) in data.chunk_tokens { - for count in tokens.values() { - self.total_token_count += *count as u64; - } - self.chunk_tokens.insert(chunk_id, tokens); - } - - Ok(()) + self.restore(serde_json::from_str(&content)?) } pub fn save(&self) -> Result<()> { @@ -87,42 +95,82 @@ impl InvertedIndexInner { term_to_chunks: &self.term_to_chunks, chunk_tokens: &self.chunk_tokens, avg_doc_length: self.get_avg_doc_length(), + tokenizer_version: TOKENIZER_VERSION, + legacy_chunks: &self.legacy_chunks, } } pub fn deserialize(&mut self, json: &str) -> Result<()> { - let data: InvertedIndexData = serde_json::from_str(json)?; + self.restore(serde_json::from_str(json)?) + } + + fn restore(&mut self, data: InvertedIndexData) -> Result<()> { + if data.tokenizer_version > TOKENIZER_VERSION { + return Err(UnsupportedTokenizerVersionError(data.tokenizer_version).into()); + } + let legacy_chunks = if data.tokenizer_version < TOKENIZER_VERSION { + data.chunk_tokens.keys().cloned().collect() + } else { + data.legacy_chunks + }; self.term_to_chunks.clear(); for (term, chunk_ids) in data.term_to_chunks { self.term_to_chunks .insert(term, chunk_ids.into_iter().collect()); } - self.chunk_tokens.clear(); - self.total_token_count = 0; - for (chunk_id, tokens) in data.chunk_tokens { - for count in tokens.values() { - self.total_token_count += *count as u64; - } - self.chunk_tokens.insert(chunk_id, tokens); - } + self.total_token_count = data + .chunk_tokens + .values() + .flat_map(|tokens| tokens.values()) + .map(|count| *count as u64) + .sum(); + self.chunk_tokens = data.chunk_tokens; + self.legacy_chunks = legacy_chunks; Ok(()) } + pub fn has_current_tokenizer(&self, chunk_ids: Option<&[String]>) -> bool { + if self.legacy_chunks.is_empty() { + return true; + } + match chunk_ids { + Some(ids) => ids.iter().all(|id| !self.legacy_chunks.contains(id)), + None => false, + } + } + + pub fn outdated_tokenizer_chunk_ids(&self, chunk_ids: Option<&[String]>) -> Vec { + let mut ids: Vec = match chunk_ids { + Some(ids) => ids + .iter() + .filter(|id| self.legacy_chunks.contains(*id)) + .cloned() + .collect(), + None => self.legacy_chunks.iter().cloned().collect(), + }; + ids.sort_unstable(); + ids.dedup(); + ids + } + pub fn add_chunk(&mut self, chunk_id: &str, content: &str) { let tokens = self.tokenize(content); + let token_count = tokens.len() as u64; let mut term_freq: HashMap = HashMap::new(); - - for token in &tokens { - *term_freq.entry(token.clone()).or_insert(0) += 1; - - self.term_to_chunks - .entry(token.clone()) - .or_default() - .insert(chunk_id.to_string()); + for token in tokens { + *term_freq.entry(token).or_insert(0) += 1; } - + for token in term_freq.keys() { + if let Some(chunks) = self.term_to_chunks.get_mut(token) { + chunks.insert(chunk_id.to_string()); + } else { + self.term_to_chunks + .insert(token.clone(), HashSet::from([chunk_id.to_string()])); + } + } + self.legacy_chunks.remove(chunk_id); self.chunk_tokens.insert(chunk_id.to_string(), term_freq); - self.total_token_count += tokens.len() as u64; + self.total_token_count += token_count; } pub fn remove_chunk(&mut self, chunk_id: &str) -> bool { @@ -130,6 +178,7 @@ impl InvertedIndexInner { Some(t) => t, None => return false, }; + self.legacy_chunks.remove(chunk_id); for (token, count) in &tokens { self.total_token_count = self.total_token_count.saturating_sub(*count as u64); @@ -146,7 +195,7 @@ impl InvertedIndexInner { } pub fn search(&self, query: &str) -> Vec<(String, f64)> { - let query_tokens = self.tokenize(query); + let query_tokens: Vec = Self::words(query).map(str::to_lowercase).collect(); if query_tokens.is_empty() { return Vec::new(); } @@ -216,6 +265,7 @@ impl InvertedIndexInner { self.term_to_chunks.clear(); self.chunk_tokens.clear(); self.total_token_count = 0; + self.legacy_chunks.clear(); } pub fn document_count(&self) -> usize { @@ -231,15 +281,44 @@ impl InvertedIndexInner { } } + fn words(text: &str) -> impl Iterator { + text.split(|character: char| !character.is_alphanumeric()) + .filter(|word| word.len() > 2) + } + fn tokenize(&self, text: &str) -> Vec { - text.to_lowercase() - .chars() - .map(|c| if c.is_alphanumeric() { c } else { ' ' }) - .collect::() - .split_whitespace() - .filter(|t| t.len() > 2) - .map(|s| s.to_string()) - .collect() + let mut tokens = Vec::new(); + for word in Self::words(text) { + tokens.push(word.to_lowercase()); + let mut start = 0; + let mut previous: Option = None; + let mut characters = word.char_indices().peekable(); + while let Some((offset, character)) = characters.next() { + let boundary = offset > 0 + && character.is_uppercase() + && previous.is_some_and(|previous| { + previous.is_lowercase() + || previous.is_numeric() + || (previous.is_uppercase() + && characters.peek().is_some_and(|(_, next)| next.is_lowercase())) + }); + if boundary { + let part = &word[start..offset]; + if part.len() > 2 { + tokens.push(part.to_lowercase()); + } + start = offset; + } + previous = Some(character); + } + if start > 0 { + let part = &word[start..]; + if part.len() > 2 { + tokens.push(part.to_lowercase()); + } + } + } + tokens } } diff --git a/native/src/lib.rs b/native/src/lib.rs index 3603da9a..77281ccf 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -468,6 +468,15 @@ pub struct KeywordSearchResult { pub score: f64, } +fn keyword_index_error(error: anyhow::Error) -> Error { + let status = if error.is::() { + napi::Status::InvalidArg + } else { + napi::Status::GenericFailure + }; + Error::new(status, error.to_string()) +} + #[napi] pub struct InvertedIndex { inner: inverted_index::InvertedIndexInner, @@ -483,9 +492,7 @@ impl InvertedIndex { #[napi] pub fn load(&mut self) -> Result<()> { - self.inner - .load() - .map_err(|e| Error::from_reason(e.to_string())) + self.inner.load().map_err(keyword_index_error) } #[napi] @@ -523,6 +530,16 @@ impl InvertedIndex { self.inner.has_chunk(&chunk_id) } + #[napi] + pub fn has_current_tokenizer(&self, chunk_ids: Option>) -> bool { + self.inner.has_current_tokenizer(chunk_ids.as_deref()) + } + + #[napi] + pub fn outdated_tokenizer_chunk_ids(&self, chunk_ids: Option>) -> Vec { + self.inner.outdated_tokenizer_chunk_ids(chunk_ids.as_deref()) + } + #[napi] pub fn clear(&mut self) { self.inner.clear(); @@ -542,8 +559,6 @@ impl InvertedIndex { #[napi] pub fn deserialize(&mut self, json: String) -> Result<()> { - self.inner - .deserialize(&json) - .map_err(|e| Error::from_reason(e.to_string())) + self.inner.deserialize(&json).map_err(keyword_index_error) } } diff --git a/src/indexer/definition-ranking.ts b/src/indexer/definition-ranking.ts index 9427c520..e0421a72 100644 --- a/src/indexer/definition-ranking.ts +++ b/src/indexer/definition-ranking.ts @@ -2,6 +2,7 @@ import { analyzeQueryIntent, extractFilePathHint, extractIntentIdentifierHints, + identifierTokens, isConfigPath, isDocumentationPath as isIntentDocumentationPath, isFixturePath, @@ -16,7 +17,6 @@ export type ExternalRerankBand = "implementation" | "documentation" | "test" | " const RANKING_TOKEN_CACHE_LIMIT = 4096; -const rankingQueryTokenCache = new Map>(); const rankingPathTokenCache = new Map>(); const rankingTextTokenCache = new Map>(); @@ -47,21 +47,24 @@ export function tokenizeTextForRanking(text: string): Set { return new Set(); } - const lowered = normalizeRankingText(text); - const cache = rankingQueryTokenCache.get(lowered) ?? rankingTextTokenCache.get(lowered); + const normalized = text.normalize("NFKC"); + const cache = rankingTextTokenCache.get(normalized); if (cache) { return cache; } const tokens = new Set( - lowered + normalized.toLowerCase() .replace(/[^\p{L}\p{N}_$\s]/gu, " ") .split(/\s+/) .filter((token) => token.length > 1 && !STOPWORDS.has(token)) ); - setBoundedCache(rankingQueryTokenCache, lowered, tokens); - setBoundedCache(rankingTextTokenCache, lowered, tokens); + for (const token of identifierTokens(normalized)) { + if (!STOPWORDS.has(token)) tokens.add(token); + } + + setBoundedCache(rankingTextTokenCache, normalized, tokens); return tokens; } diff --git a/src/indexer/index.ts b/src/indexer/index.ts index c924b48f..b747b8cc 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -578,7 +578,8 @@ export type IndexFreshnessReason = | "files-changed" | "metadata-changed" | "branch-changed" - | "migration-required"; + | "migration-required" + | "keyword-index-outdated"; export interface IndexFreshnessResult { readable: boolean; @@ -1121,7 +1122,7 @@ export function buildSymbolDefinitionLane( const pathLower = candidate.metadata.filePath.toLowerCase(); const exactHintMatch = normalizedHints.some((hint) => nameLower === hint || nameLower.replace(/_/g, "") === hint.replace(/_/g, "")); - const tokenizedName = tokenizeTextForRanking(nameLower); + const tokenizedName = tokenizeTextForRanking(candidate.metadata.name ?? ""); const tokenHits = codeTermHints.filter((term) => tokenizedName.has(term) || pathLower.includes(term)).length; if (!exactHintMatch && tokenHits === 0) { @@ -1314,6 +1315,11 @@ function unionCandidates( return Array.from(byId.values()); } +function isUnsupportedKeywordIndexVersion(error: unknown): boolean { + // The native keyword loader reserves InvalidArg for incompatible tokenizer versions. + return error instanceof Error && "code" in error && error.code === "InvalidArg"; +} + export class Indexer { private readonly host: HostMode; private config: ParsedCodebaseIndexConfig; @@ -1758,6 +1764,13 @@ export class Indexer { } } + private needsKeywordTokenizerUpgrade(invertedIndex: InvertedIndex, database: Database): boolean { + if (invertedIndex.hasCurrentTokenizer()) return false; + return this.getBranchCatalogKeys().some((branchKey) => + !invertedIndex.hasCurrentTokenizer(database.getBranchChunkIds(branchKey)) + ); + } + private getScopedRoots(projectRoot = this.projectRoot): string[] { const roots = new Set([this.getCanonicalPath(projectRoot)]); @@ -3887,7 +3900,8 @@ export class Indexer { this.invertedIndex = new InvertedIndex(invertedIndexPath); try { this.invertedIndex.load(); - } catch { + } catch (error) { + if (isUnsupportedKeywordIndexVersion(error)) throw error; if (existsSync(invertedIndexPath)) { await fsPromises.unlink(invertedIndexPath); } @@ -4031,6 +4045,7 @@ export class Indexer { } this.invertedIndex.load(); } catch (error) { + if (isUnsupportedKeywordIndexVersion(error)) throw error; this.structuralKeywordRebuildRequired = existsSync(invertedIndexPath); this.logger.warn("Structural keyword index is unreadable; rebuilding it from source files", { error: getErrorMessage(error), @@ -4050,6 +4065,9 @@ export class Indexer { if (storedGeneration !== publishedGeneration && (this.database.getStats()?.chunkCount ?? 0) > 0) { this.structuralKeywordRebuildRequired = true; } + if (!this.invertedIndex.hasCurrentTokenizer()) { + this.structuralKeywordRebuildRequired = true; + } if (this.structuralKeywordRebuildRequired) { this.logger.info("Rebuilding structural keyword index from the persisted chunk catalog", { chunks: this.database.getStats()?.chunkCount ?? 0, @@ -4980,6 +4998,14 @@ export class Indexer { this.loadFileHashCache(); const rebuildStructuralKeyword = isStructural && this.structuralKeywordRebuildRequired; + let keywordTokenizerUpgradeChunks: Set | undefined; + if (!invertedIndex.hasCurrentTokenizer()) { + const activeChunkIds = this.getBranchCatalogKeys().flatMap((key) => database.getBranchChunkIds(key)); + const outdated = invertedIndex.outdatedTokenizerChunkIds(activeChunkIds); + if (outdated.length > 0) keywordTokenizerUpgradeChunks = new Set(outdated); + } + const upgradeKeywordTokenizer = keywordTokenizerUpgradeChunks !== undefined; + let keywordTokenizerUpdated = false; const javaScriptParserMetadataKey = this.getJavaScriptParserVersionMetadataKey(); const reparseCachedJavaScriptFiles = database.getMetadata(javaScriptParserMetadataKey) !== JAVASCRIPT_PARSER_VERSION; @@ -5186,6 +5212,7 @@ export class Indexer { && !requiresKotlinParserUpgrade && !requiresRubyParserUpgrade && !refreshCachedSymbols + && !upgradeKeywordTokenizer ) { unchangedFilePaths.add(storedPath); } else { @@ -5671,6 +5698,11 @@ export class Indexer { }); if (existingContentHash === contentHash && !rebuildStructuralKeyword) { + if (keywordTokenizerUpgradeChunks?.has(id)) { + invertedIndex.removeChunk(id); + invertedIndex.addChunk(id, chunk.content); + keywordTokenizerUpdated = true; + } continue; } @@ -6048,7 +6080,7 @@ export class Indexer { if (backfilledBlameMetadata || shouldFingerprintLegacyPair || removedStoredChunks) { if (!isStructural) requireLoadedCapability(store, "vector store").save(); } - if (removedStoredChunks && !isStructural) { + if ((removedStoredChunks || keywordTokenizerUpdated) && !isStructural) { this.saveInvertedIndex(invertedIndex); } database.setMetadata(javaScriptParserMetadataKey, JAVASCRIPT_PARSER_VERSION); @@ -7097,6 +7129,9 @@ export class Indexer { return { readable: true, current: false, reason: "branch-changed" }; } } + if (this.needsKeywordTokenizerUpgrade(requireLoadedCapability(this.invertedIndex, "keyword index"), database)) { + return { readable: true, current: false, reason: "keyword-index-outdated" }; + } return { readable: true, current: true, reason: "current" }; } diff --git a/src/indexer/intent-aware-ranking.ts b/src/indexer/intent-aware-ranking.ts index 023414a7..5d889564 100644 --- a/src/indexer/intent-aware-ranking.ts +++ b/src/indexer/intent-aware-ranking.ts @@ -143,7 +143,7 @@ function compactIdentifier(value: string): string { return normalizeRankingText(value).replace(/[^\p{L}\p{N}$]+/gu, ""); } -function identifierTokens(value: string): string[] { +export function identifierTokens(value: string): string[] { const normalized = value .normalize("NFKC") .replace(/([\p{Ll}\p{N}])([\p{Lu}])/gu, "$1 $2") @@ -233,7 +233,8 @@ export function analyzeQueryIntent(query: string): QueryIntentProfile { const explicitArtifactIntent = primary === "test" || primary === "docs" || primary === "config" || primary === "call-flow"; const preferSourcePaths = primary === "definition" || primary === "implementation" || primary === "call-flow" || - (primary === "neutral" && identifierHints.length > 0); + (primary === "neutral" && identifierHints.length > 0) || + (!explicitArtifactIntent && /\bcode\b/u.test(normalized)); return { primary, diff --git a/src/native/inverted-index.ts b/src/native/inverted-index.ts index ca520f20..c68ff6fa 100644 --- a/src/native/inverted-index.ts +++ b/src/native/inverted-index.ts @@ -44,6 +44,14 @@ export class InvertedIndex { return this.inner.hasChunk(chunkId); } + hasCurrentTokenizer(chunkIds?: string[]): boolean { + return this.inner.hasCurrentTokenizer(chunkIds); + } + + outdatedTokenizerChunkIds(chunkIds?: string[]): string[] { + return this.inner.outdatedTokenizerChunkIds(chunkIds); + } + clear(): void { this.inner.clear(); } diff --git a/src/watcher/file-watcher.ts b/src/watcher/file-watcher.ts index 5cb22009..3f5276ff 100644 --- a/src/watcher/file-watcher.ts +++ b/src/watcher/file-watcher.ts @@ -484,17 +484,10 @@ export class FileWatcher { return true; } - private getFullChokidarWatchTargets(): string | string[] { - if (this.configPath) { - return [this.projectRoot, this.configPath]; - } - - const externalConfigTargets = this.getExternalConfigWatchTargets(); - if (externalConfigTargets.length === 0) { - return this.projectRoot; - } - - return [this.projectRoot, ...externalConfigTargets]; + private getFullChokidarWatchTargets(): string[] { + const configTargets = this.configPath ? [this.configPath] : this.getExternalConfigWatchTargets(); + // Explicit config roots retain Chokidar's first-event coverage when recursive callbacks are omitted. + return [this.projectRoot, ...configTargets, ...this.localModuleConfigTracker.getPaths()]; } private getExternalConfigWatchTargets(): string[] { diff --git a/tests/definition-ranking.test.ts b/tests/definition-ranking.test.ts index a3f2652e..8fc2c40f 100644 --- a/tests/definition-ranking.test.ts +++ b/tests/definition-ranking.test.ts @@ -10,11 +10,11 @@ import { isImplementationChunkType, normalizeIdentifierVariants, pathMatchesHint, - splitPathTokens, tokenizeTextForRanking, } from "../src/indexer/definition-ranking.js"; -import { Indexer } from "../src/indexer/index.js"; +import { buildSymbolDefinitionLane, Indexer } from "../src/indexer/index.js"; import { parseConfig } from "../src/config/schema.js"; +import { Database } from "../src/native/index.js"; import { resolveSearchContext } from "../src/tools/context-search.js"; import type { RankedCandidate } from "../src/indexer/search-ranking.js"; @@ -73,18 +73,31 @@ describe("definition ranking helpers", () => { ]); }); - it("tokenizes ranking text and paths without generic stopwords", () => { - expect(Array.from(tokenizeTextForRanking("Find the PaymentValidator implementation"))).toEqual([ - "paymentvalidator", - ]); - expect(Array.from(splitPathTokens("src/services/payment-validator.ts"))).toEqual([ - "src", - "services", - "payment", - "validator", - "ts", - ]); - }); + it.each(["persistFailedBatches", "persist_failed_batches"])( + "ranks identifier word evidence above a partial-name match (%s)", + (name) => { + const database = Database.createEmptyReadOnly(); + const candidates = [ + candidate("partial", "src/entry.ts", "persistFailednessBatchwork", "function_declaration", 0.2), + candidate("target", "src/state.ts", name, "function_declaration", 0.2), + ]; + try { + tokenizeTextForRanking(name.toLowerCase()); + const ranked = buildSymbolDefinitionLane( + "persist failed batches", database, null, null, 5, candidates, true, + ); + expect(ranked.map((entry) => entry.id)).toEqual(["target", "partial"]); + expect(ranked[0].score).toBeGreaterThan(ranked[1].score); + + const exact = buildSymbolDefinitionLane( + name.toLowerCase(), database, null, null, 5, candidates, true, + ); + expect(exact.map((entry) => entry.id)).toEqual(["target"]); + } finally { + database.close(); + } + }, + ); it("shares implementation chunk classification with call-graph symbol types", () => { expect(isImplementationChunkType("trigger_declaration")).toBe(true); diff --git a/tests/include-excluded-paths-indexing.test.ts b/tests/include-excluded-paths-indexing.test.ts index ca3b892f..89feb7fd 100644 --- a/tests/include-excluded-paths-indexing.test.ts +++ b/tests/include-excluded-paths-indexing.test.ts @@ -835,10 +835,9 @@ describe("include-excluded path indexing acceptance", () => { }); it.each([ - { label: "default root build file", file: "build-helper.ts", exclude: undefined, denyAccess: false, indexed: true }, - { label: "explicitly excluded file", file: "ignored.ts", exclude: ["ignored.ts"], denyAccess: false, indexed: false }, - { label: "source denied by the access boundary", file: "blocked/blocked.ts", exclude: undefined, denyAccess: true, indexed: true }, - ])("preserves consumer filtering after a permission error for a $label", async ({ file, exclude, denyAccess, indexed }) => { + { label: "default root build file", file: "build-helper.ts", exclude: undefined, indexed: true }, + { label: "explicitly excluded file", file: "ignored.ts", exclude: ["ignored.ts"], indexed: false }, + ])("preserves consumer filtering after a permission error for a $label", async ({ file, exclude, indexed }) => { const ordinaryPath = path.join(projectDir, "src", "ordinary.ts"); const rejectedPath = path.join(projectDir, file); writeSource(ordinaryPath, "permissionFilteredOriginalOrdinaryDefinition"); @@ -847,20 +846,8 @@ describe("include-excluded path indexing acceptance", () => { const indexer = createIndexer(config); await indexer.index(); snapshotFaults.perDirectoryBackend = true; - const watchPath = denyAccess ? path.dirname(rejectedPath) : projectDir; - - const actualAccess = fs.accessSync; - let denyRecoveryAccess = denyAccess; - if (denyAccess) { - // Portable classification fault only: the fixture's real reads and file modes are unchanged. - vi.spyOn(fs, "accessSync").mockImplementation(function (this: unknown, ...args) { - if (String(args[0]) === watchPath && denyRecoveryAccess) { - denyRecoveryAccess = false; - throw Object.assign(new Error("CONTROLLED source access EACCES"), { code: "EACCES" }); - } - return Reflect.apply(actualAccess, this, args); - }); - } + const watchPath = projectDir; + const actualWatch = fs.watch; let faulted = false; vi.spyOn(fs, "watch").mockImplementation(function (this: unknown, ...args) { diff --git a/tests/intent-aware-source-paths.test.ts b/tests/intent-aware-source-paths.test.ts index e645b9bd..c60f6ee5 100644 --- a/tests/intent-aware-source-paths.test.ts +++ b/tests/intent-aware-source-paths.test.ts @@ -36,4 +36,21 @@ describe("intent-aware source path preference", () => { expect(rankIntentAwareCandidates(query, candidates, candidates.length, { prioritizeSourcePaths: true }).map(({ id }) => id)) .toEqual(["source", "test"]); }); + + it.each([ + ["find code that batches outbound notifications", "source"], + ["find tests for code that batches outbound notifications", "test"], + ["documentation for code that batches outbound notifications", "documentation"], + ["configuration for code that batches outbound notifications", "configuration"], + ])("preserves the requested artifact preference for %s", (query, expected) => { + const candidates = [ + candidate("test", 0.82, "tests/notifications.test.ts", "function"), + candidate("source", 0.55, "src/notifications/batch.ts", "function"), + candidate("documentation", 0.62, "docs/notifications.md", "section"), + candidate("configuration", 0.60, "config/notifications.json", "object"), + ]; + + expect(rankIntentAwareCandidates(query, candidates, candidates.length)[0]?.id) + .toBe(expected); + }); }); diff --git a/tests/inverted-index.test.ts b/tests/inverted-index.test.ts index 477371c8..f2e5f0e7 100644 --- a/tests/inverted-index.test.ts +++ b/tests/inverted-index.test.ts @@ -96,6 +96,75 @@ describe("InvertedIndex", () => { } } }); + + it("matches identifier words and preserves complete identifiers across reloads", () => { + index.addChunk("fusion", "function blendOrderedScores(semanticResults, keywordResults) { const semanticRank = 1; }"); + index.addChunk("transport", "class HTTPClient { readResponsePayload() {} }"); + index.addChunk("unicode", "function processÉtat() {}"); + index.save(); + const loaded = new InvertedIndex(indexPath); + loaded.load(); + for (const candidate of [index, loaded]) { + for (const [query, expected] of [ + ["semantic", "fusion"], ["keyword", "fusion"], ["rank", "fusion"], + ["BLENDORDEREDSCORES", "fusion"], ["http", "transport"], + ["response", "transport"], ["payload", "transport"], ["état", "unicode"], + ]) { + expect([...candidate.search(query).keys()]).toEqual([expected]); + } + } + }); + + it("keeps whole identifier queries distinct from identifier-word queries", () => { + index.addChunk("retained", "const durableNeedle = true;"); + index.save(); + const loaded = new InvertedIndex(indexPath); + loaded.load(); + for (const candidate of [index, loaded]) { + expect([...candidate.search("replacementNeedle").keys()]).toEqual([]); + expect([...candidate.search("durableNeedle").keys()]).toEqual(["retained"]); + expect([...candidate.search("DURABLENEEDLE").keys()]).toEqual(["retained"]); + expect([...candidate.search("needle").keys()]).toEqual(["retained"]); + } + }); + + it("tracks legacy tokenizer coverage per chunk through partial upgrades and persistence", () => { + index.deserialize(JSON.stringify({ + term_to_chunks: { persistfailedbatches: ["old-a"], loadstoredrecords: ["old-b"] }, + chunk_tokens: { "old-a": { persistfailedbatches: 1 }, "old-b": { loadstoredrecords: 1 } }, + avg_doc_length: 1, + })); + expect([...index.search("PERSISTFAILEDBATCHES").keys()]).toEqual(["old-a"]); + expect([...index.search("failed").keys()]).toEqual([]); + expect(index.hasCurrentTokenizer(["old-a"])).toBe(false); + index.addChunk("current", "currentImplementation"); + expect(index.hasCurrentTokenizer(["current"])).toBe(true); + expect(index.outdatedTokenizerChunkIds(["current", "old-b", "old-a", "old-a"])).toEqual(["old-a", "old-b"]); + + index.removeChunk("old-a"); + index.addChunk("old-a", "persistFailedBatches"); + index.save(); + const loaded = new InvertedIndex(indexPath); + loaded.load(); + expect([...loaded.search("failed").keys()]).toEqual(["old-a"]); + expect(loaded.hasCurrentTokenizer(["old-a", "current"])).toBe(true); + expect(loaded.hasCurrentTokenizer(["old-b"])).toBe(false); + expect(loaded.hasCurrentTokenizer()).toBe(false); + + const restored = new InvertedIndex(path.join(tempDir, "restored.json")); + restored.deserialize(loaded.serialize()); + expect(restored.hasCurrentTokenizer(["old-b"])).toBe(false); + restored.removeChunk("old-b"); + expect(restored.hasCurrentTokenizer()).toBe(true); + expect([...restored.search("loadstoredrecords").keys()]).toEqual([]); + }); + + it("rejects a future tokenizer version without discarding readable state", () => { + index.addChunk("retained", "retainedEvidence"); + const future = { ...JSON.parse(index.serialize()), tokenizer_version: 999 }; + expect(() => index.deserialize(JSON.stringify(future))).toThrow(); + expect([...index.search("retained").keys()]).toEqual(["retained"]); + }); }); describe("removeChunk", () => { diff --git a/tests/keyword-tokenizer-upgrade.test.ts b/tests/keyword-tokenizer-upgrade.test.ts new file mode 100644 index 00000000..0e34a696 --- /dev/null +++ b/tests/keyword-tokenizer-upgrade.test.ts @@ -0,0 +1,152 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { parseConfig } from "../src/config/schema.js"; +import { Indexer } from "../src/indexer/index.js"; +import { Database, InvertedIndex } from "../src/native/index.js"; + +describe("keyword tokenizer upgrades", () => { + it.each(["hybrid", "structural"] as const)( + "keeps a legacy %s catalog readable and upgrades normal indexing without embedding again", + async (mode) => { + const root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), "keyword-tokenizer-upgrade-"))); + fs.mkdirSync(path.join(root, "src")); + fs.writeFileSync(path.join(root, "src", "marker.ts"), "export function cacheTokenizerMarker() { return 7; }\n"); + let embeddingCalls = 0; + const fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async (_url, init) => { + embeddingCalls += 1; + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string | string[] }; + const texts = Array.isArray(body.input) ? body.input : [body.input ?? ""]; + return new Response(JSON.stringify({ + data: texts.map(() => ({ embedding: [1, 0, 0, 0, 0, 0, 0, 0] })), + usage: { total_tokens: texts.length }, + }), { status: 200 }); + }); + const config = parseConfig({ + embeddingProvider: "custom", + customProvider: { baseUrl: "http://localhost:11434/v1", model: "keyword-upgrade", dimensions: 8 }, + indexing: { mode, autoIndex: false, watchFiles: false, requireProjectMarker: false }, + include: ["src/**/*.ts"], + search: { minScore: 0 }, + }); + const first = new Indexer(root, config, "pi"); + let reader: Indexer | undefined; + try { + await first.index(); + const initial = await first.getStatus(); + await first.close(); + const keywordPath = path.join(initial.indexPath, "inverted-index.json"); + const databasePath = path.join(initial.indexPath, "codebase.db"); + const snapshot = JSON.parse(fs.readFileSync(keywordPath, "utf8")) as { + chunk_tokens: Record>; + }; + const chunkIds = Object.keys(snapshot.chunk_tokens); + const legacyTerms = ["export", "function", "cachetokenizermarker", "return"]; + fs.writeFileSync(keywordPath, JSON.stringify({ + term_to_chunks: Object.fromEntries(legacyTerms.map((term) => [term, chunkIds])), + chunk_tokens: Object.fromEntries(chunkIds.map((id) => [id, Object.fromEntries(legacyTerms.map((term) => [term, 1]))])), + avg_doc_length: legacyTerms.length, + })); + const legacyBytes = fs.readFileSync(keywordPath); + const catalogBytes = fs.readFileSync(databasePath); + const vectorPath = path.join(initial.indexPath, "vectors"); + const vectorBytes = mode === "hybrid" ? fs.readFileSync(vectorPath) : undefined; + const database = Database.openReadOnly(databasePath); + let embedding: Buffer | null = null; + let contentHash = ""; + try { + contentHash = database.getChunk(chunkIds[0])!.contentHash; + if (mode === "hybrid") embedding = database.getEmbedding(contentHash); + } finally { + database.close(); + } + embeddingCalls = 0; + reader = new Indexer(root, config, "pi"); + expect(await reader.getStatus()).toMatchObject({ indexed: true, mode, migrationRequired: false }); + expect(await reader.getIndexFreshness()).toEqual({ readable: true, current: false, reason: "keyword-index-outdated" }); + expect(fs.readFileSync(keywordPath)).toEqual(legacyBytes); + expect(fs.readFileSync(databasePath)).toEqual(catalogBytes); + if (vectorBytes) expect(fs.readFileSync(vectorPath)).toEqual(vectorBytes); + expect(embeddingCalls).toBe(0); + + const refreshed = await reader.index(); + expect(refreshed.tokensUsed).toBe(0); + expect(embeddingCalls).toBe(0); + expect(await reader.getIndexFreshness()).toEqual({ readable: true, current: true, reason: "current" }); + await reader.close(); + const upgraded = new InvertedIndex(keywordPath); + upgraded.load(); + expect(upgraded.hasCurrentTokenizer(chunkIds)).toBe(true); + expect(new Set(upgraded.search("tokenizer").keys())).toEqual(new Set(chunkIds)); + const retained = Database.openReadOnly(databasePath); + try { + expect(retained.getChunk(chunkIds[0])?.contentHash).toBe(contentHash); + if (mode === "hybrid") expect(retained.getEmbedding(contentHash)).toEqual(embedding); + } finally { + retained.close(); + } + } finally { + await reader?.close(); + await first.close(); + fetch.mockRestore(); + fs.rmSync(root, { recursive: true, force: true }); + } + }, + ); + + it.each(["hybrid", "structural"] as const)( + "rejects a future %s keyword snapshot without deleting catalog or vector data", + async (mode) => { + const root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), "keyword-future-version-"))); + fs.mkdirSync(path.join(root, "src")); + fs.writeFileSync(path.join(root, "src", "marker.ts"), "export function futureFormatMarker() { return 7; }\n"); + let embeddingCalls = 0; + const fetch = vi.spyOn(globalThis, "fetch").mockImplementation(async (_url, init) => { + embeddingCalls += 1; + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string | string[] }; + const texts = Array.isArray(body.input) ? body.input : [body.input ?? ""]; + return new Response(JSON.stringify({ + data: texts.map(() => ({ embedding: [1, 0, 0, 0, 0, 0, 0, 0] })), + usage: { total_tokens: texts.length }, + }), { status: 200 }); + }); + const config = parseConfig({ + embeddingProvider: "custom", + customProvider: { baseUrl: "http://localhost:11434/v1", model: "keyword-upgrade", dimensions: 8 }, + indexing: { mode, autoIndex: false, watchFiles: false, requireProjectMarker: false }, + include: ["src/**/*.ts"], + }); + const first = new Indexer(root, config, "pi"); + let reader: Indexer | undefined; + try { + await first.index(); + const initial = await first.getStatus(); + await first.close(); + const keywordPath = path.join(initial.indexPath, "inverted-index.json"); + const snapshot = JSON.parse(fs.readFileSync(keywordPath, "utf8")) as Record; + fs.writeFileSync(keywordPath, JSON.stringify({ ...snapshot, tokenizer_version: 2 })); + const artifacts = new Map( + ["inverted-index.json", "codebase.db", "vectors", "vectors.meta.json"] + .map((name) => path.join(initial.indexPath, name)) + .filter((file) => fs.existsSync(file)) + .map((file) => [file, fs.readFileSync(file)]), + ); + embeddingCalls = 0; + reader = new Indexer(root, config, "pi"); + await expect(reader.index()).rejects.toThrow(); + await reader.close(); + expect(embeddingCalls).toBe(0); + for (const [file, bytes] of artifacts) { + expect(fs.existsSync(file)).toBe(true); + expect(fs.readFileSync(file)).toEqual(bytes); + } + } finally { + await reader?.close(); + await first.close(); + fetch.mockRestore(); + fs.rmSync(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/tests/watcher-selected-config-dependencies.test.ts b/tests/watcher-selected-config-dependencies.test.ts index de7ee2a5..7f3a67fd 100644 --- a/tests/watcher-selected-config-dependencies.test.ts +++ b/tests/watcher-selected-config-dependencies.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import * as fs from "node:fs"; +import fs from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; import * as os from "node:os"; import * as path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -115,16 +116,20 @@ describe("selected local module config dependencies", () => { } cleanupPaths.clear(); vi.restoreAllMocks(); + vi.unstubAllEnvs(); + syncBuiltinESMExports(); }); it.each([ - ["chokidar", "vendor/app/src/*.ts"], - ["chokidar", "vendor/app/src/main.ts"], - ["native", "vendor/app/src/*.ts"], - ["native", "vendor/app/src/main.ts"], + ["chokidar", "vendor/app/src/*.ts", false], + ["chokidar", "vendor/app/src/main.ts", false], + ["native", "vendor/app/src/*.ts", false], + ["native", "vendor/app/src/main.ts", false], + ["chokidar", "vendor/app/src/*.ts", true], + ["native", "vendor/app/src/*.ts", true], ] as const)( - "refreshes the structural alias graph for %s with includeExcluded %s", - async (backend, includeExcluded) => { + "refreshes the structural alias graph for %s with includeExcluded %s and explicit config %s", + async (backend, includeExcluded, explicitConfig) => { const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "watcher-selected-config-")); cleanupPaths.add(fixtureRoot); const projectRoot = path.join(fixtureRoot, "project"); @@ -147,6 +152,8 @@ describe("selected local module config dependencies", () => { fs.writeFileSync(unrelatedSourcePath, "export function excludedSibling() { return 3; }\n"); fs.writeFileSync(appConfigPath, JSON.stringify({ extends: "../shared/base" })); writeAliasConfig(baseConfigPath, "one"); + const configPath = explicitConfig ? path.join(projectRoot, "custom-config.json") : undefined; + if (configPath) fs.writeFileSync(configPath, "{}"); const config = parseConfig({ include: ["**/*.ts"], @@ -158,11 +165,14 @@ describe("selected local module config dependencies", () => { }, }); const indexer = new Indexer(projectRoot, config, "jcode", { indexPath }); - const watcher = new FileWatcher(projectRoot, config, "jcode", { backend, indexPath }); + const watcher = new FileWatcher(projectRoot, config, "jcode", { backend, indexPath, configPath }); const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); const observedChanges: FileChange[] = []; let handlerFailure: unknown; let indexingTail: Promise = Promise.resolve(); + const omitRecursiveChange = backend === "chokidar" && process.platform === "darwin"; + let mutationStarted = false; + let recursiveRootRegistered = false; try { await indexer.index(); @@ -173,6 +183,34 @@ describe("selected local module config dependencies", () => { symbol.filePath === appConfigPath || symbol.filePath === baseConfigPath ))).toBe(false); + if (omitRecursiveChange) { + vi.stubEnv("CHOKIDAR_USEPOLLING", "false"); + const actualWatch = fs.watch; + vi.spyOn(fs, "watch").mockImplementation(function (this: unknown, ...args) { + const options = args[1]; + const listener = args[2]; + if ( + path.resolve(String(args[0])) === projectRoot + && typeof options === "object" && options?.recursive + && typeof listener === "function" + ) { + recursiveRootRegistered = true; + return Reflect.apply(actualWatch, this, [args[0], options, + (event: fs.WatchEventType, filename: string | Buffer | null) => { + // Model the documented first recursive callback omission; real explicit callbacks still run. + if ( + mutationStarted && filename !== null + && path.resolve(projectRoot, String(filename)) === baseConfigPath + ) return; + Reflect.apply(listener, undefined, [event, filename]); + }, + ]); + } + return Reflect.apply(actualWatch, this, args); + }); + syncBuiltinESMExports(); + } + watcher.start(async (changes) => { observedChanges.push(...changes); const indexing = indexingTail.then(() => indexer.index()); @@ -184,6 +222,7 @@ describe("selected local module config dependencies", () => { await watcher.waitUntilReady(); expectRequestedBackend(watcher, backend); + mutationStarted = true; writeAliasConfig(baseConfigPath, "two", 1); await waitForAssertion(async () => { if (handlerFailure) throw handlerFailure; @@ -198,6 +237,13 @@ describe("selected local module config dependencies", () => { }); }); + if (omitRecursiveChange) expect(recursiveRootRegistered).toBe(true); + const finalSymbols = await indexer.getSymbolsForBranch(); + expect(finalSymbols.some((symbol) => symbol.name === "excludedSibling")).toBe(false); + expect(finalSymbols.some((symbol) => ( + symbol.filePath === appConfigPath || symbol.filePath === baseConfigPath + ))).toBe(false); + expect(observedChanges.some((change) => change.path === unrelatedSourcePath)).toBe(false); expect(hasNativeFallbackWarning(warnSpy)).toBe(false); } finally { await watcher.stop(); From 016440af03c8a614d4e56076cd776d89b7c37587 Mon Sep 17 00:00:00 2001 From: Helweg Date: Thu, 29 Oct 2026 13:53:01 +0100 Subject: [PATCH 2/6] style: satisfy Rust formatting checks --- native/src/inverted_index.rs | 4 +++- native/src/lib.rs | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/native/src/inverted_index.rs b/native/src/inverted_index.rs index c24cef47..0b475e5a 100644 --- a/native/src/inverted_index.rs +++ b/native/src/inverted_index.rs @@ -300,7 +300,9 @@ impl InvertedIndexInner { previous.is_lowercase() || previous.is_numeric() || (previous.is_uppercase() - && characters.peek().is_some_and(|(_, next)| next.is_lowercase())) + && characters + .peek() + .is_some_and(|(_, next)| next.is_lowercase())) }); if boundary { let part = &word[start..offset]; diff --git a/native/src/lib.rs b/native/src/lib.rs index 77281ccf..d9a6fdf0 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -537,7 +537,8 @@ impl InvertedIndex { #[napi] pub fn outdated_tokenizer_chunk_ids(&self, chunk_ids: Option>) -> Vec { - self.inner.outdated_tokenizer_chunk_ids(chunk_ids.as_deref()) + self.inner + .outdated_tokenizer_chunk_ids(chunk_ids.as_deref()) } #[napi] From 0cc5014237b3d867da1e7721b81c8af3a2c2cb39 Mon Sep 17 00:00:00 2001 From: Helweg Date: Thu, 29 Oct 2026 14:08:09 +0100 Subject: [PATCH 3/6] fix: stop late diagnostic writes after ordered shutdown --- CHANGELOG.md | 1 + docs/tools.md | 2 ++ src/adapters/mcp/runtime-diagnostics.ts | 4 +++- tests/mcp-runtime-diagnostics.test.ts | 14 ++++++++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e1948d2..a8983fb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Keyword cache upgrades**: Keep older keyword snapshots readable and expose `keyword-index-outdated` through explicit freshness checks. Ordinary indexing refreshes affected active-branch tokenization without regenerating unchanged embeddings; structural caches rebuild from persisted source. Track partially refreshed branch chunks across snapshot reloads. Reject unsupported newer keyword formats without deleting their snapshots or resetting catalog/vector data. - **Code-request source preference**: Prefer implementation paths when a search explicitly asks for code, while preserving explicit test, documentation, and configuration intent. - **Initial module-config watching**: Register startup-discovered local module configuration dependencies as explicit Chokidar targets, preserving transitive alias-graph updates when an initial recursive directory notification is omitted, including when an explicit host configuration path is supplied. +- **MCP shutdown persistence**: Treat late completion of an already-interrupted operation as a no-op. Prevent cancelled startup diagnostics from scheduling writes after ordered shutdown has drained, including recreation of removed runtime storage. - **Consumer-visible evidence correctness**: Count only retained source and graph rows in pre-edit details, close truncated source fences at whole lines, and reserve outer-heading/final-truncation overhead before source fitting. Credit source only when its complete fitted block survives final composition. Allow structural-mode architecture maps to read branch files from SQLite without a vector store. Add native-backed citation, Unicode/CRLF, scope, and token-budget acceptance. - **Pre-edit source identity and cancellation**: Resolve canonical native symbol IDs before fitting target source, including large fragmented declarations, instead of accepting ranked body slices, suffix paths, or enclosing declarations. Cap source blocks at 512 tokens, withhold missing source, preserve default host roots, and share the exact lookup with evaluation. Keep selection-hook test references metadata-only and honor cancellation after empty lookups. - **Working-tree snapshot integrity**: Detect concurrent database publication even when source hashes and catalog counts are unchanged. Reject lossy UTF-8 Git paths and retain only project-local endpoints for cross-boundary renames. diff --git a/docs/tools.md b/docs/tools.md index a5f5a721..48ae3d82 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -274,6 +274,8 @@ A cache written with a newer, unsupported tokenizer version is not a corrupt cac MCP responses add `structuredContent.mcpDiagnostics` with schema version `1`. `activeOperations` reports each operation, its current phase, start and last-activity timestamps, and `active` or `suspected_stall` status. The current `index_status` call is excluded. `latestInterruptedOperation` is included only after an ordered shutdown marks an active call or a later local process confirms that the recorded PID is absent. A stale heartbeat alone is never classified as an interruption. +Once ordered shutdown marks an operation interrupted, its late completion does not write diagnostics again or recreate removed runtime storage. The interruption remains available to a subsequent connected session. + ### `index_codebase` Creates or updates the index. Incremental indexing is the default. Use `force: true` only for a required full rebuild. `estimateOnly` reports estimated embedding work without indexing. `dryRun` parses the real file set and reports the exact embedding token total (files, source chunks, tokens) without requesting embeddings or writing to the index — a read-only preflight. diff --git a/src/adapters/mcp/runtime-diagnostics.ts b/src/adapters/mcp/runtime-diagnostics.ts index 42632dcd..df8f331a 100644 --- a/src/adapters/mcp/runtime-diagnostics.ts +++ b/src/adapters/mcp/runtime-diagnostics.ts @@ -268,7 +268,9 @@ class ProcessRuntimeStore { complete: async (): Promise => { if (completed) return; completed = true; - this.state.activeOperations = this.state.activeOperations.filter((entry) => entry.id !== tracked.id); + const activeIndex = this.state.activeOperations.findIndex((entry) => entry.id === tracked.id); + if (activeIndex === -1) return; + this.state.activeOperations.splice(activeIndex, 1); await this.persist(); }, }; diff --git a/tests/mcp-runtime-diagnostics.test.ts b/tests/mcp-runtime-diagnostics.test.ts index 1faa61d4..1e4670f6 100644 --- a/tests/mcp-runtime-diagnostics.test.ts +++ b/tests/mcp-runtime-diagnostics.test.ts @@ -73,6 +73,20 @@ describe("MCP runtime diagnostics", () => { } }); + it("does not recreate removed runtime storage when an interrupted operation completes late", async () => { + const diagnostics = new McpRuntimeDiagnostics(indexRoot); + const operation = await diagnostics.begin("index_codebase"); + await diagnostics.markOrderedShutdown(); + await expect(new McpRuntimeDiagnostics(indexRoot).snapshot(undefined, 1000)).resolves.toMatchObject({ + activeOperations: [], + latestInterruptedOperation: { operation: "index_codebase", cause: "ordered_shutdown" }, + }); + + fs.rmSync(indexRoot, { recursive: true, force: true }); + await operation.complete(); + expect(fs.existsSync(indexRoot)).toBe(false); + }); + it("corrects permissions on an existing runtime directory", async () => { const runtimeDir = path.join(indexRoot, "mcp-runtime"); fs.mkdirSync(runtimeDir, { recursive: true, mode: 0o755 }); From 5e2e525cdfae978d5c149a2a4ee394c23d8adbd4 Mon Sep 17 00:00:00 2001 From: Helweg Date: Thu, 29 Oct 2026 14:56:23 +0100 Subject: [PATCH 4/6] fix: reconcile source edits across watcher startup and promotion --- CHANGELOG.md | 1 + docs/configuration.md | 4 +- src/watcher/index.ts | 8 ++- tests/fixtures/watcher-probe.mjs | 35 ++++++++++ tests/multiprocess-indexing.test.ts | 100 +++++++++++++++++++++++++++- 5 files changed, 144 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8983fb9..4a56f425 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Code-request source preference**: Prefer implementation paths when a search explicitly asks for code, while preserving explicit test, documentation, and configuration intent. - **Initial module-config watching**: Register startup-discovered local module configuration dependencies as explicit Chokidar targets, preserving transitive alias-graph updates when an initial recursive directory notification is omitted, including when an explicit host configuration path is supplied. - **MCP shutdown persistence**: Treat late completion of an already-interrupted operation as a no-op. Prevent cancelled startup diagnostics from scheduling writes after ordered shutdown has drained, including recreation of removed runtime storage. +- **Startup watcher catch-up**: Request one ordinary coordinated refresh after file/Git subscriptions become ready when automatic indexing is enabled. Recover edits made during initial watcher startup or background-leader promotion without changing watcher-only startup or queuing work after stop. - **Consumer-visible evidence correctness**: Count only retained source and graph rows in pre-edit details, close truncated source fences at whole lines, and reserve outer-heading/final-truncation overhead before source fitting. Credit source only when its complete fitted block survives final composition. Allow structural-mode architecture maps to read branch files from SQLite without a vector store. Add native-backed citation, Unicode/CRLF, scope, and token-budget acceptance. - **Pre-edit source identity and cancellation**: Resolve canonical native symbol IDs before fitting target source, including large fragmented declarations, instead of accepting ranked body slices, suffix paths, or enclosing declarations. Cap source blocks at 512 tokens, withhold missing source, preserve default host roots, and share the exact lookup with evaluation. Keep selection-hook test references metadata-only and honor cancellation after empty lookups. - **Working-tree snapshot integrity**: Detect concurrent database publication even when source hashes and catalog counts are unchanged. Reject lossy UTF-8 Git paths and retain only project-local endpoints for cross-boundary renames. diff --git a/docs/configuration.md b/docs/configuration.md index d6d8a87a..e3693d4f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -309,10 +309,12 @@ Ollama server is stopped. The standard suite uses deterministic HTTP replicas wi Unlimited depth retains hidden/build/ignored-path exclusions, does not follow symlinks, and keeps file-size and per-directory caps. Compared with older defaults, normal indexing can discover more source files and therefore take more time or use more embedding tokens in hybrid mode. Existing explicit `maxDepth` settings are preserved; set `5` to retain the former depth bound. -When file watching uses Chokidar, an identified Node `watch` registration failure (`EPERM` or `EACCES`) for a readable, selected source file can trigger the existing one-shot polling recovery, just like open-file exhaustion. The error must name an absolute path inside the project, and the file and all its ancestors must remain eligible under the watcher traversal, exclusion, Git-ignore, and protected-path policy. Readiness follows the replacement watcher, with policy-filtered catch-up for changes during replacement. Polling is not the default; unreadable, restricted, unselected, configuration-only, directory, and unidentified or relative error targets do not enable this recovery. +When file watching uses Chokidar, an identified Node `watch` registration failure (`EPERM` or `EACCES`) for a readable, selected source file or directory, including the project root, can trigger the existing one-shot polling recovery, just like open-file exhaustion. The error must name an absolute path inside the project, and the target and all its ancestors must remain eligible under the watcher traversal, exclusion, Git-ignore, and protected-path policy. Readiness follows the replacement watcher, with policy-filtered catch-up for changes during replacement. Polling is not the default; unreadable, restricted, unselected, configuration-only, and unidentified or relative error targets do not enable this recovery. Chokidar also receives explicit targets for local module configuration dependencies discovered at startup, including transitive `tsconfig` dependencies. This retains its independent first-change coverage when macOS omits an initial recursive directory notification. Existing selection, exclusion, and protected-path policies still apply; configuration notifications do not broaden which files are indexed. +With automatic indexing enabled, file/Git watcher readiness requests one ordinary coordinated refresh. This catches edits made after the initial scan but before subscriptions became ready, including during background-leader promotion. Existing in-flight indexing coalesces the refresh; unchanged chunks reuse their embeddings. Watcher-only mode (`autoIndex: false`) does not request a startup refresh. + For the disabled-by-default compiler-index pilot, see [Optional SCIP TypeScript enrichment](scip-typescript.md). Python call graphs support conservative local relative imports such as `from .formatting import format_payment` and explicit aliases. The importer and target must be indexed, package paths must have indexed `__init__.py` files, and the target must be a unique top-level function. Absolute/namespace imports, multiline imports, decorated targets, ambiguous modules, shadowing and detected dynamic rebinding are not resolved by this extension. Complex string interpolation also causes conservative abstention. This is static source analysis, not execution of Python's runtime import machinery. Normal indexing updates older graph metadata and refreshes affected Python graph sources in both indexing modes. diff --git a/src/watcher/index.ts b/src/watcher/index.ts index 31fcaf8f..5ee93d83 100644 --- a/src/watcher/index.ts +++ b/src/watcher/index.ts @@ -84,6 +84,7 @@ export function createWatcherWithIndexer( requestReindex(); }); } + let startupCatchupRequested = false; return { fileWatcher, @@ -95,7 +96,12 @@ export function createWatcherWithIndexer( return Promise.all([ fileWatcher.waitUntilReady(), gitWatcher?.waitUntilReady(), - ]).then(() => undefined); + ]).then(() => { + if (startupCatchupRequested) return; + startupCatchupRequested = true; + // Reconcile edits made after the startup scan but before subscriptions became ready. + if (config.indexing?.autoIndex !== false) requestReindex(); + }); }, async stop() { stopped = true; diff --git a/tests/fixtures/watcher-probe.mjs b/tests/fixtures/watcher-probe.mjs index 74244bce..cd79b341 100644 --- a/tests/fixtures/watcher-probe.mjs +++ b/tests/fixtures/watcher-probe.mjs @@ -4,6 +4,9 @@ import path from "node:path"; const eventsPath = process.env.TEST_WATCHER_EVENTS_PATH; const projectRoot = process.env.TEST_PROJECT_ROOT; +const readyPath = process.env.TEST_WATCHER_READY_PATH; +const startReleaseDirectory = process.env.TEST_WATCHER_START_RELEASE_DIRECTORY; +const heldPath = process.env.TEST_WATCHER_HELD_PATH; const originalWatch = fs.watch; if (eventsPath && projectRoot) { @@ -17,3 +20,35 @@ if (eventsPath && projectRoot) { }; syncBuiltinESMExports(); } + +if ((readyPath || startReleaseDirectory) && projectRoot) { + const { FileWatcher } = await import("../../src/watcher/file-watcher.ts"); + const pendingStarts = new WeakMap(); + if (startReleaseDirectory) { + const start = FileWatcher.prototype.start; + FileWatcher.prototype.start = function startWithGate(...args) { + if (heldPath) fs.appendFileSync(heldPath, `${process.pid}\n`, "utf-8"); + const pending = new Promise((resolve) => { + const releasePath = path.join(startReleaseDirectory, String(process.pid)); + const check = () => { + if (!fs.existsSync(releasePath)) { + setTimeout(check, 10); + return; + } + start.apply(this, args); + resolve(); + }; + check(); + }); + pendingStarts.set(this, pending); + }; + } + const waitUntilReady = FileWatcher.prototype.waitUntilReady; + FileWatcher.prototype.waitUntilReady = async function waitWithReadyProbe() { + await pendingStarts.get(this); + await waitUntilReady.call(this); + if (readyPath && path.resolve(this.projectRoot) === path.resolve(projectRoot)) { + fs.appendFileSync(readyPath, `${process.pid}\n`, "utf-8"); + } + }; +} diff --git a/tests/multiprocess-indexing.test.ts b/tests/multiprocess-indexing.test.ts index 2a8304a5..1339a56b 100644 --- a/tests/multiprocess-indexing.test.ts +++ b/tests/multiprocess-indexing.test.ts @@ -1656,9 +1656,15 @@ describe("multiprocess indexing", () => { await embeddingServer.waitForIdle(); embeddingServer.reset(); + const watcherReadyPath = path.join(tempDir, "busy-watcher-ready.log"); + const environment = { + TEST_PROJECT_ROOT: projectRoot, + TEST_WATCHER_READY_PATH: watcherReadyPath, + }; + const preloads = [new URL("./fixtures/watcher-probe.mjs", import.meta.url).href]; const [first, second] = await Promise.all([ - createMcpClient(configPath, "codex"), - createMcpClient(configPath, "codex"), + createMcpClient(configPath, "codex", { environment, preloads }), + createMcpClient(configPath, "codex", { environment, preloads }), ]); const leasePath = getBackgroundWorkerLeasePath(projectRoot, parsedConfig, "codex"); await waitForCondition(() => fs.existsSync(leasePath), "busy-shutdown background worker lease"); @@ -1667,6 +1673,9 @@ describe("multiprocess indexing", () => { const follower = leader === first ? second : first; const leaderPid = leader.transport.pid; if (leaderPid === null) throw new Error("Expected an MCP leader process"); + await waitForCondition(() => fs.existsSync(watcherReadyPath) + && fs.readFileSync(watcherReadyPath, "utf-8").split("\n").includes(String(leaderPid)), + "busy leader watcher readiness"); embeddingServer.block(); fs.writeFileSync(sourcePath, "export function busyhandoff() { return 'before-bounded-shutdown'; }\n"); @@ -1697,6 +1706,9 @@ describe("multiprocess indexing", () => { const text = (status.content as Array<{ text?: string }>).map(({ text }) => text ?? "").join("\n"); expect(text).toContain("state: ready"); }, { timeout: 10_000 }); + await waitForCondition(() => fs.existsSync(watcherReadyPath) + && fs.readFileSync(watcherReadyPath, "utf-8").split("\n").includes(String(follower.transport.pid)), + "promoted busy follower watcher readiness"); embeddingServer.reset(); fs.writeFileSync(sourcePath, "export function recoveredbusyhandoff() { return 'after-bounded-shutdown'; }\n"); @@ -1710,6 +1722,90 @@ describe("multiprocess indexing", () => { mcpClients = mcpClients.filter((client) => client !== follower.client); }); + it("indexes edits made before the initial and promoted Codex watchers become ready", async () => { + const configPath = path.join(tempDir, "codex-startup-gap.json"); + const parsedConfig = writeBackgroundMcpConfig(configPath); + const seededIndexer = new Indexer(projectRoot, parsedConfig, "codex"); + localIndexers.push(seededIndexer); + await seededIndexer.index(); + await embeddingServer.waitForIdle(); + embeddingServer.reset(); + + const releaseDirectory = path.join(tempDir, "watcher-start-releases"); + fs.mkdirSync(releaseDirectory); + const heldPath = path.join(tempDir, "watcher-start-held.log"); + const readyPath = path.join(tempDir, "watcher-start-ready.log"); + const environment = { + TEST_PROJECT_ROOT: projectRoot, + TEST_WATCHER_START_RELEASE_DIRECTORY: releaseDirectory, + TEST_WATCHER_HELD_PATH: heldPath, + TEST_WATCHER_READY_PATH: readyPath, + }; + const preloads = [new URL("./fixtures/watcher-probe.mjs", import.meta.url).href]; + const [first, second] = await Promise.all([ + createMcpClient(configPath, "codex", { environment, preloads }), + createMcpClient(configPath, "codex", { environment, preloads }), + ]); + try { + const leasePath = getBackgroundWorkerLeasePath(projectRoot, parsedConfig, "codex"); + await waitForCondition(() => fs.existsSync(leasePath), "startup-gap leader lease"); + const owner = JSON.parse(fs.readFileSync(path.join(leasePath, "owner.json"), "utf-8")) as { pid: number }; + const leader = first.transport.pid === owner.pid ? first : second; + const follower = leader === first ? second : first; + const phases = [ + { process: leader, name: "startupgap", marker: "before-initial-watcher-ready" }, + { process: follower, name: "promotiongap", marker: "before-promoted-watcher-ready" }, + ]; + for (const phase of phases) { + const pid = phase.process.transport.pid; + if (pid === null) throw new Error("Expected a live startup-gap MCP process"); + await waitForCondition(() => fs.existsSync(heldPath) + && fs.readFileSync(heldPath, "utf-8").split("\n").includes(String(pid)), + "controlled watcher startup hold", 12_000); + await vi.waitFor(async () => { + const status = await phase.process.client.callTool({ name: "index_status", arguments: {} }); + expect((status.content as Array<{ text?: string }>).map(({ text }) => text ?? "").join("\n")) + .toContain("state: ready"); + }, { timeout: 10_000 }); + await embeddingServer.waitForIdle(); + embeddingServer.reset(); + + fs.writeFileSync(sourcePath, `export function ${phase.name}() { return '${phase.marker}'; }\n`); + fs.writeFileSync(path.join(releaseDirectory, String(pid)), "release"); + await embeddingServer.waitForRequestCount(1); + await embeddingServer.waitForIdle(); + expect(embeddingServer.inputs).toEqual(expect.arrayContaining([expect.stringContaining(phase.marker)])); + await vi.waitFor(async () => { + const definition = await phase.process.client.callTool({ + name: "implementation_lookup", arguments: { query: phase.name, limit: 1 }, + }); + expect((definition.content as Array<{ text?: string }>).map(({ text }) => text ?? "").join("\n")) + .toContain(phase.name); + }, { timeout: 5000 }); + + if (phase.process === leader) { + await leader.client.close(); + mcpClients = mcpClients.filter((client) => client !== leader.client); + await waitForCondition(() => { + try { + const replacement = JSON.parse(fs.readFileSync(path.join(leasePath, "owner.json"), "utf-8")) as { pid: number }; + return replacement.pid === follower.transport.pid; + } catch { + return false; + } + }, "startup-gap follower promotion", 12_000); + } + } + } finally { + for (const process of [first, second]) { + if (process.transport.pid !== null) { + fs.writeFileSync(path.join(releaseDirectory, String(process.transport.pid)), "release"); + } + } + } + }); + + it("keeps watcher-driven indexing when auto-index startup is disabled", async () => { const configPath = path.join(tempDir, "codex-watcher-only.json"); const parsedConfig = writeBackgroundMcpConfig(configPath, { autoIndex: false, watchFiles: true }); From 78f1f7aac1c59eed5b0c61dd46416952236a7eca Mon Sep 17 00:00:00 2001 From: Helweg Date: Thu, 29 Oct 2026 16:21:33 +0100 Subject: [PATCH 5/6] fix: keep degraded keyword reads out of current-index shortcuts --- CHANGELOG.md | 1 + docs/tools.md | 2 ++ src/indexer/index.ts | 7 +++++- src/utils/auto-index.ts | 5 +++- tests/keyword-tokenizer-upgrade.test.ts | 33 ++++++++++++++++++++++--- 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a56f425..ab78cc6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Identifier-word retrieval**: Match natural-language terms inside camelCase, PascalCase, and acronym-bearing identifiers in BM25, and preserve word boundaries in lexical definition fallback without colliding with lowercased cache entries. Keep whole-identifier queries distinct from searches for shared identifier words. - **Keyword cache upgrades**: Keep older keyword snapshots readable and expose `keyword-index-outdated` through explicit freshness checks. Ordinary indexing refreshes affected active-branch tokenization without regenerating unchanged embeddings; structural caches rebuild from persisted source. Track partially refreshed branch chunks across snapshot reloads. Reject unsupported newer keyword formats without deleting their snapshots or resetting catalog/vector data. +- **Hybrid keyword readability**: Keep semantic retrieval available when only optional keyword reads degrade, but expose `keyword-index-unreadable` rather than claiming the index is current. Prevent ordinary coordinated indexing from skipping validation of an unsupported newer keyword format as healthy/current; retain source-freshness and blocking-read safeguards. - **Code-request source preference**: Prefer implementation paths when a search explicitly asks for code, while preserving explicit test, documentation, and configuration intent. - **Initial module-config watching**: Register startup-discovered local module configuration dependencies as explicit Chokidar targets, preserving transitive alias-graph updates when an initial recursive directory notification is omitted, including when an explicit host configuration path is supplied. - **MCP shutdown persistence**: Treat late completion of an already-interrupted operation as a no-op. Prevent cancelled startup diagnostics from scheduling writes after ordered shutdown has drained, including recreation of removed runtime storage. diff --git a/docs/tools.md b/docs/tools.md index 48ae3d82..13479cf4 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -272,6 +272,8 @@ An older keyword snapshot remains readable. With freshness checking, `keyword-in A cache written with a newer, unsupported tokenizer version is not a corrupt cache to reset. Ordinary indexing rejects it without deleting or rewriting its keyword snapshot, catalog, or vectors. Use a compatible package version. +If only a hybrid keyword artifact is unreadable, `keyword-index-unreadable` distinguishes degraded retrieval from a healthy, fully current index. With valid vectors and current source/configuration/HEAD, freshness is `{ readable: true, current: false, reason: "keyword-index-unreadable" }` and semantic retrieval remains available. Source changes, incompatible providers, missing vectors, and blocking read failures keep their existing freshness checks and priority. Ordinary coordinated indexing does not skip this degraded state as healthy/current: it repairs recoverable keyword damage or rejects an unsupported newer format without resetting stored data. + MCP responses add `structuredContent.mcpDiagnostics` with schema version `1`. `activeOperations` reports each operation, its current phase, start and last-activity timestamps, and `active` or `suspected_stall` status. The current `index_status` call is excluded. `latestInterruptedOperation` is included only after an ordered shutdown marks an active call or a later local process confirms that the recorded PID is absent. A stale heartbeat alone is never classified as an interruption. Once ordered shutdown marks an operation interrupted, its late completion does not write diagnostics again or recreate removed runtime storage. The interruption remains available to a subsequent connected session. diff --git a/src/indexer/index.ts b/src/indexer/index.ts index b747b8cc..93e61501 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -579,7 +579,8 @@ export type IndexFreshnessReason = | "metadata-changed" | "branch-changed" | "migration-required" - | "keyword-index-outdated"; + | "keyword-index-outdated" + | "keyword-index-unreadable"; export interface IndexFreshnessResult { readable: boolean; @@ -7129,6 +7130,10 @@ export class Indexer { return { readable: true, current: false, reason: "branch-changed" }; } } + // Current source and valid vectors remain searchable when only keyword reads degrade. + if (readIssues.length > 0) { + return { readable: true, current: false, reason: "keyword-index-unreadable" }; + } if (this.needsKeywordTokenizerUpgrade(requireLoadedCapability(this.invertedIndex, "keyword index"), database)) { return { readable: true, current: false, reason: "keyword-index-outdated" }; } diff --git a/src/utils/auto-index.ts b/src/utils/auto-index.ts index 4156d407..dfd701e4 100644 --- a/src/utils/auto-index.ts +++ b/src/utils/auto-index.ts @@ -1248,7 +1248,10 @@ async function getSearchReadiness( heartbeat: control?.heartbeat, }); throwIfOperationAborted(control?.signal); - const searchable = freshness.readable && freshness.current && freshness.reason === "current"; + const searchable = freshness.readable && ( + (freshness.current && freshness.reason === "current") + || freshness.reason === "keyword-index-unreadable" + ); return { blocked: freshness.reason === "unreadable" || freshness.reason === "incompatible" diff --git a/tests/keyword-tokenizer-upgrade.test.ts b/tests/keyword-tokenizer-upgrade.test.ts index 0e34a696..1ca458c1 100644 --- a/tests/keyword-tokenizer-upgrade.test.ts +++ b/tests/keyword-tokenizer-upgrade.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { parseConfig } from "../src/config/schema.js"; import { Indexer } from "../src/indexer/index.js"; import { Database, InvertedIndex } from "../src/native/index.js"; +import { configureAutoIndex, runCoordinatedIndex, stopAutoIndex, waitForAutoIndexForRetrieval } from "../src/utils/auto-index.js"; describe("keyword tokenizer upgrades", () => { it.each(["hybrid", "structural"] as const)( @@ -96,7 +97,7 @@ describe("keyword tokenizer upgrades", () => { ); it.each(["hybrid", "structural"] as const)( - "rejects a future %s keyword snapshot without deleting catalog or vector data", + "rejects coordinated indexing of a future %s keyword snapshot without deleting catalog or vector data", async (mode) => { const root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), "keyword-future-version-"))); fs.mkdirSync(path.join(root, "src")); @@ -114,7 +115,7 @@ describe("keyword tokenizer upgrades", () => { const config = parseConfig({ embeddingProvider: "custom", customProvider: { baseUrl: "http://localhost:11434/v1", model: "keyword-upgrade", dimensions: 8 }, - indexing: { mode, autoIndex: false, watchFiles: false, requireProjectMarker: false }, + indexing: { mode, autoIndex: true, watchFiles: false, requireProjectMarker: false }, include: ["src/**/*.ts"], }); const first = new Indexer(root, config, "pi"); @@ -134,7 +135,32 @@ describe("keyword tokenizer upgrades", () => { ); embeddingCalls = 0; reader = new Indexer(root, config, "pi"); - await expect(reader.index()).rejects.toThrow(); + const currentReader = reader; + configureAutoIndex(root, "pi", config, () => currentReader); + if (mode === "hybrid") { + await expect(waitForAutoIndexForRetrieval(root, "pi")).resolves.toEqual({ ready: true }); + expect((await reader.search("future format marker", 1))[0]).toMatchObject({ + name: "futureFormatMarker", + filePath: path.join(root, "src", "marker.ts"), + }); + const sourcePath = path.join(root, "src", "marker.ts"); + const sourceBytes = fs.readFileSync(sourcePath); + fs.writeFileSync(sourcePath, "export function futureFormatMarker() { return 8; }\n"); + expect(await reader.getIndexFreshness()).toEqual({ + readable: true, current: false, reason: "files-changed", + }); + fs.writeFileSync(sourcePath, sourceBytes); + embeddingCalls = 0; + } + expect(await reader.getIndexFreshness()).toEqual({ + readable: mode === "hybrid", + current: false, + reason: mode === "hybrid" ? "keyword-index-unreadable" : "unreadable", + }); + expect(await runCoordinatedIndex(root, "pi", false)).toMatchObject({ + outcome: "failed", + error: expect.objectContaining({ code: "InvalidArg" }), + }); await reader.close(); expect(embeddingCalls).toBe(0); for (const [file, bytes] of artifacts) { @@ -142,6 +168,7 @@ describe("keyword tokenizer upgrades", () => { expect(fs.readFileSync(file)).toEqual(bytes); } } finally { + await stopAutoIndex(root, "pi", true); await reader?.close(); await first.close(); fetch.mockRestore(); From c7d6894f4a4fcd71383bae4421a580a3e926b1c2 Mon Sep 17 00:00:00 2001 From: Helweg Date: Fri, 30 Oct 2026 09:10:07 +0100 Subject: [PATCH 6/6] fix: validate reused structural writers without discarding pending keywords --- CHANGELOG.md | 2 +- docs/tools.md | 2 +- native/src/inverted_index.rs | 28 +++++++++++++++++++++---- native/src/lib.rs | 7 +++++++ src/indexer/index.ts | 12 ++++++++++- src/native/inverted-index.ts | 4 ++++ tests/inverted-index.test.ts | 23 ++++++++++++++++++++ tests/keyword-tokenizer-upgrade.test.ts | 25 +++++++++++++++------- 8 files changed, 88 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab78cc6c..edea5bc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **Identifier-word retrieval**: Match natural-language terms inside camelCase, PascalCase, and acronym-bearing identifiers in BM25, and preserve word boundaries in lexical definition fallback without colliding with lowercased cache entries. Keep whole-identifier queries distinct from searches for shared identifier words. -- **Keyword cache upgrades**: Keep older keyword snapshots readable and expose `keyword-index-outdated` through explicit freshness checks. Ordinary indexing refreshes affected active-branch tokenization without regenerating unchanged embeddings; structural caches rebuild from persisted source. Track partially refreshed branch chunks across snapshot reloads. Reject unsupported newer keyword formats without deleting their snapshots or resetting catalog/vector data. +- **Keyword cache upgrades**: Keep older keyword snapshots readable and expose `keyword-index-outdated` through explicit freshness checks. Ordinary indexing refreshes affected active-branch tokenization without regenerating unchanged embeddings; structural caches rebuild from persisted source. Track partially refreshed branch chunks across snapshot reloads. Reject unsupported newer keyword formats without deleting their snapshots or resetting catalog/vector data, including reused structural writers. Validate persisted versions without replacing unsaved keyword state needed after a failed publication. - **Hybrid keyword readability**: Keep semantic retrieval available when only optional keyword reads degrade, but expose `keyword-index-unreadable` rather than claiming the index is current. Prevent ordinary coordinated indexing from skipping validation of an unsupported newer keyword format as healthy/current; retain source-freshness and blocking-read safeguards. - **Code-request source preference**: Prefer implementation paths when a search explicitly asks for code, while preserving explicit test, documentation, and configuration intent. - **Initial module-config watching**: Register startup-discovered local module configuration dependencies as explicit Chokidar targets, preserving transitive alias-graph updates when an initial recursive directory notification is omitted, including when an explicit host configuration path is supplied. diff --git a/docs/tools.md b/docs/tools.md index 13479cf4..aae1069c 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -270,7 +270,7 @@ Catalog presence and provider/model compatibility remain separate from pending s An older keyword snapshot remains readable. With freshness checking, `keyword-index-outdated` means the active branch needs the current identifier-word tokenizer. Run ordinary `index_codebase` or `ocbi index`; unchanged embeddings are reused, without deleting the index or forcing re-embedding. Read-only status checks never rewrite the cache. Other hybrid branches' pending tokenizer state is not counted against the refreshed active branch. -A cache written with a newer, unsupported tokenizer version is not a corrupt cache to reset. Ordinary indexing rejects it without deleting or rewriting its keyword snapshot, catalog, or vectors. Use a compatible package version. +A cache written with a newer, unsupported tokenizer version is not a corrupt cache to reset. Ordinary indexing rejects it without deleting or rewriting its keyword snapshot, catalog, or vectors, including when a long-lived process reuses an existing structural writer. Version validation runs under the mutation lease without replacing the live keyword state needed to retry a failed publication. Use a compatible package version. If only a hybrid keyword artifact is unreadable, `keyword-index-unreadable` distinguishes degraded retrieval from a healthy, fully current index. With valid vectors and current source/configuration/HEAD, freshness is `{ readable: true, current: false, reason: "keyword-index-unreadable" }` and semantic retrieval remains available. Source changes, incompatible providers, missing vectors, and blocking read failures keep their existing freshness checks and priority. Ordinary coordinated indexing does not skip this degraded state as healthy/current: it repairs recoverable keyword damage or rejects an unsupported newer format without resetting stored data. diff --git a/native/src/inverted_index.rs b/native/src/inverted_index.rs index 0b475e5a..936fc925 100644 --- a/native/src/inverted_index.rs +++ b/native/src/inverted_index.rs @@ -2,7 +2,7 @@ use anyhow::Result; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fs; -use std::io::{BufWriter, Write}; +use std::io::{BufReader, BufWriter, Write}; use std::path::{Path, PathBuf}; const TOKENIZER_VERSION: u32 = 1; @@ -22,6 +22,19 @@ impl std::fmt::Display for UnsupportedTokenizerVersionError { impl std::error::Error for UnsupportedTokenizerVersionError {} +fn validate_tokenizer_version(version: u32) -> Result<()> { + if version > TOKENIZER_VERSION { + return Err(UnsupportedTokenizerVersionError(version).into()); + } + Ok(()) +} + +#[derive(Deserialize)] +struct InvertedIndexVersion { + #[serde(default)] + tokenizer_version: u32, +} + #[derive(Serialize, Deserialize, Default)] struct InvertedIndexData { term_to_chunks: HashMap>, @@ -70,6 +83,15 @@ impl InvertedIndexInner { self.restore(serde_json::from_str(&content)?) } + pub fn validate_snapshot_version(&self) -> Result<()> { + if !self.index_path.exists() { + return Ok(()); + } + let reader = BufReader::new(fs::File::open(&self.index_path)?); + let version: InvertedIndexVersion = serde_json::from_reader(reader)?; + validate_tokenizer_version(version.tokenizer_version) + } + pub fn save(&self) -> Result<()> { self.save_to(&self.index_path) } @@ -105,9 +127,7 @@ impl InvertedIndexInner { } fn restore(&mut self, data: InvertedIndexData) -> Result<()> { - if data.tokenizer_version > TOKENIZER_VERSION { - return Err(UnsupportedTokenizerVersionError(data.tokenizer_version).into()); - } + validate_tokenizer_version(data.tokenizer_version)?; let legacy_chunks = if data.tokenizer_version < TOKENIZER_VERSION { data.chunk_tokens.keys().cloned().collect() } else { diff --git a/native/src/lib.rs b/native/src/lib.rs index d9a6fdf0..188ef7bc 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -495,6 +495,13 @@ impl InvertedIndex { self.inner.load().map_err(keyword_index_error) } + #[napi] + pub fn validate_snapshot_version(&self) -> Result<()> { + self.inner + .validate_snapshot_version() + .map_err(keyword_index_error) + } + #[napi] pub fn save(&self, destination: Option) -> Result<()> { let result = match destination { diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 93e61501..947df284 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -4635,7 +4635,17 @@ export class Indexer { await this.initializationPromise; } - if (recoveredOwners.length > 0 || !this.isInitializedFor("writer")) { + let needsWriterInitialization = recoveredOwners.length > 0 || !this.isInitializedFor("writer"); + if (!needsWriterInitialization && this.config.indexing.mode === "structural") { + try { + this.requireLoadedIndexState().invertedIndex.validateSnapshotVersion(); + needsWriterInitialization = this.readIssues.some((issue) => issue.component === "keyword"); + } catch (error) { + if (isUnsupportedKeywordIndexVersion(error)) throw error; + needsWriterInitialization = true; + } + } + if (needsWriterInitialization) { const retireReaderDatabase = this.initializationMode === "reader"; this.resetLoadedIndexState(retireReaderDatabase); await this.initializeOnce("writer", recoveredOwners, { skipAutoGc: true }); diff --git a/src/native/inverted-index.ts b/src/native/inverted-index.ts index c68ff6fa..c3e2026f 100644 --- a/src/native/inverted-index.ts +++ b/src/native/inverted-index.ts @@ -11,6 +11,10 @@ export class InvertedIndex { this.inner.load(); } + validateSnapshotVersion(): void { + this.inner.validateSnapshotVersion(); + } + save(destination?: string): void { this.inner.save(destination); } diff --git a/tests/inverted-index.test.ts b/tests/inverted-index.test.ts index f2e5f0e7..720f5a85 100644 --- a/tests/inverted-index.test.ts +++ b/tests/inverted-index.test.ts @@ -165,6 +165,29 @@ describe("InvertedIndex", () => { expect(() => index.deserialize(JSON.stringify(future))).toThrow(); expect([...index.search("retained").keys()]).toEqual(["retained"]); }); + + it("validates persisted versions without replacing unsaved keyword state", () => { + index.addChunk("saved", "originalNeedle"); + index.save(); + const persisted = fs.readFileSync(indexPath); + index.clear(); + index.addChunk("pending", "replacementNeedle"); + + index.validateSnapshotVersion(); + expect([...index.search("replacementNeedle").keys()]).toEqual(["pending"]); + expect([...index.search("originalNeedle").keys()]).toEqual([]); + expect(fs.readFileSync(indexPath)).toEqual(persisted); + + const future = Buffer.from(JSON.stringify({ + ...JSON.parse(persisted.toString("utf8")), + tokenizer_version: 2, + })); + fs.writeFileSync(indexPath, future); + expect(() => index.validateSnapshotVersion()).toThrow("unsupported tokenizer version"); + expect([...index.search("replacementNeedle").keys()]).toEqual(["pending"]); + expect([...index.search("originalNeedle").keys()]).toEqual([]); + expect(fs.readFileSync(indexPath)).toEqual(future); + }); }); describe("removeChunk", () => { diff --git a/tests/keyword-tokenizer-upgrade.test.ts b/tests/keyword-tokenizer-upgrade.test.ts index 1ca458c1..a869fb5f 100644 --- a/tests/keyword-tokenizer-upgrade.test.ts +++ b/tests/keyword-tokenizer-upgrade.test.ts @@ -96,9 +96,14 @@ describe("keyword tokenizer upgrades", () => { }, ); - it.each(["hybrid", "structural"] as const)( - "rejects coordinated indexing of a future %s keyword snapshot without deleting catalog or vector data", - async (mode) => { + it.each([ + { mode: "hybrid", reuseWriter: false }, + { mode: "structural", reuseWriter: false }, + { mode: "hybrid", reuseWriter: true }, + { mode: "structural", reuseWriter: true }, + ] as const)( + "rejects coordinated future-cache indexing ($mode, reused writer: $reuseWriter) without changing stored data", + async ({ mode, reuseWriter }) => { const root = fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), "keyword-future-version-"))); fs.mkdirSync(path.join(root, "src")); fs.writeFileSync(path.join(root, "src", "marker.ts"), "export function futureFormatMarker() { return 7; }\n"); @@ -123,18 +128,18 @@ describe("keyword tokenizer upgrades", () => { try { await first.index(); const initial = await first.getStatus(); - await first.close(); + if (!reuseWriter) await first.close(); const keywordPath = path.join(initial.indexPath, "inverted-index.json"); const snapshot = JSON.parse(fs.readFileSync(keywordPath, "utf8")) as Record; fs.writeFileSync(keywordPath, JSON.stringify({ ...snapshot, tokenizer_version: 2 })); const artifacts = new Map( - ["inverted-index.json", "codebase.db", "vectors", "vectors.meta.json"] + ["inverted-index.json", "codebase.db", "codebase.db-wal", "structural-generation", "vectors", "vectors.meta.json"] .map((name) => path.join(initial.indexPath, name)) .filter((file) => fs.existsSync(file)) .map((file) => [file, fs.readFileSync(file)]), ); embeddingCalls = 0; - reader = new Indexer(root, config, "pi"); + reader = reuseWriter ? first : new Indexer(root, config, "pi"); const currentReader = reader; configureAutoIndex(root, "pi", config, () => currentReader); if (mode === "hybrid") { @@ -159,14 +164,18 @@ describe("keyword tokenizer upgrades", () => { }); expect(await runCoordinatedIndex(root, "pi", false)).toMatchObject({ outcome: "failed", - error: expect.objectContaining({ code: "InvalidArg" }), + error: expect.objectContaining({ + code: "InvalidArg", + message: expect.stringContaining("unsupported tokenizer version"), + }), }); - await reader.close(); expect(embeddingCalls).toBe(0); for (const [file, bytes] of artifacts) { expect(fs.existsSync(file)).toBe(true); expect(fs.readFileSync(file)).toEqual(bytes); } + await reader.close(); + expect(fs.readFileSync(keywordPath)).toEqual(artifacts.get(keywordPath)); } finally { await stopAutoIndex(root, "pi", true); await reader?.close();