diff --git a/AGENTS.md b/AGENTS.md index 4bb5266..cf50fa7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,8 +22,8 @@ | Rust Edition | `2024`（要求 rustc 1.95+） | | Registry Schema | `v36`（`src/registry/migrate.rs`） | | MCP Tools | **71** 个（`src/mcp/tools/*.rs` 中 `pub struct Devkit*Tool`） | -| 测试函数 | **605** 个（`cargo test --workspace -- --list`） | -| Ignored 测试 | 7 个（`#[ignore]` 在 `src/` 6 个 + `crates/devbase-embedding` 1 个） | +| 测试函数 | **616** 个（`cargo test --workspace -- --list`） | +| Ignored 测试 | 6 个（`#[ignore]` 在 `src/`） | | Workspace Crates | **12** 个（`crates/` 目录） | | `src/main.rs` 行数 | 833 行（RF-4 限界 1000 行内） | | Clippy | `-D warnings` / CI `-W warnings` | @@ -108,7 +108,8 @@ devbase/ │ └── cli.rs # 11 个集成测试 ├── benches/ │ ├── registry_bench.rs -│ └── semantic_index.rs +│ ├── semantic_index.rs +│ └── vault_bench.rs ├── skills/ # 示例 Skill（embed-repo / knowledge-report / search-workspace） ├── scripts/ │ ├── install.ps1 / install.sh @@ -173,7 +174,7 @@ scripts/invariant-checks/run-checks.ps1 - **单元测试**：分布在 `src/**/tests.rs` 与 `#[cfg(test)]` 块中。 - **集成测试**：`tests/cli.rs`，使用 `assert_cmd` + `tempfile`，通过 `DEVBASE_DATA_DIR` 隔离数据目录。 - **Crate 测试**：每个 `crates/*/src/*.rs` 自带测试。 -- **Bench**：`criterion` 驱动的 `benches/registry_bench.rs`、`benches/semantic_index.rs`。 +- **Bench**：`criterion` 驱动的 `benches/registry_bench.rs`、`benches/semantic_index.rs`、`benches/vault_bench.rs`。 - **测试隔离**： - 所有 IO 测试使用 `TempDir` 与 `StorageBackend` 注入，禁止直接写 `%LOCALAPPDATA%`。 - `.cargo/config.toml` 默认 `RUST_TEST_THREADS=1`；CI 使用 `--test-threads=4`。 diff --git a/CHANGELOG.md b/CHANGELOG.md index 107b94f..14d1ae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Ontology 导入** — `devkit_ontology_import` MCP 工具（Beta tier），`devbase ontology` CLI（`--dry-run` 预览），支持 OpenClaw workspace `ontology/entities/*.json` + `ontology/relations/*.jsonl` 批量导入 - MCP 工具数: 69 → **71**（5 stable / 58 beta / 8 experimental） - `devkit_document_convert` — Experimental tier MCP tool，PDF/PPTX → Markdown 转换（`pdftotext` / `python-pptx` 流水线），含 frontmatter 质量标注 -- Stable 工具 invocation 测试补全：`devkit_query_repos`、`devkit_vault_search`、`devkit_vault_read`、`devkit_status`、`devkit_workflow_list`、`devkit_index` +- **Vault Tantivy 全文搜索** — `devkit_vault_search` 优先使用 BM25（`search_vault_at`），空结果/失败时回退到内存扫描；`run_index_with_progress` 在索引仓库时同步 `reindex_vault_with_writer` 写入同一 Tantivy 段 +- **Criterion vault benchmark 基线** — 新增 `benches/vault_bench.rs`（`reindex_vault` 50/200 笔记，`search_vault` 单/多关键词 50/200 笔记），保存 baseline `v0.20.1` +- Stable 工具 invocation 测试补全：`devkit_query_repos`、`devkit_vault_search`（覆盖 Tantivy 路径）、`devkit_vault_read`、`devkit_status`、`devkit_workflow_list`、`devkit_index` - `seed_repo()` 轻量测试 helper（仅插入 `entities` 表，无副作用） ### Fixed diff --git a/Cargo.toml b/Cargo.toml index 2250f6e..75d22aa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,10 @@ harness = false name = "registry_bench" harness = false +[[bench]] +name = "vault_bench" +harness = false + [features] default = ["tui", "mcp", "lang-rust", "lang-python", "lang-js-ts", "lang-go"] diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index f4529d7..bd3fc3b 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -62,28 +62,18 @@ **建议**：考虑使用宏或 derive 自动生成 `McpToolEnum` 和 `tier()`，减少 boilerplate。 -### Vault 笔记全文搜索性能 - -**现状**：`devkit_vault_search` 在内存中对所有笔记做线性扫描 + 字符串匹配。 - -**影响**：Vault 笔记数量 >1000 时，搜索延迟可能超过 1s。 - -**建议**：为 Vault 内容建立 Tantivy 索引（复用现有 symbol_index 基础设施），或至少增加关键词索引表。 - --- ## P3 — 文档与可观测性 -### 性能基准缺失 - -**现状**：Criterion 已列为 dev-dependency，但无实际 benchmark 套件。 - -**建议**：为 Index、Query、VaultSearch 建立 Criterion benchmarks，记录基线到 CI 产物。 +当前无活跃 P3 债务。 ## 已解决（归档） -| 问题 | 解决版本 | Commit | -|------|----------|--------| +| 问题 | 解决版本 | Commit / 实现 | +|------|----------|---------------| +| Vault 笔记全文搜索性能 | Unreleased | `devkit_vault_search` 优先使用 Tantivy BM25（`search_vault_at`），回退内存扫描；`reindex_vault_with_writer` 与仓库索引同 writer | +| 性能基准缺失 | Unreleased | 新增 `benches/vault_bench.rs`，保存 Criterion baseline `v0.20.1` | | `relations` 表零生产读取路径 | v0.20.1 | `devkit_relation_store/query/delete` + `project_context` 读取 | | Workflow 引擎零 MCP 暴露 | v0.20.1 | `devkit_workflow_list/run/status` | | `project_context` 不完整 | v0.20.1 | 补充 `known_limits` + `skills` | diff --git a/README.md b/README.md index d022aec..9a35c6b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ 一套引擎，统一代码上下文、知识记忆与智能体推理。 [![Version](https://img.shields.io/badge/version-v0.20.1-blue)](https://github.com/juice094/devbase/releases) -[![Tests](https://img.shields.io/badge/tests-605%2B%20passed-brightgreen)](https://github.com/juice094/devbase/actions) +[![Tests](https://img.shields.io/badge/tests-616%2B%20passed-brightgreen)](https://github.com/juice094/devbase/actions) [![Clippy](https://img.shields.io/badge/clippy-0%20warnings-green)](https://github.com/juice094/devbase/actions) [![License](https://img.shields.io/badge/license-AGPL--3.0%20%2F%20Commercial-orange)](LICENSE) [![Rust](https://img.shields.io/badge/rust-1.95%2B-9cf)](https://www.rust-lang.org) @@ -36,7 +36,7 @@ devbase 将代码库、笔记与工作流编译为 AI 可推理的结构化情 | 📊 **TUI 仪表盘** | ratatui 终端界面：跨仓库搜索、安全同步、Skill/Workflow 发现 | | 🔌 **71 个 MCP 工具** | stdio 本地进程通信：仓库管理、代码分析、知识图谱、智能体记忆 | | 🏠 **本地优先** | 零数据离开本机 — SQLite + Tantivy + tree-sitter，无需云端 | -| 🔍 **混合检索** | BM25 全文 + FTS5 技能搜索 + 纯 SQL 向量搜索（`cosine_similarity` UDF），零 ML 运行时依赖 | +| 🔍 **混合检索** | BM25 全文（仓库 + Vault）+ FTS5 技能搜索 + 纯 SQL 向量搜索（`cosine_similarity` UDF），零 ML 运行时依赖 | > [完整 71 个 Tool 矩阵 → docs/reference/mcp-tools.md](docs/reference/mcp-tools.md) diff --git a/benches/vault_bench.rs b/benches/vault_bench.rs new file mode 100644 index 0000000..68546df --- /dev/null +++ b/benches/vault_bench.rs @@ -0,0 +1,107 @@ +use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; +use devbase::registry::VaultNote; +use devbase::search::{commit_writer, get_writer, init_index_at, search_vault_at}; +use devbase::vault::indexer::reindex_vault_core; + +fn generate_notes(root: &std::path::Path, count: usize) -> Vec { + let mut notes = Vec::with_capacity(count); + for i in 0..count { + let id = format!("note-{}.md", i); + let path = root.join(&id); + let title = format!("Vault Note {}", i); + let tags = if i % 3 == 0 { + vec!["rust".to_string(), "architecture".to_string()] + } else if i % 3 == 1 { + vec!["design".to_string()] + } else { + vec!["cli".to_string(), "rust".to_string()] + }; + let body_tag = if i % 2 == 0 { "rust" } else { "architecture" }; + let content = format!( + "---\ntitle: {}\ntags: [{}]\n---\n\n# {}\n\nThis is content for note {}. It discusses {} patterns and design decisions.\n", + title, + tags.join(", "), + title, + i, + body_tag + ); + std::fs::write(&path, content).unwrap(); + notes.push(VaultNote { + id, + path: path.to_string_lossy().to_string(), + title: Some(title), + content: String::new(), + frontmatter: None, + tags, + outgoing_links: vec![], + block_refs: vec![], + linked_repo: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + }); + } + notes +} + +fn bench_reindex_vault_core(c: &mut Criterion) { + let mut group = c.benchmark_group("reindex_vault"); + group.sample_size(50); + + for count in [50usize, 200usize] { + let notes_tmp = tempfile::tempdir().unwrap(); + let notes = generate_notes(notes_tmp.path(), count); + + let index_tmp = tempfile::tempdir().unwrap(); + let (index, _reader) = init_index_at(index_tmp.path()).unwrap(); + let mut writer = get_writer(&index).unwrap(); + let schema = index.schema(); + + group.bench_with_input(BenchmarkId::from_parameter(count), ¬es, |b, notes| { + b.iter(|| { + reindex_vault_core(notes, &mut writer, &schema).unwrap(); + commit_writer(&mut writer).unwrap(); + black_box(&writer); + }); + }); + } + + group.finish(); +} + +fn bench_search_vault_at(c: &mut Criterion) { + let mut group = c.benchmark_group("search_vault"); + group.sample_size(50); + + for count in [50usize, 200usize] { + let notes_tmp = tempfile::tempdir().unwrap(); + let notes = generate_notes(notes_tmp.path(), count); + + let index_tmp = tempfile::tempdir().unwrap(); + let index_path = index_tmp.path().to_path_buf(); + let (index, _reader) = init_index_at(&index_path).unwrap(); + let mut writer = get_writer(&index).unwrap(); + let schema = index.schema(); + reindex_vault_core(¬es, &mut writer, &schema).unwrap(); + commit_writer(&mut writer).unwrap(); + drop(writer); + drop(index); + + for (label, query) in [("single", "rust"), ("multi", "rust design")] { + group.bench_with_input( + BenchmarkId::new(format!("{}_docs", count), label), + &(index_path.clone(), query), + |b, (path, q)| { + b.iter(|| { + let results = search_vault_at(path, q, 10).unwrap(); + black_box(results); + }); + }, + ); + } + } + + group.finish(); +} + +criterion_group!(benches, bench_reindex_vault_core, bench_search_vault_at); +criterion_main!(benches); diff --git a/docs/README.md b/docs/README.md index fd2b54c..ee5202d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # devbase 文档导航 -> **项目状态**：v0.20.1 · Schema v36 · 71 MCP tools · 605 tests · 12 workspace crates +> **项目状态**：v0.20.1 · Schema v36 · 71 MCP tools · 616 tests · 12 workspace crates > **权威入口**：[`AGENTS.md`](../AGENTS.md)（Agent 环境指引）· [`CHANGELOG.md`](../CHANGELOG.md)（版本变更） > **最后整理**：2026-06-13 @@ -12,7 +12,7 @@ |------|------|------| | 版本 | v0.20.1 | `Cargo.toml` | | Rust Edition | 2024 | `Cargo.toml` | -| 测试 | 605 passed / 0 failed / 7 ignored | `cargo test --workspace -- --list` | +| 测试 | 616 passed / 0 failed / 6 ignored | `cargo test --workspace -- --list` | | Clippy | `-D warnings` 全绿 | CI | | Schema | v36 | `src/registry/migrate.rs` | | MCP Tools | **71**（5 Stable / 62 Beta / 4 Experimental） | `src/mcp/mod.rs` | diff --git a/docs/reference/stable-tools/vault_search.md b/docs/reference/stable-tools/vault_search.md index eece042..67da0a4 100644 --- a/docs/reference/stable-tools/vault_search.md +++ b/docs/reference/stable-tools/vault_search.md @@ -1,84 +1,82 @@ -# devkit_vault_search - -> **Tier**: Stable (frozen at v0.21.0) -> **Source**: `src/mcp/tools/vault.rs` — `DevkitVaultSearchTool` - -Search the devbase Vault (Markdown notes) by keywords across note titles, tags, and full content. - -## Purpose - -- Find notes related to a topic, architecture decision, or project -- Discover linked concepts via tags or wikilinks -- Locate a note when you only remember fragments of its content -- Check if a topic has been documented before writing a new note - -## When NOT to use - -- Reading the full content of a known note → use `devkit_vault_read` -- Writing or updating notes → use `devkit_vault_write` -- Finding backlinks to a specific note → use `devkit_vault_backlinks` -- Searching across code repositories → use `devkit_query_repos` - -## Input Schema - -```json -{ - "type": "object", - "properties": { - "query": { "type": "string", "description": "Search keywords" } - }, - "required": ["query"] -} -``` - -| Parameter | Type | Required | Default | Description | -|-----------|--------|----------|---------|--------------------------------| -| `query` | string | Yes | — | Space-separated keywords (AND) | - -## Matching behavior - -- All keywords must match (AND logic) -- Case-insensitive matching across: - - Note ID - - Note title - - Tags (comma-joined) - - Full Markdown body content -- No stemming or fuzzy matching — exact substring only - -## Output Schema - -```json -{ - "success": true, - "count": 2, - "query": "mcp integration", - "notes": [ - { - "id": "mcp-integration-guide", - "title": "MCP Integration Guide", - "path": "references/mcp-integration.md", - "tags": ["mcp", "integration", "architecture"] - } - ] -} -``` - -| Field | Type | Description | -|---------|----------|------------------------------------------| -| `id` | string | Note identifier (usually filename stem) | -| `title` | string | Parsed from YAML frontmatter | -| `path` | string | Vault-relative file path | -| `tags` | string[] | Parsed from YAML frontmatter | - -## Errors - -| Error | Cause | -|--------------------|------------------------------------------| -| `query required` | Missing or empty `query` argument | -| Vault unreadable | Vault directory missing or permission denied | - -## Changelog - -| Version | Change | -|---------|------------------------------------------| -| v0.21.0 | Schema frozen as Stable | +# devkit_vault_search + +> **Tier**: Stable (frozen at v0.21.0) +> **Source**: `src/mcp/tools/vault.rs` — `DevkitVaultSearchTool` + +Search the devbase Vault (Markdown notes) by keywords across note titles, tags, and full content using Tantivy BM25 full-text search. + +## Purpose + +- Find notes related to a topic, architecture decision, or project +- Discover linked concepts via tags or wikilinks +- Locate a note when you only remember fragments of its content +- Check if a topic has been documented before writing a new note + +## When NOT to use + +- Reading the full content of a known note → use `devkit_vault_read` +- Writing or updating notes → use `devkit_vault_write` +- Finding backlinks to a specific note → use `devkit_vault_backlinks` +- Searching across code repositories → use `devkit_query_repos` + +## Input Schema + +```json +{ + "type": "object", + "properties": { + "query": { "type": "string", "description": "Search keywords" } + }, + "required": ["query"] +} +``` + +| Parameter | Type | Required | Default | Description | +|-----------|--------|----------|---------|--------------------------------| +| `query` | string | Yes | — | Space-separated keywords (AND) | + +## Matching behavior + +- Primary path: Tantivy BM25 full-text search across note title, tags, and full Markdown body content. +- Fallback path: if the Tantivy index is empty/unavailable, case-insensitive substring scan across the same fields. +- All keywords must match (AND logic). +- Run `devbase vault reindex` or `devkit_index` to build/update the search index. + +## Output Schema + +```json +{ + "success": true, + "count": 2, + "query": "mcp integration", + "notes": [ + { + "id": "mcp-integration-guide", + "title": "MCP Integration Guide", + "path": "references/mcp-integration.md", + "tags": ["mcp", "integration", "architecture"] + } + ] +} +``` + +| Field | Type | Description | +|---------|----------|------------------------------------------| +| `id` | string | Note identifier (usually filename stem) | +| `title` | string | Parsed from YAML frontmatter | +| `path` | string | Vault-relative file path | +| `tags` | string[] | Parsed from YAML frontmatter | + +## Errors + +| Error | Cause | +|--------------------|------------------------------------------| +| `query required` | Missing or empty `query` argument | +| Vault unreadable | Vault directory missing or permission denied | + +## Changelog + +| Version | Change | +|---------|------------------------------------------| +| Unreleased | Primary search path switched to Tantivy BM25; substring scan retained as fallback | +| v0.21.0 | Schema frozen as Stable | diff --git a/src/knowledge_engine/index.rs b/src/knowledge_engine/index.rs index b20964f..7e969f3 100644 --- a/src/knowledge_engine/index.rs +++ b/src/knowledge_engine/index.rs @@ -1,597 +1,606 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2026 juice094 -use crate::registry::RepoEntry; -use std::path::PathBuf; -use tantivy::{IndexWriter, schema::Schema}; - -fn index_repo_core( - conn: &mut rusqlite::Connection, - repo: &crate::registry::RepoEntry, - config: Option<&crate::config::Config>, - writer: &mut IndexWriter, - schema: &Schema, -) -> anyhow::Result<()> { - use tracing::{info, warn}; - - let (summary, keywords) = config - .as_ref() - .and_then(|cfg| super::try_llm_summary(&repo.local_path, &cfg.llm)) - .or_else(|| super::extract_readme_summary(&repo.local_path).map(|(s, k)| (s, k.join(", ")))) - .unwrap_or_else(|| { - warn!("No README found for {}, generating fallback summary", repo.id); - super::generate_fallback_summary(&repo.local_path) - }); - - let modules = super::extract_module_structure(&repo.local_path); - - crate::registry::knowledge::save_summary(conn, &repo.id, &summary, &keywords)?; - - if let Err(e) = crate::search::delete_repo_doc(writer, schema, &repo.id).and_then(|_| { - crate::search::add_repo_doc(writer, schema, &repo.id, &summary, &keywords, &repo.tags) - }) { - warn!("Failed to index repo in search: {}", e); - } - - let modules_tuple: Vec<(String, String)> = - modules.into_iter().map(|m| (m.name, m.kind)).collect(); - crate::registry::knowledge::save_modules(conn, &repo.id, &modules_tuple)?; - - let detected_lang = crate::scan::detect_language(&repo.local_path); - if let Some(ref lang) = detected_lang { - crate::registry::repo::update_repo_language(conn, &repo.id, Some(lang))?; - } - - info!( - "Indexed [{}] -> \"{}\" (keywords: {}) language={:?}", - repo.id, summary, keywords, detected_lang - ); - Ok(()) -} - -/// Index a single repo with a standalone Tantivy writer. -/// Suitable for one-off indexing where writer reuse is not needed. -pub fn index_repo( - conn: &mut rusqlite::Connection, - repo: &crate::registry::RepoEntry, - config: Option<&crate::config::Config>, -) -> anyhow::Result<()> { - let (index, _reader) = crate::search::init_index()?; - let mut writer = crate::search::get_writer(&index)?; - let schema = index.schema(); - index_repo_core(conn, repo, config, &mut writer, &schema)?; - crate::search::commit_writer(&mut writer)?; - Ok(()) -} - -/// Index a single repo reusing an existing Tantivy writer. -/// Callers must commit the writer after the batch. -pub fn index_repo_with_writer( - conn: &mut rusqlite::Connection, - repo: &crate::registry::RepoEntry, - config: Option<&crate::config::Config>, - writer: &mut IndexWriter, - schema: &Schema, -) -> anyhow::Result<()> { - index_repo_core(conn, repo, config, writer, schema) -} - -/// 兼容旧调用的包装层：执行索引逻辑 -pub fn run_index( - conn: &mut rusqlite::Connection, - path: &str, - skip_embeddings: bool, -) -> anyhow::Result { - run_index_with_progress(conn, path, None, skip_embeddings) -} - -/// Resolve the list of repositories to index for a given path. -/// If `path` is empty, returns all registered repos. -/// If `path` points to an unregistered repo, auto-registers it before returning. -fn prepare_repos(conn: &mut rusqlite::Connection, path: &str) -> anyhow::Result> { - use tracing::info; - - if path.is_empty() { - return crate::registry::repo::list_repos(conn); - } - - let p = PathBuf::from(path); - if !p.exists() { - anyhow::bail!("Path does not exist: {}", path); - } - let registered = crate::registry::repo::list_repos(conn)?; - if let Some(repo) = registered.into_iter().find(|r| r.local_path == p) { - Ok(vec![repo]) - } else { - info!("Registering {} before indexing", path); - let repo = crate::scan::inspect_repo(&p, None)?; - crate::registry::repo::save_repo(conn, &repo)?; - Ok(vec![repo]) - } -} - -/// 带进度上报的索引逻辑。 -/// `progress_tx` 接收阶段性进度消息，用于 MCP streaming 等实时反馈场景。 -pub fn run_index_with_progress( - conn: &mut rusqlite::Connection, - path: &str, - progress_tx: Option>, - skip_embeddings: bool, -) -> anyhow::Result { - use tracing::{info, warn}; - - let notify = |msg: String| { - if let Some(ref tx) = progress_tx { - let _ = tx.send(msg); - } - }; - - let repos = prepare_repos(conn, path)?; - - // Initialize Tantivy search index writer once for the batch - let (search_index, _reader) = crate::search::init_index()?; - let mut search_writer = crate::search::get_writer(&search_index)?; - let search_schema = search_index.schema(); - - // Load orphan list for lazy repair; delete_repo_doc below will clean them. - let orphaned_repos: Vec = conn - .prepare("SELECT repo_id FROM orphan_tantivy_docs")? - .query_map([], |row| row.get::<_, String>(0))? - .filter_map(Result::ok) - .collect(); - - let config = crate::config::Config::load().ok(); - let mut count = 0; - for repo in &repos { - let t0 = std::time::Instant::now(); - let (summary, keywords) = config - .as_ref() - .and_then(|cfg| super::try_llm_summary(&repo.local_path, &cfg.llm)) - .or_else(|| { - super::extract_readme_summary(&repo.local_path).map(|(s, k)| (s, k.join(", "))) - }) - .unwrap_or_else(|| { - warn!("No README found for {}, generating fallback summary", repo.id); - super::generate_fallback_summary(&repo.local_path) - }); - let t1 = std::time::Instant::now(); - - let modules = super::extract_module_structure(&repo.local_path); - let t2 = std::time::Instant::now(); - - crate::registry::knowledge::save_summary(conn, &repo.id, &summary, &keywords)?; - - // Add/update repo document in Tantivy index - crate::search::delete_repo_doc(&mut search_writer, &search_schema, &repo.id)?; - crate::search::add_repo_doc( - &mut search_writer, - &search_schema, - &repo.id, - &summary, - &keywords, - &repo.tags, - )?; - let t3 = std::time::Instant::now(); - - let modules_tuple: Vec<(String, String)> = - modules.into_iter().map(|m| (m.name, m.kind)).collect(); - crate::registry::knowledge::save_modules(conn, &repo.id, &modules_tuple)?; - - let detected_lang = crate::scan::detect_language(&repo.local_path); - if let Some(ref lang) = detected_lang { - crate::registry::repo::update_repo_language(conn, &repo.id, Some(lang))?; - } - - // Determine incremental vs full index - let changed_opt = detect_changes(conn, repo); - if let Some(ref changed) = changed_opt - && changed.added.is_empty() - && changed.modified.is_empty() - && changed.deleted.is_empty() - { - println!("[{}] Already up-to-date", repo.id); - count += 1; - continue; - } - let is_incremental = changed_opt.is_some(); - notify(format!("detect_changes:{},incremental={}", repo.id, is_incremental)); - - // Semantic code indexing (tree-sitter AST extraction + call graph) - let (symbols, calls) = if let Some(ref changed) = changed_opt { - // Incremental: delete old symbols for modified/deleted files - let files_to_delete: Vec = - changed.modified.iter().chain(changed.deleted.iter()).cloned().collect(); - if !files_to_delete.is_empty() { - let _ = crate::semantic_index::persist::delete_symbols_for_files( - conn, - &repo.id, - &files_to_delete, - ); - } - crate::semantic_index::index_repo_incremental(&repo.local_path, changed) - } else { - // Full index - crate::semantic_index::index_repo_full(&repo.local_path)? - }; - let t4 = std::time::Instant::now(); - - if !symbols.is_empty() { - let result = if is_incremental { - crate::semantic_index::persist::save_symbols_incremental(conn, &repo.id, &symbols) - } else { - crate::semantic_index::save_symbols(conn, &repo.id, &symbols) - }; - match result { - Ok(n) => { - info!("Saved {} code symbols for {}", n, repo.id); - notify(format!("semantic_index:{},symbols={}", repo.id, n)); - } - Err(e) => warn!("Failed to save code symbols for {}: {}", repo.id, e), - } - - // Index symbols in Tantivy for BM25 keyword search - if let Err(e) = index_symbols_in_search(&repo.id, &symbols, is_incremental) { - warn!("Failed to index symbols in search for {}: {}", repo.id, e); - } else { - notify(format!("symbol_index:{},symbols={}", repo.id, symbols.len())); - } - } - if !calls.is_empty() { - let result = if is_incremental { - crate::semantic_index::persist::save_calls_incremental(conn, &repo.id, &calls) - } else { - crate::semantic_index::save_calls(conn, &repo.id, &calls) - }; - match result { - Ok(n) => { - info!("Saved {} call edges for {}", n, repo.id); - notify(format!("call_graph:{},calls={}", repo.id, n)); - } - Err(e) => warn!("Failed to save call graph for {}: {}", repo.id, e), - } - } - let t5 = std::time::Instant::now(); - - // Generate embeddings for code symbols (local candle, Sprint 14) - if !skip_embeddings && !symbols.is_empty() { - let result = if is_incremental { - save_symbol_embeddings_incremental(conn, &repo.id, &symbols) - } else { - save_symbol_embeddings(conn, &repo.id, &symbols) - }; - match result { - Ok(n) => { - info!("Saved {} symbol embeddings for {}", n, repo.id); - notify(format!("embeddings:{},count={}", repo.id, n)); - } - Err(e) => warn!("Failed to save symbol embeddings for {}: {}", repo.id, e), - } - } - let t6 = std::time::Instant::now(); - - // Save repo_index_state for next incremental run - if let Ok(Some(hash)) = crate::semantic_index::git_diff::current_head_hash(&repo.local_path) - { - let _ = save_repo_index_state(conn, &repo.id, &hash); - } - - // Cross-repo dependency graph - match crate::dependency_graph::build_dependency_graph(conn, &repo.id, &repo.local_path) { - Ok(n) => { - if n > 0 { - info!("Resolved {} local dependencies for {}", n, repo.id); - } - notify(format!("dependency_graph:{},count={}", repo.id, n)); - } - Err(e) => warn!("Failed to build dependency graph for {}: {}", repo.id, e), - } - let t7 = std::time::Instant::now(); - - println!( - "Indexed [{}] -> \"{}\" (keywords: {}) language={:?} symbols={} calls={}", - repo.id, - summary, - keywords, - detected_lang, - symbols.len(), - calls.len(), - ); - println!( - " timings: readme={:.0}ms module={:.0}ms tantivy={:.0}ms semantic={:.0}ms save={:.0}ms embed={:.0}ms deps={:.0}ms total={:.0}ms", - (t1 - t0).as_millis(), - (t2 - t1).as_millis(), - (t3 - t2).as_millis(), - (t4 - t3).as_millis(), - (t5 - t4).as_millis(), - (t6 - t5).as_millis(), - (t7 - t6).as_millis(), - (t7 - t0).as_millis(), - ); - count += 1; - } - - crate::search::commit_writer(&mut search_writer)?; - notify("tantivy_commit".to_string()); - - // Clean up orphan records for repos that were successfully indexed this run. - if count > 0 && !orphaned_repos.is_empty() { - let indexed_ids: std::collections::HashSet<&str> = - repos.iter().map(|r| r.id.as_str()).collect(); - for orphan_id in &orphaned_repos { - if indexed_ids.contains(orphan_id.as_str()) { - let _ = - conn.execute("DELETE FROM orphan_tantivy_docs WHERE repo_id = ?1", [orphan_id]); - } - } - } - - println!("\nIndexed {} repositories.", count); - Ok(count) -} - -/// Parallel embedding generation for code symbols. -/// Phase 1: CPU-intensive encoding across all available cores (rayon). -/// Phase 2: Single-threaded SQLite batch write to avoid lock contention. -fn generate_and_save_embeddings( - conn: &mut rusqlite::Connection, - repo_id: &str, - symbols: &[crate::semantic_index::CodeSymbol], - clear_existing: bool, -) -> anyhow::Result { - use rayon::prelude::*; - use tracing::{info, warn}; - - // Phase 1: parallel encoding (rayon par_iter gives best throughput for - // Candle CPU BERT because per-symbol sequences are short and variable; - // batching causes excessive padding and Candle's CPU matmul is slower - // for large padded batches than many small single inferences). - let items: Vec<(String, String, Vec)> = symbols - .par_iter() - .filter_map(|sym| { - let text = format!("{} {}", sym.name, sym.signature.as_deref().unwrap_or("")); - match crate::embedding::generate_query_embedding(&text) { - Ok(emb) => { - let fp = sym.file_path.to_string_lossy().to_string(); - Some((fp, sym.name.clone(), emb)) - } - Err(e) => { - warn!("Embedding generation failed for '{}': {}", sym.name, e); - None - } - } - }) - .collect(); - - // Phase 2: single-threaded batch write - let tx = conn.transaction()?; - if clear_existing { - tx.execute("DELETE FROM code_embeddings WHERE repo_id = ?1", [repo_id])?; - } - let now = chrono::Utc::now().to_rfc3339(); - let mut inserted = 0usize; - for (file_path, name, embedding) in items { - let blob = crate::embedding::embedding_to_bytes(&embedding); - let sql = "INSERT INTO code_embeddings (repo_id, file_path, symbol_name, embedding, generated_at) VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT(repo_id, file_path, symbol_name) DO UPDATE SET - embedding = excluded.embedding, - generated_at = excluded.generated_at"; - match tx.execute(sql, rusqlite::params![repo_id, &file_path, &name, &blob, &now]) { - Ok(_) => inserted += 1, - Err(e) => warn!("Failed to insert embedding for {}: {}", name, e), - } - } - tx.commit()?; - - let mode = if clear_existing { "" } else { " (incremental)" }; - info!("Saved {} symbol embeddings{} for {}", inserted, mode, repo_id); - Ok(inserted) -} - -fn save_symbol_embeddings( - conn: &mut rusqlite::Connection, - repo_id: &str, - symbols: &[crate::semantic_index::CodeSymbol], -) -> anyhow::Result { - generate_and_save_embeddings(conn, repo_id, symbols, true) -} - -fn save_symbol_embeddings_incremental( - conn: &mut rusqlite::Connection, - repo_id: &str, - symbols: &[crate::semantic_index::CodeSymbol], -) -> anyhow::Result { - generate_and_save_embeddings(conn, repo_id, symbols, false) -} - -/// Detect whether a repo can be incrementally indexed. -/// Returns `Some(ChangedFiles)` if incremental is possible and worthwhile. -/// Returns `None` for first-time index, non-Git repos, too many changes, or errors. -fn detect_changes( - conn: &rusqlite::Connection, - repo: &crate::registry::RepoEntry, -) -> Option { - use tracing::{info, warn}; - - match super::index_state::get_repo_index_state(conn, repo) { - super::index_state::IndexState::Fresh => { - Some(crate::semantic_index::git_diff::ChangedFiles { - added: vec![], - modified: vec![], - deleted: vec![], - }) - } - super::index_state::IndexState::Stale { added, modified, deleted } => { - let total = added.len() + modified.len() + deleted.len(); - if total > 100 { - info!( - "Repo {} has {} changed files (>100 threshold), falling back to full index", - repo.id, total - ); - return None; - } - info!( - "Repo {}: incremental index ({} added, {} modified, {} deleted)", - repo.id, - added.len(), - modified.len(), - deleted.len() - ); - Some(crate::semantic_index::git_diff::ChangedFiles { added, modified, deleted }) - } - super::index_state::IndexState::Missing => { - info!("Repo {}: no prior index state, falling back to full index", repo.id); - None - } - super::index_state::IndexState::Unknown { ref reason } => { - warn!("Repo {}: index state unknown ({}), falling back to full index", repo.id, reason); - None - } - } -} - -fn index_symbols_in_search( - repo_id: &str, - symbols: &[crate::semantic_index::CodeSymbol], - _is_incremental: bool, -) -> anyhow::Result<()> { - let (index, _reader) = crate::search::symbol_index::init_index()?; - let mut writer = crate::search::symbol_index::get_writer(&index)?; - let schema = index.schema(); - crate::search::symbol_index::add_symbols(&mut writer, &schema, repo_id, symbols)?; - crate::search::symbol_index::commit_writer(&mut writer)?; - Ok(()) -} - -fn save_repo_index_state( - conn: &mut rusqlite::Connection, - repo_id: &str, - hash: &str, -) -> anyhow::Result<()> { - conn.execute( - "INSERT OR REPLACE INTO repo_index_state (repo_id, last_commit_hash, indexed_at) - VALUES (?1, ?2, datetime('now'))", - [repo_id, hash], - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::registry::RepoEntry; - use crate::registry::test_helpers::WorkspaceRegistry; - use std::path::Path; - - fn init_git_repo(path: &Path) -> git2::Repository { - let repo = git2::Repository::init(path).unwrap(); - let mut config = repo.config().unwrap(); - config.set_str("user.name", "Test").unwrap(); - config.set_str("user.email", "test@example.com").unwrap(); - let sig = repo.signature().unwrap(); - let tree_id = { - let mut index = repo.index().unwrap(); - index.write_tree().unwrap() - }; - let tree = repo.find_tree(tree_id).unwrap(); - repo.commit(Some("HEAD"), &sig, &sig, "initial", &tree, &[]).unwrap(); - drop(tree); - repo - } - - #[test] - fn test_prepare_repos_empty_path_returns_all() -> anyhow::Result<()> { - let mut conn = WorkspaceRegistry::init_in_memory()?; - let _ = WorkspaceRegistry::seed_test_repo(&mut conn, "repo1")?; - let _ = WorkspaceRegistry::seed_test_repo(&mut conn, "repo2")?; - - let repos = prepare_repos(&mut conn, "")?; - assert_eq!(repos.len(), 2); - assert!(repos.iter().any(|r| r.id == "repo1")); - assert!(repos.iter().any(|r| r.id == "repo2")); - Ok(()) - } - - #[test] - fn test_prepare_repos_matching_path_returns_one() -> anyhow::Result<()> { - let mut conn = WorkspaceRegistry::init_in_memory()?; - let tmp = tempfile::tempdir()?; - let path = tmp.path().join("myrepo"); - std::fs::create_dir(&path)?; - - let repo = RepoEntry { - id: "myrepo".to_string(), - local_path: path.clone(), - tags: vec![], - language: Some("rust".to_string()), - discovered_at: chrono::Utc::now(), - workspace_type: "git".to_string(), - data_tier: "private".to_string(), - last_synced_at: None, - stars: None, - remotes: vec![], - }; - crate::registry::repo::save_repo(&mut conn, &repo)?; - - let repos = prepare_repos(&mut conn, path.to_str().unwrap())?; - assert_eq!(repos.len(), 1); - assert_eq!(repos[0].id, "myrepo"); - Ok(()) - } - - #[test] - fn test_prepare_repos_nonexistent_path_errors() -> anyhow::Result<()> { - let mut conn = WorkspaceRegistry::init_in_memory()?; - let result = prepare_repos(&mut conn, "/nonexistent/path/12345"); - assert!(result.is_err()); - Ok(()) - } - - #[test] - fn test_prepare_repos_unregistered_existing_path_auto_registers() -> anyhow::Result<()> { - let mut conn = WorkspaceRegistry::init_in_memory()?; - let tmp = tempfile::tempdir()?; - let path = tmp.path().join("unregistered"); - std::fs::create_dir(&path)?; - let _ = init_git_repo(&path); - - let repos = prepare_repos(&mut conn, path.to_str().unwrap())?; - assert_eq!(repos.len(), 1); - // Use file_name comparison to avoid Windows short-name (8.3) path mismatches - assert_eq!(repos[0].local_path.file_name(), path.file_name()); - assert!(repos[0].local_path.exists()); - // Verify it was saved to registry - let all = crate::registry::repo::list_repos(&conn)?; - assert_eq!(all.len(), 1); - Ok(()) - } - - #[test] - fn test_save_and_get_repo_index_state() -> anyhow::Result<()> { - let mut conn = WorkspaceRegistry::init_in_memory()?; - let tmp = tempfile::tempdir()?; - let path = tmp.path().join("gitrepo"); - std::fs::create_dir(&path)?; - let repo = git2::Repository::init(&path)?; - let mut config = repo.config().unwrap(); - config.set_str("user.name", "Test").unwrap(); - config.set_str("user.email", "test@example.com").unwrap(); - let sig = repo.signature().unwrap(); - let tree_id = { - let mut index = repo.index().unwrap(); - index.write_tree().unwrap() - }; - let tree = repo.find_tree(tree_id).unwrap(); - let oid = repo.commit(Some("HEAD"), &sig, &sig, "initial", &tree, &[])?; - - save_repo_index_state(&mut conn, "test-repo", &oid.to_string())?; - - let hash: Option = conn - .query_row( - "SELECT last_commit_hash FROM repo_index_state WHERE repo_id = ?1", - ["test-repo"], - |row| row.get(0), - ) - .unwrap_or(None); - assert_eq!(hash, Some(oid.to_string())); - Ok(()) - } -} +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 juice094 +use crate::registry::RepoEntry; +use std::path::PathBuf; +use tantivy::{IndexWriter, schema::Schema}; + +fn index_repo_core( + conn: &mut rusqlite::Connection, + repo: &crate::registry::RepoEntry, + config: Option<&crate::config::Config>, + writer: &mut IndexWriter, + schema: &Schema, +) -> anyhow::Result<()> { + use tracing::{info, warn}; + + let (summary, keywords) = config + .as_ref() + .and_then(|cfg| super::try_llm_summary(&repo.local_path, &cfg.llm)) + .or_else(|| super::extract_readme_summary(&repo.local_path).map(|(s, k)| (s, k.join(", ")))) + .unwrap_or_else(|| { + warn!("No README found for {}, generating fallback summary", repo.id); + super::generate_fallback_summary(&repo.local_path) + }); + + let modules = super::extract_module_structure(&repo.local_path); + + crate::registry::knowledge::save_summary(conn, &repo.id, &summary, &keywords)?; + + if let Err(e) = crate::search::delete_repo_doc(writer, schema, &repo.id).and_then(|_| { + crate::search::add_repo_doc(writer, schema, &repo.id, &summary, &keywords, &repo.tags) + }) { + warn!("Failed to index repo in search: {}", e); + } + + let modules_tuple: Vec<(String, String)> = + modules.into_iter().map(|m| (m.name, m.kind)).collect(); + crate::registry::knowledge::save_modules(conn, &repo.id, &modules_tuple)?; + + let detected_lang = crate::scan::detect_language(&repo.local_path); + if let Some(ref lang) = detected_lang { + crate::registry::repo::update_repo_language(conn, &repo.id, Some(lang))?; + } + + info!( + "Indexed [{}] -> \"{}\" (keywords: {}) language={:?}", + repo.id, summary, keywords, detected_lang + ); + Ok(()) +} + +/// Index a single repo with a standalone Tantivy writer. +/// Suitable for one-off indexing where writer reuse is not needed. +pub fn index_repo( + conn: &mut rusqlite::Connection, + repo: &crate::registry::RepoEntry, + config: Option<&crate::config::Config>, +) -> anyhow::Result<()> { + let (index, _reader) = crate::search::init_index()?; + let mut writer = crate::search::get_writer(&index)?; + let schema = index.schema(); + index_repo_core(conn, repo, config, &mut writer, &schema)?; + crate::search::commit_writer(&mut writer)?; + Ok(()) +} + +/// Index a single repo reusing an existing Tantivy writer. +/// Callers must commit the writer after the batch. +pub fn index_repo_with_writer( + conn: &mut rusqlite::Connection, + repo: &crate::registry::RepoEntry, + config: Option<&crate::config::Config>, + writer: &mut IndexWriter, + schema: &Schema, +) -> anyhow::Result<()> { + index_repo_core(conn, repo, config, writer, schema) +} + +/// 兼容旧调用的包装层：执行索引逻辑 +pub fn run_index( + conn: &mut rusqlite::Connection, + path: &str, + skip_embeddings: bool, +) -> anyhow::Result { + run_index_with_progress(conn, path, None, skip_embeddings) +} + +/// Resolve the list of repositories to index for a given path. +/// If `path` is empty, returns all registered repos. +/// If `path` points to an unregistered repo, auto-registers it before returning. +fn prepare_repos(conn: &mut rusqlite::Connection, path: &str) -> anyhow::Result> { + use tracing::info; + + if path.is_empty() { + return crate::registry::repo::list_repos(conn); + } + + let p = PathBuf::from(path); + if !p.exists() { + anyhow::bail!("Path does not exist: {}", path); + } + let registered = crate::registry::repo::list_repos(conn)?; + if let Some(repo) = registered.into_iter().find(|r| r.local_path == p) { + Ok(vec![repo]) + } else { + info!("Registering {} before indexing", path); + let repo = crate::scan::inspect_repo(&p, None)?; + crate::registry::repo::save_repo(conn, &repo)?; + Ok(vec![repo]) + } +} + +/// 带进度上报的索引逻辑。 +/// `progress_tx` 接收阶段性进度消息，用于 MCP streaming 等实时反馈场景。 +pub fn run_index_with_progress( + conn: &mut rusqlite::Connection, + path: &str, + progress_tx: Option>, + skip_embeddings: bool, +) -> anyhow::Result { + use tracing::{info, warn}; + + let notify = |msg: String| { + if let Some(ref tx) = progress_tx { + let _ = tx.send(msg); + } + }; + + let repos = prepare_repos(conn, path)?; + + // Initialize Tantivy search index writer once for the batch + let (search_index, _reader) = crate::search::init_index()?; + let mut search_writer = crate::search::get_writer(&search_index)?; + let search_schema = search_index.schema(); + + // Load orphan list for lazy repair; delete_repo_doc below will clean them. + let orphaned_repos: Vec = conn + .prepare("SELECT repo_id FROM orphan_tantivy_docs")? + .query_map([], |row| row.get::<_, String>(0))? + .filter_map(Result::ok) + .collect(); + + let config = crate::config::Config::load().ok(); + let mut count = 0; + for repo in &repos { + let t0 = std::time::Instant::now(); + let (summary, keywords) = config + .as_ref() + .and_then(|cfg| super::try_llm_summary(&repo.local_path, &cfg.llm)) + .or_else(|| { + super::extract_readme_summary(&repo.local_path).map(|(s, k)| (s, k.join(", "))) + }) + .unwrap_or_else(|| { + warn!("No README found for {}, generating fallback summary", repo.id); + super::generate_fallback_summary(&repo.local_path) + }); + let t1 = std::time::Instant::now(); + + let modules = super::extract_module_structure(&repo.local_path); + let t2 = std::time::Instant::now(); + + crate::registry::knowledge::save_summary(conn, &repo.id, &summary, &keywords)?; + + // Add/update repo document in Tantivy index + crate::search::delete_repo_doc(&mut search_writer, &search_schema, &repo.id)?; + crate::search::add_repo_doc( + &mut search_writer, + &search_schema, + &repo.id, + &summary, + &keywords, + &repo.tags, + )?; + let t3 = std::time::Instant::now(); + + let modules_tuple: Vec<(String, String)> = + modules.into_iter().map(|m| (m.name, m.kind)).collect(); + crate::registry::knowledge::save_modules(conn, &repo.id, &modules_tuple)?; + + let detected_lang = crate::scan::detect_language(&repo.local_path); + if let Some(ref lang) = detected_lang { + crate::registry::repo::update_repo_language(conn, &repo.id, Some(lang))?; + } + + // Determine incremental vs full index + let changed_opt = detect_changes(conn, repo); + if let Some(ref changed) = changed_opt + && changed.added.is_empty() + && changed.modified.is_empty() + && changed.deleted.is_empty() + { + println!("[{}] Already up-to-date", repo.id); + count += 1; + continue; + } + let is_incremental = changed_opt.is_some(); + notify(format!("detect_changes:{},incremental={}", repo.id, is_incremental)); + + // Semantic code indexing (tree-sitter AST extraction + call graph) + let (symbols, calls) = if let Some(ref changed) = changed_opt { + // Incremental: delete old symbols for modified/deleted files + let files_to_delete: Vec = + changed.modified.iter().chain(changed.deleted.iter()).cloned().collect(); + if !files_to_delete.is_empty() { + let _ = crate::semantic_index::persist::delete_symbols_for_files( + conn, + &repo.id, + &files_to_delete, + ); + } + crate::semantic_index::index_repo_incremental(&repo.local_path, changed) + } else { + // Full index + crate::semantic_index::index_repo_full(&repo.local_path)? + }; + let t4 = std::time::Instant::now(); + + if !symbols.is_empty() { + let result = if is_incremental { + crate::semantic_index::persist::save_symbols_incremental(conn, &repo.id, &symbols) + } else { + crate::semantic_index::save_symbols(conn, &repo.id, &symbols) + }; + match result { + Ok(n) => { + info!("Saved {} code symbols for {}", n, repo.id); + notify(format!("semantic_index:{},symbols={}", repo.id, n)); + } + Err(e) => warn!("Failed to save code symbols for {}: {}", repo.id, e), + } + + // Index symbols in Tantivy for BM25 keyword search + if let Err(e) = index_symbols_in_search(&repo.id, &symbols, is_incremental) { + warn!("Failed to index symbols in search for {}: {}", repo.id, e); + } else { + notify(format!("symbol_index:{},symbols={}", repo.id, symbols.len())); + } + } + if !calls.is_empty() { + let result = if is_incremental { + crate::semantic_index::persist::save_calls_incremental(conn, &repo.id, &calls) + } else { + crate::semantic_index::save_calls(conn, &repo.id, &calls) + }; + match result { + Ok(n) => { + info!("Saved {} call edges for {}", n, repo.id); + notify(format!("call_graph:{},calls={}", repo.id, n)); + } + Err(e) => warn!("Failed to save call graph for {}: {}", repo.id, e), + } + } + let t5 = std::time::Instant::now(); + + // Generate embeddings for code symbols (local candle, Sprint 14) + if !skip_embeddings && !symbols.is_empty() { + let result = if is_incremental { + save_symbol_embeddings_incremental(conn, &repo.id, &symbols) + } else { + save_symbol_embeddings(conn, &repo.id, &symbols) + }; + match result { + Ok(n) => { + info!("Saved {} symbol embeddings for {}", n, repo.id); + notify(format!("embeddings:{},count={}", repo.id, n)); + } + Err(e) => warn!("Failed to save symbol embeddings for {}: {}", repo.id, e), + } + } + let t6 = std::time::Instant::now(); + + // Save repo_index_state for next incremental run + if let Ok(Some(hash)) = crate::semantic_index::git_diff::current_head_hash(&repo.local_path) + { + let _ = save_repo_index_state(conn, &repo.id, &hash); + } + + // Cross-repo dependency graph + match crate::dependency_graph::build_dependency_graph(conn, &repo.id, &repo.local_path) { + Ok(n) => { + if n > 0 { + info!("Resolved {} local dependencies for {}", n, repo.id); + } + notify(format!("dependency_graph:{},count={}", repo.id, n)); + } + Err(e) => warn!("Failed to build dependency graph for {}: {}", repo.id, e), + } + let t7 = std::time::Instant::now(); + + println!( + "Indexed [{}] -> \"{}\" (keywords: {}) language={:?} symbols={} calls={}", + repo.id, + summary, + keywords, + detected_lang, + symbols.len(), + calls.len(), + ); + println!( + " timings: readme={:.0}ms module={:.0}ms tantivy={:.0}ms semantic={:.0}ms save={:.0}ms embed={:.0}ms deps={:.0}ms total={:.0}ms", + (t1 - t0).as_millis(), + (t2 - t1).as_millis(), + (t3 - t2).as_millis(), + (t4 - t3).as_millis(), + (t5 - t4).as_millis(), + (t6 - t5).as_millis(), + (t7 - t6).as_millis(), + (t7 - t0).as_millis(), + ); + count += 1; + } + + // Index vault notes into Tantivy for full-text search (same writer as repos) + if let Err(e) = + crate::vault::indexer::reindex_vault_with_writer(conn, &mut search_writer, &search_schema) + { + warn!("Failed to index vault notes: {}", e); + } else { + notify("vault_index".to_string()); + } + + crate::search::commit_writer(&mut search_writer)?; + notify("tantivy_commit".to_string()); + + // Clean up orphan records for repos that were successfully indexed this run. + if count > 0 && !orphaned_repos.is_empty() { + let indexed_ids: std::collections::HashSet<&str> = + repos.iter().map(|r| r.id.as_str()).collect(); + for orphan_id in &orphaned_repos { + if indexed_ids.contains(orphan_id.as_str()) { + let _ = + conn.execute("DELETE FROM orphan_tantivy_docs WHERE repo_id = ?1", [orphan_id]); + } + } + } + + println!("\nIndexed {} repositories.", count); + Ok(count) +} + +/// Parallel embedding generation for code symbols. +/// Phase 1: CPU-intensive encoding across all available cores (rayon). +/// Phase 2: Single-threaded SQLite batch write to avoid lock contention. +fn generate_and_save_embeddings( + conn: &mut rusqlite::Connection, + repo_id: &str, + symbols: &[crate::semantic_index::CodeSymbol], + clear_existing: bool, +) -> anyhow::Result { + use rayon::prelude::*; + use tracing::{info, warn}; + + // Phase 1: parallel encoding (rayon par_iter gives best throughput for + // Candle CPU BERT because per-symbol sequences are short and variable; + // batching causes excessive padding and Candle's CPU matmul is slower + // for large padded batches than many small single inferences). + let items: Vec<(String, String, Vec)> = symbols + .par_iter() + .filter_map(|sym| { + let text = format!("{} {}", sym.name, sym.signature.as_deref().unwrap_or("")); + match crate::embedding::generate_query_embedding(&text) { + Ok(emb) => { + let fp = sym.file_path.to_string_lossy().to_string(); + Some((fp, sym.name.clone(), emb)) + } + Err(e) => { + warn!("Embedding generation failed for '{}': {}", sym.name, e); + None + } + } + }) + .collect(); + + // Phase 2: single-threaded batch write + let tx = conn.transaction()?; + if clear_existing { + tx.execute("DELETE FROM code_embeddings WHERE repo_id = ?1", [repo_id])?; + } + let now = chrono::Utc::now().to_rfc3339(); + let mut inserted = 0usize; + for (file_path, name, embedding) in items { + let blob = crate::embedding::embedding_to_bytes(&embedding); + let sql = "INSERT INTO code_embeddings (repo_id, file_path, symbol_name, embedding, generated_at) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(repo_id, file_path, symbol_name) DO UPDATE SET + embedding = excluded.embedding, + generated_at = excluded.generated_at"; + match tx.execute(sql, rusqlite::params![repo_id, &file_path, &name, &blob, &now]) { + Ok(_) => inserted += 1, + Err(e) => warn!("Failed to insert embedding for {}: {}", name, e), + } + } + tx.commit()?; + + let mode = if clear_existing { "" } else { " (incremental)" }; + info!("Saved {} symbol embeddings{} for {}", inserted, mode, repo_id); + Ok(inserted) +} + +fn save_symbol_embeddings( + conn: &mut rusqlite::Connection, + repo_id: &str, + symbols: &[crate::semantic_index::CodeSymbol], +) -> anyhow::Result { + generate_and_save_embeddings(conn, repo_id, symbols, true) +} + +fn save_symbol_embeddings_incremental( + conn: &mut rusqlite::Connection, + repo_id: &str, + symbols: &[crate::semantic_index::CodeSymbol], +) -> anyhow::Result { + generate_and_save_embeddings(conn, repo_id, symbols, false) +} + +/// Detect whether a repo can be incrementally indexed. +/// Returns `Some(ChangedFiles)` if incremental is possible and worthwhile. +/// Returns `None` for first-time index, non-Git repos, too many changes, or errors. +fn detect_changes( + conn: &rusqlite::Connection, + repo: &crate::registry::RepoEntry, +) -> Option { + use tracing::{info, warn}; + + match super::index_state::get_repo_index_state(conn, repo) { + super::index_state::IndexState::Fresh => { + Some(crate::semantic_index::git_diff::ChangedFiles { + added: vec![], + modified: vec![], + deleted: vec![], + }) + } + super::index_state::IndexState::Stale { added, modified, deleted } => { + let total = added.len() + modified.len() + deleted.len(); + if total > 100 { + info!( + "Repo {} has {} changed files (>100 threshold), falling back to full index", + repo.id, total + ); + return None; + } + info!( + "Repo {}: incremental index ({} added, {} modified, {} deleted)", + repo.id, + added.len(), + modified.len(), + deleted.len() + ); + Some(crate::semantic_index::git_diff::ChangedFiles { added, modified, deleted }) + } + super::index_state::IndexState::Missing => { + info!("Repo {}: no prior index state, falling back to full index", repo.id); + None + } + super::index_state::IndexState::Unknown { ref reason } => { + warn!("Repo {}: index state unknown ({}), falling back to full index", repo.id, reason); + None + } + } +} + +fn index_symbols_in_search( + repo_id: &str, + symbols: &[crate::semantic_index::CodeSymbol], + _is_incremental: bool, +) -> anyhow::Result<()> { + let (index, _reader) = crate::search::symbol_index::init_index()?; + let mut writer = crate::search::symbol_index::get_writer(&index)?; + let schema = index.schema(); + crate::search::symbol_index::add_symbols(&mut writer, &schema, repo_id, symbols)?; + crate::search::symbol_index::commit_writer(&mut writer)?; + Ok(()) +} + +fn save_repo_index_state( + conn: &mut rusqlite::Connection, + repo_id: &str, + hash: &str, +) -> anyhow::Result<()> { + conn.execute( + "INSERT OR REPLACE INTO repo_index_state (repo_id, last_commit_hash, indexed_at) + VALUES (?1, ?2, datetime('now'))", + [repo_id, hash], + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::registry::RepoEntry; + use crate::registry::test_helpers::WorkspaceRegistry; + use std::path::Path; + + fn init_git_repo(path: &Path) -> git2::Repository { + let repo = git2::Repository::init(path).unwrap(); + let mut config = repo.config().unwrap(); + config.set_str("user.name", "Test").unwrap(); + config.set_str("user.email", "test@example.com").unwrap(); + let sig = repo.signature().unwrap(); + let tree_id = { + let mut index = repo.index().unwrap(); + index.write_tree().unwrap() + }; + let tree = repo.find_tree(tree_id).unwrap(); + repo.commit(Some("HEAD"), &sig, &sig, "initial", &tree, &[]).unwrap(); + drop(tree); + repo + } + + #[test] + fn test_prepare_repos_empty_path_returns_all() -> anyhow::Result<()> { + let mut conn = WorkspaceRegistry::init_in_memory()?; + let _ = WorkspaceRegistry::seed_test_repo(&mut conn, "repo1")?; + let _ = WorkspaceRegistry::seed_test_repo(&mut conn, "repo2")?; + + let repos = prepare_repos(&mut conn, "")?; + assert_eq!(repos.len(), 2); + assert!(repos.iter().any(|r| r.id == "repo1")); + assert!(repos.iter().any(|r| r.id == "repo2")); + Ok(()) + } + + #[test] + fn test_prepare_repos_matching_path_returns_one() -> anyhow::Result<()> { + let mut conn = WorkspaceRegistry::init_in_memory()?; + let tmp = tempfile::tempdir()?; + let path = tmp.path().join("myrepo"); + std::fs::create_dir(&path)?; + + let repo = RepoEntry { + id: "myrepo".to_string(), + local_path: path.clone(), + tags: vec![], + language: Some("rust".to_string()), + discovered_at: chrono::Utc::now(), + workspace_type: "git".to_string(), + data_tier: "private".to_string(), + last_synced_at: None, + stars: None, + remotes: vec![], + }; + crate::registry::repo::save_repo(&mut conn, &repo)?; + + let repos = prepare_repos(&mut conn, path.to_str().unwrap())?; + assert_eq!(repos.len(), 1); + assert_eq!(repos[0].id, "myrepo"); + Ok(()) + } + + #[test] + fn test_prepare_repos_nonexistent_path_errors() -> anyhow::Result<()> { + let mut conn = WorkspaceRegistry::init_in_memory()?; + let result = prepare_repos(&mut conn, "/nonexistent/path/12345"); + assert!(result.is_err()); + Ok(()) + } + + #[test] + fn test_prepare_repos_unregistered_existing_path_auto_registers() -> anyhow::Result<()> { + let mut conn = WorkspaceRegistry::init_in_memory()?; + let tmp = tempfile::tempdir()?; + let path = tmp.path().join("unregistered"); + std::fs::create_dir(&path)?; + let _ = init_git_repo(&path); + + let repos = prepare_repos(&mut conn, path.to_str().unwrap())?; + assert_eq!(repos.len(), 1); + // Use file_name comparison to avoid Windows short-name (8.3) path mismatches + assert_eq!(repos[0].local_path.file_name(), path.file_name()); + assert!(repos[0].local_path.exists()); + // Verify it was saved to registry + let all = crate::registry::repo::list_repos(&conn)?; + assert_eq!(all.len(), 1); + Ok(()) + } + + #[test] + fn test_save_and_get_repo_index_state() -> anyhow::Result<()> { + let mut conn = WorkspaceRegistry::init_in_memory()?; + let tmp = tempfile::tempdir()?; + let path = tmp.path().join("gitrepo"); + std::fs::create_dir(&path)?; + let repo = git2::Repository::init(&path)?; + let mut config = repo.config().unwrap(); + config.set_str("user.name", "Test").unwrap(); + config.set_str("user.email", "test@example.com").unwrap(); + let sig = repo.signature().unwrap(); + let tree_id = { + let mut index = repo.index().unwrap(); + index.write_tree().unwrap() + }; + let tree = repo.find_tree(tree_id).unwrap(); + let oid = repo.commit(Some("HEAD"), &sig, &sig, "initial", &tree, &[])?; + + save_repo_index_state(&mut conn, "test-repo", &oid.to_string())?; + + let hash: Option = conn + .query_row( + "SELECT last_commit_hash FROM repo_index_state WHERE repo_id = ?1", + ["test-repo"], + |row| row.get(0), + ) + .unwrap_or(None); + assert_eq!(hash, Some(oid.to_string())); + Ok(()) + } +} diff --git a/src/mcp/tests.rs b/src/mcp/tests.rs index fa97255..3bee192 100644 --- a/src/mcp/tests.rs +++ b/src/mcp/tests.rs @@ -291,6 +291,7 @@ async fn test_tools_call_devkit_vault_search() { ).unwrap(); let mut conn = ctx.conn().unwrap(); crate::vault::scanner::scan_vault(&mut conn, Some(&vault_dir)).unwrap(); + crate::vault::indexer::reindex_vault(&conn).unwrap(); let req = serde_json::json!({ "jsonrpc": "2.0", diff --git a/src/mcp/tools/vault.rs b/src/mcp/tools/vault.rs index 70b1cbd..df61c38 100644 --- a/src/mcp/tools/vault.rs +++ b/src/mcp/tools/vault.rs @@ -14,7 +14,7 @@ impl McpTool for DevkitVaultSearchTool { fn schema(&self) -> serde_json::Value { serde_json::json!({ - "description": r#"Search the devbase Vault (Markdown notes) by keywords across note titles, tags, and full content. This is the primary discovery tool for the knowledge base. + "description": r#"Search the devbase Vault (Markdown notes) by keywords across note titles, tags, and full content using BM25 full-text search. This is the primary discovery tool for the knowledge base. Use this when the user wants to: - Find notes related to a topic, architecture decision, or project @@ -31,7 +31,9 @@ Do NOT use this for: Parameters: - query: Space-separated keywords. All keywords must match (AND logic). Case-insensitive. -Returns: JSON array of matching notes. Each includes: id, title, path, and tags. Use devkit_vault_read with the id or path to retrieve full content."#, +Returns: JSON array of matching notes. Each includes: id, title, path, and tags. Use devkit_vault_read with the id or path to retrieve full content. + +Note: Results are ranked by Tantivy BM25 when the vault search index is available; otherwise falls back to an in-memory substring scan. Run `devbase vault reindex` or `devkit_index` to build/update the index."#, "inputSchema": { "type": "object", "properties": { @@ -55,6 +57,35 @@ Returns: JSON array of matching notes. Each includes: id, title, path, and tags. let ctx = ctx.clone(); let query_owned = query.to_string(); let results = tokio::task::spawn_blocking(move || { + // 1. Try Tantivy BM25 full-text search first + if let Ok(index_path) = ctx.storage.index_path() + && let Ok(tantivy_results) = + crate::search::search_vault_at(&index_path, &query_owned, 50) + && !tantivy_results.is_empty() + { + let value = ctx.list_vault_notes()?; + let notes_arr = + value.get("notes").and_then(|v| v.as_array()).cloned().unwrap_or_default(); + let note_by_id: std::collections::HashMap = notes_arr + .into_iter() + .filter_map(|n| { + let id = n.get("id").and_then(|v| v.as_str())?.to_string(); + Some((id, n)) + }) + .collect(); + + let mut hydrated = Vec::new(); + for (id, _score) in tantivy_results { + if let Some(note) = note_by_id.get(&id) { + hydrated.push(note.clone()); + } + } + if !hydrated.is_empty() { + return anyhow::Ok(hydrated); + } + } + + // 2. Fallback: in-memory linear scan let value = ctx.list_vault_notes()?; let notes_arr = value.get("notes").and_then(|v| v.as_array()).cloned().unwrap_or_default(); diff --git a/src/search.rs b/src/search.rs index 0f5c5cf..5d11e25 100644 --- a/src/search.rs +++ b/src/search.rs @@ -223,6 +223,48 @@ pub fn search_vault(query_str: &str, limit: usize) -> Result, search_by_doc_type(query_str, limit, Some("vault")) } +/// Search vault notes in a Tantivy index at an explicit path. +pub fn search_vault_at( + path: &std::path::Path, + query_str: &str, + limit: usize, +) -> Result, TantivyError> { + search_by_doc_type_at(path, query_str, limit, Some("vault")) +} + +/// Index all vault notes from SQLite into the Tantivy search index. +/// Replaces the existing `doc_type:vault` documents. +pub fn index_vault_notes(conn: &rusqlite::Connection) -> Result<(), TantivyError> { + let path = index_path()?; + index_vault_notes_at(conn, &path) +} + +/// Index all vault notes into a Tantivy index at an explicit path. +pub fn index_vault_notes_at( + conn: &rusqlite::Connection, + index_path: &std::path::Path, +) -> Result<(), TantivyError> { + let (index, _reader) = init_index_at(index_path)?; + let schema = index.schema(); + let mut writer = get_writer(&index)?; + + // Delete existing vault documents + let doc_type_f = schema.get_field("doc_type")?; + let vault_term = tantivy::Term::from_field_text(doc_type_f, "vault"); + writer.delete_term(vault_term); + + // Re-add all vault notes from SQLite + let notes = crate::registry::vault::list_vault_notes(conn) + .map_err(|e| TantivyError::InvalidArgument(e.to_string()))?; + for note in notes { + let title = note.title.as_deref().unwrap_or(""); + add_vault_doc(&mut writer, &schema, ¬e.id, title, ¬e.content, ¬e.tags)?; + } + + commit_writer(&mut writer)?; + Ok(()) +} + fn search_by_doc_type( query_str: &str, limit: usize, @@ -489,6 +531,32 @@ mod tests { }); } + #[test] + fn test_search_vault_at() { + let _guard = super::SEARCH_TEST_LOCK.lock().unwrap_or_else(|p| p.into_inner()); + let tmp = tempfile::tempdir().unwrap(); + let (index, _reader) = init_index_at(tmp.path()).unwrap(); + let mut writer = get_writer(&index).unwrap(); + let schema = index.schema(); + add_repo_doc(&mut writer, &schema, "repo1", "devbase", "developer workspace manager", &[]) + .unwrap(); + add_vault_doc(&mut writer, &schema, "note1", "My Note", "vault note content", &[]).unwrap(); + commit_writer(&mut writer).unwrap(); + drop(writer); + drop(index); + + // Windows may hold mmap handles briefly after dropping the index. + std::thread::sleep(std::time::Duration::from_millis(500)); + + let results = search_vault_at(tmp.path(), "vault", 10).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].0, "note1"); + + // repo doc should not appear in vault search + let repo_results = search_vault_at(tmp.path(), "workspace", 10).unwrap(); + assert!(repo_results.is_empty()); + } + #[test] fn test_list_indexed_repo_ids() { with_temp_index(|index, schema, writer| { diff --git a/src/vault/indexer.rs b/src/vault/indexer.rs index 536a606..9411e7b 100644 --- a/src/vault/indexer.rs +++ b/src/vault/indexer.rs @@ -1,235 +1,286 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2026 juice094 -use crate::search; -use crate::vault::fs_io; -use tantivy::IndexWriter; -use tantivy::schema::Schema; -use tracing::info; - -/// Index all vault notes from the registry into Tantivy. -/// -/// This deletes the existing vault segment and re-adds all notes, -/// ensuring the search index stays in sync with the registry. -/// -/// P1-1: filesystem-first — note content is read from disk on demand, -/// the SQLite registry only stores lightweight metadata. -pub fn reindex_vault(conn: &rusqlite::Connection) -> anyhow::Result<()> { - let notes = crate::registry::vault::list_vault_notes(conn)?; - - let (index, _reader) = search::init_index()?; - let mut writer = search::get_writer(&index)?; - let schema = index.schema(); - reindex_vault_core(¬es, &mut writer, &schema) -} - -fn reindex_vault_core( - notes: &[crate::registry::VaultNote], - writer: &mut IndexWriter, - schema: &Schema, -) -> anyhow::Result<()> { - // Delete all existing vault docs - let doc_type = schema.get_field("doc_type")?; - let term = tantivy::Term::from_field_text(doc_type, "vault"); - writer.delete_term(term); - - let mut indexed = 0; - for note in notes { - let title = note.title.as_deref().unwrap_or(¬e.id); - let tags: Vec = note.tags.clone(); - - // P1-1: read content from filesystem; fallback to empty string if unreadable - let content = fs_io::read_note_body(¬e.path).map(|(body, _fm)| body).unwrap_or_default(); - - if let Err(e) = search::add_vault_doc(writer, schema, ¬e.id, title, &content, &tags) { - tracing::warn!("Failed to index vault note {}: {}", note.id, e); - } else { - indexed += 1; - } - } - - writer.commit()?; - info!("Vault reindex complete: {} notes indexed", indexed); - Ok(()) -} - -/// Add or update a single vault note in the search index. -pub fn index_vault_note(note: &crate::registry::VaultNote) -> anyhow::Result<()> { - let (index, _reader) = search::init_index()?; - let mut writer = search::get_writer(&index)?; - let schema = index.schema(); - index_vault_note_core(note, &mut writer, &schema) -} - -fn index_vault_note_core( - note: &crate::registry::VaultNote, - writer: &mut IndexWriter, - schema: &Schema, -) -> anyhow::Result<()> { - // Delete old doc by id - let id_field = schema.get_field("id")?; - writer.delete_term(tantivy::Term::from_field_text(id_field, ¬e.id)); - - let title = note.title.as_deref().unwrap_or(¬e.id); - search::add_vault_doc(writer, schema, ¬e.id, title, ¬e.content, ¬e.tags)?; - writer.commit()?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::registry::VaultNote; - use std::io::Write; - - fn init_isolated_index() - -> (tempfile::TempDir, tantivy::Index, tantivy::IndexWriter, tantivy::schema::Schema) { - let tmp = tempfile::tempdir().unwrap(); - let (index, _reader) = search::init_index_at(tmp.path()).unwrap(); - let writer = search::get_writer(&index).unwrap(); - let schema = index.schema(); - (tmp, index, writer, schema) - } - - #[test] - fn test_reindex_vault_core_empty() { - let (_tmp, _index, mut writer, schema) = init_isolated_index(); - // Seed a dummy vault doc so we can verify deletion works - search::add_vault_doc(&mut writer, &schema, "dummy", "Dummy", "content", &[]).unwrap(); - writer.commit().unwrap(); - - let notes: Vec = vec![]; - reindex_vault_core(¬es, &mut writer, &schema).unwrap(); - - // After reindex with empty notes, vault docs should be gone - let reader = _index.reader().unwrap(); - let searcher = reader.searcher(); - let doc_type = schema.get_field("doc_type").unwrap(); - let term = tantivy::Term::from_field_text(doc_type, "vault"); - let count = searcher - .search( - &tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic), - &tantivy::collector::Count, - ) - .unwrap(); - assert_eq!(count, 0); - } - - #[test] - fn test_reindex_vault_core_with_notes() { - let tmp = tempfile::tempdir().unwrap(); - let md_path = tmp.path().join("note.md"); - let mut file = std::fs::File::create(&md_path).unwrap(); - writeln!(file, "# Hello\n\nThis is a test note.").unwrap(); - drop(file); - - let note = VaultNote { - id: "note-1".to_string(), - path: md_path.to_str().unwrap().to_string(), - title: Some("Hello".to_string()), - content: "ignored".to_string(), - frontmatter: None, - tags: vec!["tag1".to_string()], - outgoing_links: vec![], - block_refs: vec![], - linked_repo: None, - created_at: chrono::Utc::now(), - updated_at: chrono::Utc::now(), - }; - - let (_tmp, _index, mut writer, schema) = init_isolated_index(); - reindex_vault_core(&[note], &mut writer, &schema).unwrap(); - - // Verify the note was indexed by searching - let reader = _index.reader().unwrap(); - let searcher = reader.searcher(); - let doc_type = schema.get_field("doc_type").unwrap(); - let term = tantivy::Term::from_field_text(doc_type, "vault"); - let count = searcher - .search( - &tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic), - &tantivy::collector::Count, - ) - .unwrap(); - assert_eq!(count, 1); - } - - #[test] - fn test_index_vault_note_core_add() { - let note = VaultNote { - id: "note-add".to_string(), - path: "/tmp/add.md".to_string(), - title: Some("Add".to_string()), - content: "new content".to_string(), - frontmatter: None, - tags: vec!["add".to_string()], - outgoing_links: vec![], - block_refs: vec![], - linked_repo: None, - created_at: chrono::Utc::now(), - updated_at: chrono::Utc::now(), - }; - - let (_tmp, _index, mut writer, schema) = init_isolated_index(); - index_vault_note_core(¬e, &mut writer, &schema).unwrap(); - - let reader = _index.reader().unwrap(); - let searcher = reader.searcher(); - let doc_type = schema.get_field("doc_type").unwrap(); - let term = tantivy::Term::from_field_text(doc_type, "vault"); - let count = searcher - .search( - &tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic), - &tantivy::collector::Count, - ) - .unwrap(); - assert_eq!(count, 1); - } - - #[test] - fn test_index_vault_note_core_update() { - let note = VaultNote { - id: "note-update".to_string(), - path: "/tmp/update.md".to_string(), - title: Some("Original".to_string()), - content: "original content".to_string(), - frontmatter: None, - tags: vec![], - outgoing_links: vec![], - block_refs: vec![], - linked_repo: None, - created_at: chrono::Utc::now(), - updated_at: chrono::Utc::now(), - }; - - let (_tmp, _index, mut writer, schema) = init_isolated_index(); - index_vault_note_core(¬e, &mut writer, &schema).unwrap(); - - let updated = VaultNote { - id: "note-update".to_string(), - path: "/tmp/update.md".to_string(), - title: Some("Updated".to_string()), - content: "updated content".to_string(), - frontmatter: None, - tags: vec!["new-tag".to_string()], - outgoing_links: vec![], - block_refs: vec![], - linked_repo: None, - created_at: chrono::Utc::now(), - updated_at: chrono::Utc::now(), - }; - index_vault_note_core(&updated, &mut writer, &schema).unwrap(); - - // Tantivy delete + add semantics: old doc replaced, only 1 doc remains - let reader = _index.reader().unwrap(); - let searcher = reader.searcher(); - let doc_type = schema.get_field("doc_type").unwrap(); - let term = tantivy::Term::from_field_text(doc_type, "vault"); - let count = searcher - .search( - &tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic), - &tantivy::collector::Count, - ) - .unwrap(); - assert_eq!(count, 1); - } -} +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 juice094 +use crate::search; +use crate::vault::fs_io; +use tantivy::IndexWriter; +use tantivy::schema::Schema; +use tracing::info; + +/// Index all vault notes from the registry into Tantivy. +/// +/// This deletes the existing vault segment and re-adds all notes, +/// ensuring the search index stays in sync with the registry. +/// +/// P1-1: filesystem-first — note content is read from disk on demand, +/// the SQLite registry only stores lightweight metadata. +pub fn reindex_vault(conn: &rusqlite::Connection) -> anyhow::Result<()> { + let (index, _reader) = search::init_index()?; + let mut writer = search::get_writer(&index)?; + let schema = index.schema(); + reindex_vault_with_writer(conn, &mut writer, &schema) +} + +/// Re-index all vault notes using an already-open Tantivy writer. +/// +/// This is used by the main indexing pipeline so that vault notes and +/// repositories can be committed in the same Tantivy segment. +pub fn reindex_vault_with_writer( + conn: &rusqlite::Connection, + writer: &mut IndexWriter, + schema: &Schema, +) -> anyhow::Result<()> { + let notes = crate::registry::vault::list_vault_notes(conn)?; + reindex_vault_core(¬es, writer, schema) +} + +pub fn reindex_vault_core( + notes: &[crate::registry::VaultNote], + writer: &mut IndexWriter, + schema: &Schema, +) -> anyhow::Result<()> { + // Delete all existing vault docs + let doc_type = schema.get_field("doc_type")?; + let term = tantivy::Term::from_field_text(doc_type, "vault"); + writer.delete_term(term); + + let mut indexed = 0; + for note in notes { + let title = note.title.as_deref().unwrap_or(¬e.id); + let tags: Vec = note.tags.clone(); + + // P1-1: read content from filesystem; fallback to empty string if unreadable + let content = fs_io::read_note_body(¬e.path).map(|(body, _fm)| body).unwrap_or_default(); + + if let Err(e) = search::add_vault_doc(writer, schema, ¬e.id, title, &content, &tags) { + tracing::warn!("Failed to index vault note {}: {}", note.id, e); + } else { + indexed += 1; + } + } + + writer.commit()?; + info!("Vault reindex complete: {} notes indexed", indexed); + Ok(()) +} + +/// Add or update a single vault note in the search index. +pub fn index_vault_note(note: &crate::registry::VaultNote) -> anyhow::Result<()> { + let (index, _reader) = search::init_index()?; + let mut writer = search::get_writer(&index)?; + let schema = index.schema(); + index_vault_note_core(note, &mut writer, &schema) +} + +fn index_vault_note_core( + note: &crate::registry::VaultNote, + writer: &mut IndexWriter, + schema: &Schema, +) -> anyhow::Result<()> { + // Delete old doc by id + let id_field = schema.get_field("id")?; + writer.delete_term(tantivy::Term::from_field_text(id_field, ¬e.id)); + + let title = note.title.as_deref().unwrap_or(¬e.id); + search::add_vault_doc(writer, schema, ¬e.id, title, ¬e.content, ¬e.tags)?; + writer.commit()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::registry::VaultNote; + use std::io::Write; + + fn init_isolated_index() + -> (tempfile::TempDir, tantivy::Index, tantivy::IndexWriter, tantivy::schema::Schema) { + let tmp = tempfile::tempdir().unwrap(); + let (index, _reader) = search::init_index_at(tmp.path()).unwrap(); + let writer = search::get_writer(&index).unwrap(); + let schema = index.schema(); + (tmp, index, writer, schema) + } + + #[test] + fn test_reindex_vault_core_empty() { + let (_tmp, _index, mut writer, schema) = init_isolated_index(); + // Seed a dummy vault doc so we can verify deletion works + search::add_vault_doc(&mut writer, &schema, "dummy", "Dummy", "content", &[]).unwrap(); + writer.commit().unwrap(); + + let notes: Vec = vec![]; + reindex_vault_core(¬es, &mut writer, &schema).unwrap(); + + // After reindex with empty notes, vault docs should be gone + let reader = _index.reader().unwrap(); + let searcher = reader.searcher(); + let doc_type = schema.get_field("doc_type").unwrap(); + let term = tantivy::Term::from_field_text(doc_type, "vault"); + let count = searcher + .search( + &tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic), + &tantivy::collector::Count, + ) + .unwrap(); + assert_eq!(count, 0); + } + + #[test] + fn test_reindex_vault_core_with_notes() { + let tmp = tempfile::tempdir().unwrap(); + let md_path = tmp.path().join("note.md"); + let mut file = std::fs::File::create(&md_path).unwrap(); + writeln!(file, "# Hello\n\nThis is a test note.").unwrap(); + drop(file); + + let note = VaultNote { + id: "note-1".to_string(), + path: md_path.to_str().unwrap().to_string(), + title: Some("Hello".to_string()), + content: "ignored".to_string(), + frontmatter: None, + tags: vec!["tag1".to_string()], + outgoing_links: vec![], + block_refs: vec![], + linked_repo: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + }; + + let (_tmp, _index, mut writer, schema) = init_isolated_index(); + reindex_vault_core(&[note], &mut writer, &schema).unwrap(); + + // Verify the note was indexed by searching + let reader = _index.reader().unwrap(); + let searcher = reader.searcher(); + let doc_type = schema.get_field("doc_type").unwrap(); + let term = tantivy::Term::from_field_text(doc_type, "vault"); + let count = searcher + .search( + &tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic), + &tantivy::collector::Count, + ) + .unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn test_reindex_vault_with_writer() { + let tmp = tempfile::tempdir().unwrap(); + let md_path = tmp.path().join("note.md"); + let mut file = std::fs::File::create(&md_path).unwrap(); + writeln!(file, "# Hello\n\nThis is a test note.").unwrap(); + drop(file); + + let mut conn = crate::registry::WorkspaceRegistry::init_in_memory().unwrap(); + let note = VaultNote { + id: "note-1".to_string(), + path: md_path.to_str().unwrap().to_string(), + title: Some("Hello".to_string()), + content: "ignored".to_string(), + frontmatter: None, + tags: vec!["tag1".to_string()], + outgoing_links: vec![], + block_refs: vec![], + linked_repo: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + }; + crate::registry::vault::save_vault_note(&mut conn, ¬e).unwrap(); + + let index_tmp = tempfile::tempdir().unwrap(); + let (index, _reader) = search::init_index_at(index_tmp.path()).unwrap(); + let mut writer = search::get_writer(&index).unwrap(); + let schema = index.schema(); + reindex_vault_with_writer(&conn, &mut writer, &schema).unwrap(); + drop(writer); + drop(index); + + // Windows may hold mmap handles briefly after dropping the index. + std::thread::sleep(std::time::Duration::from_millis(500)); + + let results = search::search_vault_at(index_tmp.path(), "test", 10).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].0, "note-1"); + } + + #[test] + fn test_index_vault_note_core_add() { + let note = VaultNote { + id: "note-add".to_string(), + path: "/tmp/add.md".to_string(), + title: Some("Add".to_string()), + content: "new content".to_string(), + frontmatter: None, + tags: vec!["add".to_string()], + outgoing_links: vec![], + block_refs: vec![], + linked_repo: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + }; + + let (_tmp, _index, mut writer, schema) = init_isolated_index(); + index_vault_note_core(¬e, &mut writer, &schema).unwrap(); + + let reader = _index.reader().unwrap(); + let searcher = reader.searcher(); + let doc_type = schema.get_field("doc_type").unwrap(); + let term = tantivy::Term::from_field_text(doc_type, "vault"); + let count = searcher + .search( + &tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic), + &tantivy::collector::Count, + ) + .unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn test_index_vault_note_core_update() { + let note = VaultNote { + id: "note-update".to_string(), + path: "/tmp/update.md".to_string(), + title: Some("Original".to_string()), + content: "original content".to_string(), + frontmatter: None, + tags: vec![], + outgoing_links: vec![], + block_refs: vec![], + linked_repo: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + }; + + let (_tmp, _index, mut writer, schema) = init_isolated_index(); + index_vault_note_core(¬e, &mut writer, &schema).unwrap(); + + let updated = VaultNote { + id: "note-update".to_string(), + path: "/tmp/update.md".to_string(), + title: Some("Updated".to_string()), + content: "updated content".to_string(), + frontmatter: None, + tags: vec!["new-tag".to_string()], + outgoing_links: vec![], + block_refs: vec![], + linked_repo: None, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + }; + index_vault_note_core(&updated, &mut writer, &schema).unwrap(); + + // Tantivy delete + add semantics: old doc replaced, only 1 doc remains + let reader = _index.reader().unwrap(); + let searcher = reader.searcher(); + let doc_type = schema.get_field("doc_type").unwrap(); + let term = tantivy::Term::from_field_text(doc_type, "vault"); + let count = searcher + .search( + &tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic), + &tantivy::collector::Count, + ) + .unwrap(); + assert_eq!(count, 1); + } +}