From a865cb918456d8b458a19c06ceac993ee0b286ad Mon Sep 17 00:00:00 2001 From: Vlad Poenaru Date: Wed, 11 Nov 2026 08:04:24 -0800 Subject: [PATCH 1/5] lore: prune old versions of the lore tables after each lore run compact_lore_tables() skipped the lore table entirely and never touched lore_vectors, so neither ever dropped an old version. Every hourly refresh rewrites the five lore FTS indices in full (the body index alone is over 800 MB) and appends dozens of fragments, and all of it stayed on disk: on one host lore grew from 3.9 GB to 20 GB in two days, 17 GB of it superseded FTS indices. It also wedged FTS updates. optimize_lore_fts_indices() refuses to run once lore.lance/_indices holds more than 100 entries, and with nothing pruned the count only grew, by five per run. After about 14 hourly runs the indices stopped covering new mail for good. Prune old versions of lore from compact_lore_tables(), which only the lore run calls. Its Compact and index optimization phases, which the existing comment blames for running out of memory, stay skipped; prune reads only each manifest's index metadata. optimize_database() still skips lore: with delete_unverified set, a prune from another process could delete files a running lore refresh has not committed. Add lore_vectors to the tables compact_lore_tables() handles. Put the prune in one helper, prune_old_versions(), used by compact_lore_tables(), optimize_database() and create_lore_fts_indices(), which carried its own copy of the same call to clean up after rebuilding the FTS indices. The prune runs before the run's FTS update, so one superseded set of FTS indices remains on disk until the next run. It deletes their files but leaves their directories behind, so count only non-empty entries in the _indices guard. --- src/database/schema.rs | 102 ++++++++++++++++++++++++++++------------- 1 file changed, 71 insertions(+), 31 deletions(-) diff --git a/src/database/schema.rs b/src/database/schema.rs index 56f0a58..b85edf1 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -1721,18 +1721,7 @@ impl SchemaManager { // Prune orphaned index data left behind by drop_index(). if dropped { tracing::info!("Pruning orphaned index data from lore table..."); - if let Err(e) = table - .optimize(OptimizeAction::Prune { - older_than: Some( - lancedb::table::Duration::try_seconds(0).expect("valid duration"), - ), - delete_unverified: Some(true), - error_if_tagged_old_versions: Some(false), - }) - .await - { - tracing::warn!("Failed to prune lore table after FTS rebuild: {}", e); - } + Self::prune_old_versions(&table, "lore").await; } Ok(()) @@ -1796,13 +1785,25 @@ impl SchemaManager { // drives semcode-index into swap and gets it OOM-killed. // Query correctness is preserved regardless: unindexed rows // still fall back to a brute-force scan. + // + // Count only non-empty entries: pruning old versions deletes a + // superseded index's files but leaves its directory behind. + // Those empty directories accumulate, five per lore run; they + // cost a directory entry each and nothing else. const MAX_LORE_INDEX_FRAGMENTS: usize = 100; let uri = self.connection.uri(); let indices_dir = std::path::Path::new(uri) .join("lore.lance") .join("_indices"); if let Ok(rd) = std::fs::read_dir(&indices_dir) { - let count = rd.count(); + let count = rd + .filter_map(|entry| entry.ok()) + .filter(|entry| { + std::fs::read_dir(entry.path()) + .map(|mut files| files.next().is_some()) + .unwrap_or(true) + }) + .count(); if count > MAX_LORE_INDEX_FRAGMENTS { tracing::warn!( "Skipping lore FTS index optimization: \ @@ -2050,19 +2051,24 @@ impl SchemaManager { /// the database, including code-index tables and content shards /// that a lore run never touches. On memory-constrained systems /// the combined working set of those compactions triggers the OOM - /// killer. This method limits work to the two lore tables and + /// killer. This method limits work to the lore tables and /// processes them sequentially to keep peak memory low. pub async fn compact_lore_tables(&self) -> Result<()> { tracing::info!("Running compaction for lore tables..."); let table_names = self.connection.table_names().execute().await?; - let lore_tables = ["lore", "lore_indexed_commits"]; + let lore_tables = ["lore", "lore_indexed_commits", "lore_vectors"]; for name in &lore_tables { if !table_names.iter().any(|n| n == name) { continue; } - match Self::optimize_single_table(&self.connection, name).await { + let outcome = if *name == "lore" { + Self::prune_lore_table(&self.connection).await + } else { + Self::optimize_single_table(&self.connection, name).await + }; + match outcome { Ok(OptimizeOutcome::Optimized) => { tracing::info!("Compacted table {}", name); } @@ -2081,6 +2087,51 @@ impl SchemaManager { Ok(()) } + /// Delete every version of `table` but the latest, along with the + /// data and index files only those versions referenced. Returns + /// false if the prune failed. + async fn prune_old_versions(table: &lancedb::table::Table, table_name: &str) -> bool { + match table + .optimize(OptimizeAction::Prune { + older_than: Some(lancedb::table::Duration::try_seconds(0).expect("valid duration")), + delete_unverified: Some(true), + error_if_tagged_old_versions: Some(false), + }) + .await + { + Ok(_) => true, + Err(e) => { + tracing::warn!( + "Failed to prune old versions from table {}: {}", + table_name, + e + ); + false + } + } + } + + /// Prune old versions of the lore table, which optimize_single_table() + /// skips entirely. Every lore run rewrites the FTS indices, and + /// unless old versions are pruned the superseded copies stay under + /// _indices/ until the count trips the guard in + /// optimize_lore_fts_indices() and FTS updates stop for good. Prune + /// reads only each manifest's index metadata, not the index files, + /// so it avoids the memory cost of Compact. + /// + /// Only the lore run calls this, not optimize_database(): with + /// delete_unverified set, a prune can delete files that another + /// process writing to lore has not committed yet. That also relies + /// on lore runs not overlapping, e.g. as a systemd oneshot unit ensures. + async fn prune_lore_table(connection: &Connection) -> Result { + let table = connection.open_table("lore").execute().await?; + Ok(if Self::prune_old_versions(&table, "lore").await { + OptimizeOutcome::Optimized + } else { + OptimizeOutcome::PartialFailure + }) + } + /// Optimize a single table - runs compact, prune, and index operations /// /// Tables with fewer than 1000 rows are skipped since the overhead of @@ -2098,11 +2149,12 @@ impl SchemaManager { // the operation completes. On memory-constrained systems the // resident set grows into swap and the OOM killer terminates // semcode-index before compaction finishes, leaving fresh - // delta fragments behind each time. Skip the table entirely. + // delta fragments behind each time. Skip the table entirely; + // compact_lore_tables() prunes its old versions. if table_name == "lore" { tracing::info!( "Skipping generic optimization for lore table \ - (handled by optimize_lore_fts_indices)" + (pruned by compact_lore_tables)" ); return Ok(OptimizeOutcome::Skipped); } @@ -2165,19 +2217,7 @@ impl SchemaManager { } // 2. Prune ALL old versions - if let Err(e) = table - .optimize(OptimizeAction::Prune { - older_than: Some(lancedb::table::Duration::try_seconds(0).expect("valid duration")), - delete_unverified: Some(true), - error_if_tagged_old_versions: Some(false), - }) - .await - { - tracing::warn!( - "Failed to prune old versions from table {}: {}", - table_name, - e - ); + if !Self::prune_old_versions(&table, table_name).await { success = false; } From 79d705e41321466a602bd4d00981d16f027adb99 Mon Sep 17 00:00:00 2001 From: Vlad Poenaru Date: Wed, 11 Nov 2026 08:04:24 -0800 Subject: [PATCH 2/5] lore: merge partial batches before inserting them Each lore parser thread sends a batch once it holds batch_size emails, and flushes whatever it has left when the commit list runs out. On an hourly refresh that adds a few hundred emails per archive, every thread ends up with a short batch, and each of them became its own append to lore and lore_indexed_commits: one host's refreshes added 40 to 80 fragments to each table per run, and the lore_vectors step then made one merge_insert per fragment. A parser only sends a short batch when its input has run out, so when an inserter receives one, keep receiving until it holds at least batch_size emails or every parser has finished. Full batches received on their own are inserted as before, and a merged batch stays under twice batch_size. In a simulated hourly refresh of 50 new emails with 32 parser threads, lore gains one fragment per run instead of 32. --- src/indexer.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/indexer.rs b/src/indexer.rs index 2752ea1..1573023 100644 --- a/src/indexer.rs +++ b/src/indexer.rs @@ -957,10 +957,24 @@ pub async fn process_lore_commits_pipeline( let handle = tokio::spawn(async move { loop { - // Get next batch from shared receiver + // Get next batch from shared receiver. A parser only + // sends a short batch when its input has run out, so + // keep receiving until the batch holds at least + // batch_size emails or every parser has finished. + // Otherwise each parser's last partial batch becomes + // its own small append, dozens of tiny fragments per + // archive on a refresh. let batch = { match result_rx_clone.lock() { - Ok(rx) => rx.recv(), + Ok(rx) => rx.recv().map(|mut emails| { + while emails.len() < batch_size { + match rx.recv() { + Ok(more) => emails.extend(more), + Err(_) => break, + } + } + emails + }), Err(e) => { error!("Inserter {} failed to lock receiver: {}", inserter_id, e); break; From 338aa4731fab44e2bbda09b065d6a4ae440baa10 Mon Sep 17 00:00:00 2001 From: Vlad Poenaru Date: Wed, 11 Nov 2026 08:04:25 -0800 Subject: [PATCH 3/5] lore: buffer vectors before merging them into lore_vectors update_lore_vectors() workers send one Vec per lore RecordBatch that still has emails without a vector, and the inserter ran a merge_insert into lore_vectors for each of them. On an incremental run most of those hold only a few vectors, while every merge_insert joins against the whole table: one host's day of new mail (6,228 emails) took about 1,600 merge_inserts and 56 minutes, nearly all of it in those joins, and each hourly run added 40 to 80 fragments to lore_vectors. Buffer vectors in the inserter and write once at least 8192 have accumulated, flushing the remainder when the workers finish. The merge semantics are unchanged. merge_insert inserts every unmatched source row, so keep one entry per message_id in a buffer, as separate merges used to. An hourly-sized run now adds one fragment. The progress bar still advances on every Vec the inserter receives, not at each write, so on an incremental run it does not sit at zero until the final write and then jump to the total. Its count can now run ahead of what has been written, so its message reads "Vectorized N emails" rather than "Inserted N vectors". --- src/database/search.rs | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/database/search.rs b/src/database/search.rs index 56458cf..b958ee8 100644 --- a/src/database/search.rs +++ b/src/database/search.rs @@ -2950,20 +2950,45 @@ impl VectorSearchManager { .execute() .await?; + // Workers send one Vec per lore RecordBatch that still has + // emails without a vector. On an incremental run most of + // those hold only a few, and every merge_insert joins + // against the whole table, so write once at least + // LORE_VECTOR_FLUSH_ROWS have accumulated. + const LORE_VECTOR_FLUSH_ROWS: usize = 8192; + let mut pending: Vec = Vec::new(); let mut total_inserted = 0; - while let Some(batch_vectors) = result_rx.recv().await { + let mut open = true; + while open { + match result_rx.recv().await { + Some(batch_vectors) => { + let received = batch_vectors.len(); + let count = + processed_clone.fetch_add(received, Ordering::Relaxed) + received; + pb_clone.set_position(count as u64); + pb_clone.set_message(format!("Vectorized {} emails", count)); + pending.extend(batch_vectors); + } + None => open = false, + } + if pending.is_empty() || (open && pending.len() < LORE_VECTOR_FLUSH_ROWS) { + continue; + } + + // merge_insert inserts every unmatched source row, so two + // copies of a message_id in one buffer would become two + // rows. Keep one. + let batch = crate::database::one_row_per_key(std::mem::take(&mut pending), |e| { + e.content_hash.clone() + }); if let Err(e) = - insert_lore_vectors_batch_with_table(&lore_vectors_table, &batch_vectors).await + insert_lore_vectors_batch_with_table(&lore_vectors_table, &batch).await { tracing::error!("Failed to insert lore vectors batch: {}", e); return Err(e); } - total_inserted += batch_vectors.len(); - let count = processed_clone.fetch_add(batch_vectors.len(), Ordering::Relaxed) - + batch_vectors.len(); - pb_clone.set_position(count as u64); - pb_clone.set_message(format!("Inserted {} vectors", count)); + total_inserted += batch.len(); } Ok::<_, anyhow::Error>(total_inserted) }); From b0cdd86b29f100efab5359fdec98a1059e7017e8 Mon Sep 17 00:00:00 2001 From: Vlad Poenaru Date: Wed, 11 Nov 2026 08:48:39 -0800 Subject: [PATCH 4/5] lore: compact lore_indexed_commits whatever its fragment count A cold lore build appends one lore_indexed_commits fragment per inserted batch: 1,013 for a production-sized set of archives, 3,660 for an upstream build. optimize_single_table() skips compaction above 500 fragments and nothing else brings the count down, so the table stays over the limit for good. Rebuilding with --clear, as the skip message advises, recreates the table over the limit. Exempt this table from the limit and keep the limit for every other table. Databases already over it compact on their next lore run that adds mail. The limit guards against a lance compaction CPU spin seen with thousands of small fragments (61e5493). lore_indexed_commits holds one 40-char SHA per row and no index. On lance 4.0.0 a table of that shape compacts in under 5 s at both 1,013 and 3,660 fragments, with no spin, whether compacted alone, as compact_lore_tables() does, or eight tables at a time, as optimize_database() does. The spin does not reproduce on lance 1.0.1 either, the version 61e5493 saw it on, so its cause is still unknown. --- Cargo.lock | 1 + Cargo.toml | 5 +++ src/database/schema.rs | 93 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bfd6219..1087441 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6263,6 +6263,7 @@ dependencies = [ "gix-pack", "hex", "indicatif 0.18.6", + "lance", "lancedb", "model2vec-rs", "ndarray 0.17.2", diff --git a/Cargo.toml b/Cargo.toml index 32ed797..5346340 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,11 @@ chrono = "0.4.44" chrono-english = "0.1" # Natural language date parsing base64 = "0.22" # Base64 decoding for MIME-encoded email bodies +[dev-dependencies] +# Tests reach lance's WriteParams, which lancedb does not re-export. Pin +# the exact version lancedb resolves to, so the two cannot drift apart. +lance = { version = "=4.0.0", default-features = false } + [[bin]] name = "semcode-index" path = "src/bin/index.rs" diff --git a/src/database/schema.rs b/src/database/schema.rs index b85edf1..0c63861 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -2181,11 +2181,22 @@ impl SchemaManager { let mut success = true; - // 1. Compact files + // 1. Compact files. Tables above MAX_COMPACT_FRAGMENTS are left + // alone, as a guard against lance compaction spinning on thousands + // of small fragments. lore_indexed_commits is exempt: it holds one + // 40-char SHA per row and no index, and a table of that shape + // compacts in seconds at 3,660 fragments on lance 4.0.0. A cold + // lore build leaves it with one fragment per inserted batch, over a + // thousand for a full archive set, so the guard would keep it + // uncompacted. const MAX_COMPACT_FRAGMENTS: usize = 500; + let fragment_limit_applies = table_name != "lore_indexed_commits"; let should_compact = match table.stats().await { - Ok(stats) if stats.fragment_stats.num_fragments > MAX_COMPACT_FRAGMENTS => { + Ok(stats) + if fragment_limit_applies + && stats.fragment_stats.num_fragments > MAX_COMPACT_FRAGMENTS => + { tracing::warn!( "Skipping compaction for table {} \ ({} fragments exceeds {} limit -- \ @@ -2521,3 +2532,81 @@ impl SchemaManager { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::StringArray; + use lancedb::connection::LanceFileVersion; + use lancedb::database::listing::ListingDatabaseOptions; + + async fn fresh_schema_manager() -> (tempfile::TempDir, SchemaManager) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("schema.lancedb"); + let options = ListingDatabaseOptions::builder() + .data_storage_version(LanceFileVersion::V2_2) + .build(); + let connection = lancedb::connect(db_path.to_str().unwrap()) + .database_options(&options) + .execute() + .await + .unwrap(); + (dir, SchemaManager::new(connection)) + } + + /// A cold lore build appends one lore_indexed_commits fragment per + /// batch. 600 two-row fragments put the table above + /// MAX_COMPACT_FRAGMENTS and above MIN_ROWS_FOR_OPTIMIZATION, so the + /// lore compaction has to compact it rather than skip it. One write + /// capped at two rows per file builds them all in a single commit. + #[tokio::test] + async fn lore_indexed_commits_compacts_above_the_fragment_limit() { + const FRAGMENTS: usize = 600; + const ROWS_PER_FRAGMENT: usize = 2; + + let (_dir, schema_manager) = fresh_schema_manager().await; + schema_manager + .create_lore_indexed_commits_table() + .await + .unwrap(); + let table = schema_manager + .connection + .open_table("lore_indexed_commits") + .execute() + .await + .unwrap(); + let shas: Vec = (0..FRAGMENTS * ROWS_PER_FRAGMENT) + .map(|row| format!("{row:040x}")) + .collect(); + let batch = RecordBatch::try_new( + table.schema().await.unwrap(), + vec![Arc::new(StringArray::from(shas))], + ) + .unwrap(); + table + .add(vec![batch]) + .write_options(lancedb::table::WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + max_rows_per_file: ROWS_PER_FRAGMENT, + max_rows_per_group: ROWS_PER_FRAGMENT, + mode: lance::dataset::WriteMode::Append, + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + let before = table.stats().await.unwrap().fragment_stats.num_fragments; + assert_eq!(before, FRAGMENTS); + + schema_manager.compact_lore_tables().await.unwrap(); + + table.checkout_latest().await.unwrap(); + let after = table.stats().await.unwrap().fragment_stats.num_fragments; + assert!(after < 10, "{after} fragments left after compaction"); + assert_eq!( + table.count_rows(None).await.unwrap(), + FRAGMENTS * ROWS_PER_FRAGMENT + ); + } +} From 4d66a4526f91337eafb22008935686fa997ca279 Mon Sep 17 00:00:00 2001 From: Vlad Poenaru Date: Wed, 11 Nov 2026 08:04:25 -0800 Subject: [PATCH 5/5] lore: report skipped FTS updates and compaction problems semcode-index printed "Lore table compaction completed successfully" and "FTS indices updated successfully" whatever had happened. The reasons for skipping or failing went to tracing::warn!, which the default SEMCODE_DEBUG level filters out, so the lore FTS indices stopped covering new mail for two days on one host without a line in the hourly unit's log. Print the FTS skip reasons and lore compaction problems to stderr. optimize_lore_fts_indices() and compact_lore_tables() now return whether they did their work, and the success lines are printed only when they did. optimize_single_table() also reports a table it did not compact because it has more than 500 fragments as a partial failure rather than success, since its fragment count only grows from there. --- src/bin/index.rs | 12 ++++++---- src/database/connection.rs | 6 +++-- src/database/schema.rs | 45 ++++++++++++++++++++++++++------------ 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/src/bin/index.rs b/src/bin/index.rs index aaa1fa3..efc33bb 100644 --- a/src/bin/index.rs +++ b/src/bin/index.rs @@ -1202,7 +1202,8 @@ async fn main() -> Result<()> { // combined working set triggers the OOM killer. println!("\nCompacting lore tables..."); match db_manager.compact_lore_tables().await { - Ok(_) => println!("Lore table compaction completed successfully"), + Ok(true) => println!("Lore table compaction completed successfully"), + Ok(false) => {} Err(e) => error!("Failed to compact lore tables: {}", e), } @@ -1216,7 +1217,8 @@ async fn main() -> Result<()> { Err(e) => eprintln!("Warning: Failed to ensure FTS indices: {}", e), } match db_manager.optimize_lore_fts_indices().await { - Ok(_) => println!("FTS indices updated successfully"), + Ok(true) => println!("FTS indices updated successfully"), + Ok(false) => {} Err(e) => eprintln!("Warning: Failed to optimize FTS indices: {}", e), } } @@ -1376,7 +1378,8 @@ async fn main() -> Result<()> { if total_new_emails > 0 { println!("\nCompacting lore tables..."); match db_manager.compact_lore_tables().await { - Ok(_) => println!("Lore table compaction completed successfully"), + Ok(true) => println!("Lore table compaction completed successfully"), + Ok(false) => {} Err(e) => error!("Failed to compact lore tables: {}", e), } @@ -1386,7 +1389,8 @@ async fn main() -> Result<()> { Err(e) => eprintln!("Warning: Failed to ensure FTS indices: {}", e), } match db_manager.optimize_lore_fts_indices().await { - Ok(_) => println!("FTS indices updated successfully"), + Ok(true) => println!("FTS indices updated successfully"), + Ok(false) => {} Err(e) => eprintln!("Warning: Failed to optimize FTS indices: {}", e), } } diff --git a/src/database/connection.rs b/src/database/connection.rs index 22d8d48..47e51c7 100644 --- a/src/database/connection.rs +++ b/src/database/connection.rs @@ -889,7 +889,8 @@ impl DatabaseManager { self.schema_manager.compact_and_cleanup().await } - pub async fn compact_lore_tables(&self) -> Result<()> { + /// Returns false, after printing the problem, if any table failed. + pub async fn compact_lore_tables(&self) -> Result { self.schema_manager.compact_lore_tables().await } @@ -909,7 +910,8 @@ impl DatabaseManager { } /// Merge newly-inserted rows into existing lore FTS indices. - pub async fn optimize_lore_fts_indices(&self) -> Result<()> { + /// Returns false, after printing why, if the merge was skipped. + pub async fn optimize_lore_fts_indices(&self) -> Result { self.schema_manager.optimize_lore_fts_indices().await } diff --git a/src/database/schema.rs b/src/database/schema.rs index 0c63861..6bfc32e 100644 --- a/src/database/schema.rs +++ b/src/database/schema.rs @@ -1776,7 +1776,9 @@ impl SchemaManager { /// brute-force fallback at query time, so queries remain correct /// even before this call. Running optimize merges those rows /// into the inverted index structure, eliminating the scan cost. - pub async fn optimize_lore_fts_indices(&self) -> Result<()> { + /// + /// Returns false, after printing why, if it skipped the merge. + pub async fn optimize_lore_fts_indices(&self) -> Result { // Guard against running on a table with a large _indices/ // backlog. lance/index/append.rs opens every delta fragment // for a column before merging any, so peak memory scales @@ -1805,15 +1807,14 @@ impl SchemaManager { }) .count(); if count > MAX_LORE_INDEX_FRAGMENTS { - tracing::warn!( - "Skipping lore FTS index optimization: \ + eprintln!( + "Warning: Skipping lore FTS index optimization: \ {} _indices/ fragments exceeds {} threshold. \ Queries remain correct via brute-force fallback. \ Rebuild the lore table on a larger host to recover.", - count, - MAX_LORE_INDEX_FRAGMENTS + count, MAX_LORE_INDEX_FRAGMENTS ); - return Ok(()); + return Ok(false); } } @@ -1829,8 +1830,8 @@ impl SchemaManager { .collect(); if fts_index_names.is_empty() { - tracing::warn!("Skipping lore FTS optimization: no FTS indices found"); - return Ok(()); + eprintln!("Warning: Skipping lore FTS index optimization: no FTS indices found"); + return Ok(false); } let start_time = std::time::Instant::now(); @@ -1850,7 +1851,7 @@ impl SchemaManager { "Lore FTS index optimization completed in {:.1}s", elapsed.as_secs_f64() ); - Ok(()) + Ok(true) } /// Shared helper: create all 5 FTS indices on an already-opened table. @@ -2053,12 +2054,19 @@ impl SchemaManager { /// the combined working set of those compactions triggers the OOM /// killer. This method limits work to the lore tables and /// processes them sequentially to keep peak memory low. - pub async fn compact_lore_tables(&self) -> Result<()> { + /// + /// Returns false, after printing the problem, if any table failed. + pub async fn compact_lore_tables(&self) -> Result { tracing::info!("Running compaction for lore tables..."); let table_names = self.connection.table_names().execute().await?; let lore_tables = ["lore", "lore_indexed_commits", "lore_vectors"]; + // Print problems to stderr: the tracing warnings carrying the + // details are filtered out below SEMCODE_DEBUG=warn, and a prune + // that fails silently lets _indices/ grow until lore FTS updates + // stop, as they did when lore was never pruned. + let mut all_ok = true; for name in &lore_tables { if !table_names.iter().any(|n| n == name) { continue; @@ -2076,15 +2084,21 @@ impl SchemaManager { tracing::info!("Skipped table {} (too small)", name); } Ok(OptimizeOutcome::PartialFailure) => { - tracing::warn!("Partial failure compacting table {}", name); + eprintln!( + "Warning: Partial failure compacting table {} \ + (set SEMCODE_DEBUG=warn for details)", + name + ); + all_ok = false; } Err(e) => { - tracing::warn!("Failed to compact table {}: {}", name, e); + eprintln!("Warning: Failed to compact table {}: {}", name, e); + all_ok = false; } } } - Ok(()) + Ok(all_ok) } /// Delete every version of `table` but the latest, along with the @@ -2205,6 +2219,9 @@ impl SchemaManager { stats.fragment_stats.num_fragments, MAX_COMPACT_FRAGMENTS ); + // Not compacting is not success: the fragment count only + // grows from here, and callers report the outcome. + success = false; false } Ok(_) => true, @@ -2599,7 +2616,7 @@ mod tests { let before = table.stats().await.unwrap().fragment_stats.num_fragments; assert_eq!(before, FRAGMENTS); - schema_manager.compact_lore_tables().await.unwrap(); + assert!(schema_manager.compact_lore_tables().await.unwrap()); table.checkout_latest().await.unwrap(); let after = table.stats().await.unwrap().fragment_stats.num_fragments;