fix(grep): report truthful insufficient-result stats (#9)

* fix(grep): report truthful insufficient-result stats

* test(grep): prove insufficient KG preservation

Author: AlexMikhalev

crates/terraphim_grep/README.md

@@ -113,7 +113,7 @@ Query
   ├── Sufficient ──→ Return chunks (SearchOnly)
   ├── NeedsSynthesis ──→ RLM fallback (if LLM configured)
   ├── NeedsExpansion ──→ RLM fallback with additional chunks
-  └── Insufficient ──→ Return empty (RlmInsufficient)
+  └── Insufficient ──→ Preserve retrieved chunks + KG metadata (RlmInsufficient)
 ```
 
 ## Key Types

crates/terraphim_grep/src/lib.rs

@@ -158,17 +158,20 @@ impl TerraphimGrep {
                     .await
             }
             sufficiency_judge::Sufficiency::Insufficient(chunks) => {
+                // Preserve the retrieved chunks and KG concepts and derive the
+                // counters from the actual vectors so the JSON stats stay
+                // truthful even when the judge deems the result insufficient.
                 let stats = GrepStats {
                     search_latency_ms,
                     rlm_latency_ms: None,
-                    chunks_returned: 0,
-                    kg_hits: 0,
+                    chunks_returned: chunks.len(),
+                    kg_hits: hybrid_results.kg_concepts.len(),
                 };
 
                 Ok(GrepResult {
                     chunks,
                     answer: None,
-                    concepts: vec![],
+                    concepts: hybrid_results.kg_concepts,
                     sufficiency: SufficiencyState::RlmInsufficient,
                     stats,
                 })
@@ -333,7 +336,7 @@ impl TerraphimGrep {
 mod tests {
     use super::*;
     #[cfg(feature = "code-search")]
-    use terraphim_types::Thesaurus;
+    use terraphim_types::{NormalizedTerm, NormalizedTermValue, Thesaurus};
 
     #[test]
     fn test_grep_options_default() {
@@ -438,6 +441,71 @@ mod tests {
         assert_eq!(result.stats.kg_hits, 0);
     }
 
+    /// When the sufficiency judge returns `Insufficient` with non-empty chunks
+    /// (fewer than `min_results` matches), the returned chunks and KG concepts
+    /// must be preserved and the stats must be truthful:
+    /// `stats.chunks_returned == chunks.len()` and `stats.kg_hits == concepts.len()`.
+    /// This guards the JSON result invariant that blocks release wrapper #3208.
+    #[cfg(feature = "code-search")]
+    #[tokio::test]
+    async fn rlm_insufficient_preserves_chunks_and_reports_truthful_stats() {
+        let tmp = tempfile::TempDir::new().expect("tempdir");
+        // Single match => chunks.len() (1) < min_results (3) => Insufficient branch.
+        let path = tmp.path().join("only_match.rs");
+        std::fs::write(&path, "fn unique_target() { /* unique_target */ }\n").unwrap();
+
+        let mut thesaurus = Thesaurus::new("t".to_string());
+        let concept_key = NormalizedTermValue::from("unique_target");
+        let concept = NormalizedTerm::new(1, concept_key.clone())
+            .with_display_value("unique_target".to_string());
+        thesaurus.insert(concept_key, concept);
+
+        let hybrid = HybridSearcher::new("test-role".to_string(), thesaurus)
+            .expect("build hybrid searcher")
+            .with_search_path(tmp.path().to_path_buf());
+        let grep = TerraphimGrep::new(Arc::new(hybrid), Arc::new(SufficiencyJudge::default()));
+
+        let result = grep
+            .search(
+                "unique_target",
+                GrepOptions {
+                    haystack: Haystack::Code,
+                    max_results: 50,
+                    ..GrepOptions::default()
+                },
+            )
+            .await
+            .expect("search should succeed");
+
+        assert!(
+            !result.chunks.is_empty(),
+            "expected at least one chunk from the known-match corpus"
+        );
+        assert!(
+            matches!(result.sufficiency, SufficiencyState::RlmInsufficient),
+            "single match must hit the Insufficient branch, got {:?}",
+            result.sufficiency
+        );
+        assert_eq!(
+            result.stats.chunks_returned,
+            result.chunks.len(),
+            "stats.chunks_returned must equal chunks.len() in the RlmInsufficient branch"
+        );
+        assert_eq!(
+            result.stats.kg_hits,
+            result.concepts.len(),
+            "stats.kg_hits must equal concepts.len() when concepts are retained"
+        );
+        assert!(result.stats.kg_hits > 0, "fixture must produce a KG hit");
+        assert!(
+            result
+                .concepts
+                .iter()
+                .any(|concept| concept.name == "unique_target"),
+            "the known KG concept must survive the RlmInsufficient branch"
+        );
+    }
+
     /// The RLM prompt for `include_answer` must embed the `AnswerSignature`
     /// JSON instructions so the model knows it must return structured output.
     #[test]

crates/terraphim_grep/tests/no_thesaurus_cli.rs

@@ -57,4 +57,22 @@ fn cli_runs_without_thesaurus() {
         Some(0),
         "kg_hits should be zero"
     );
+
+    // Truthful-stats invariant (blocks release wrapper #3208): the reported
+    // counter must always match the number of chunks actually returned, even
+    // when the sufficiency heuristic classifies the result as RlmInsufficient
+    // (fewer than min_results matches, as in this single-file corpus).
+    let chunks_returned = result["stats"]["chunks_returned"]
+        .as_u64()
+        .expect("chunks_returned is a number") as usize;
+    assert_eq!(
+        chunks_returned,
+        chunks.len(),
+        "stats.chunks_returned must equal chunks.len() (got {chunks_returned}, chunks = {})",
+        chunks.len()
+    );
+    assert!(
+        chunks_returned >= 1,
+        "known-match corpus must report at least one returned chunk"
+    );
 }

docs/plans/design-terraphim-grep-truthful-stats-2026-08-12.md

@@ -0,0 +1,20 @@
+# Design: truthful `terraphim_grep` insufficient-result statistics
+
+## Problem
+Published `terraphim_grep` 1.21.1 can return non-empty `chunks` with `stats.chunks_returned = 0` when the sufficiency heuristic classifies fewer than three matches as `RlmInsufficient`. This violates the JSON result invariant and blocks release wrapper #3208.
+
+## Decision
+In `crates/terraphim_grep/src/lib.rs`, preserve returned chunks and KG concepts in the `Sufficiency::Insufficient` branch and derive counters from the actual vectors. Do not change sufficiency thresholds or reinterpret `RlmInsufficient`.
+
+## Tests first
+Extend the real CLI known-match regression so it asserts `stats.chunks_returned == chunks.len()`. Add/extend library coverage for `RlmInsufficient` with non-empty chunks and the same invariant. Run the targeted test before implementation and retain the expected RED.
+
+## Acceptance
+- Known-match JSON has at least one chunk.
+- `stats.chunks_returned == chunks.len()` in every result branch.
+- `stats.kg_hits == concepts.len()` when concepts are retained.
+- Focused crate tests, fmt and clippy pass.
+- Exact Release Guardian known-match smoke passes from the branch binary.
+
+## Non-goals
+No release version bump, publish, sufficiency threshold change, unrelated crate refactor, or dependency update in this PR.