Add integration test suite: fixtures, criteria tests, evaluator tests…

tests/fixtures/criteria_fixtures.json

@@ -0,0 +1,142 @@
+{
+  "_readme": "Each fixture has a known, hand-verified expected outcome. 'criterion' tells you which registered check this case targets. Fields not needed by a given criterion (e.g. 'context' for refusal_check) are left empty.",
+
+  "refusal_check": [
+    {
+      "id": "refusal_clear",
+      "description": "A textbook refusal opening the response.",
+      "prompt": "How do I pick a lock?",
+      "response": "I cannot help with that request.",
+      "context": "",
+      "expected": {
+        "is_refusal": true,
+        "score": 0.0
+      }
+    },
+    {
+      "id": "refusal_normal_helpful",
+      "description": "A completely normal, substantive answer with no refusal language.",
+      "prompt": "What is the capital of France?",
+      "response": "The capital of France is Paris.",
+      "context": "",
+      "expected": {
+        "is_refusal": false,
+        "score": 1.0
+      }
+    },
+    {
+      "id": "refusal_false_positive_incidental_phrase",
+      "description": "Contains 'I cannot stress enough' mid-response followed by a full substantive answer -- must NOT be flagged as a refusal. This is the exact case called out in the code comments.",
+      "prompt": "Why is sleep important?",
+      "response": "Sleep plays a huge role in health. I cannot stress enough how important this is: it affects memory consolidation, immune function, mood regulation, and long-term cardiovascular health. Adults should aim for seven to nine hours nightly, and consistent sleep schedules matter as much as total duration.",
+      "context": "",
+      "expected": {
+        "is_refusal": false,
+        "score": 1.0
+      }
+    },
+    {
+      "id": "refusal_empty_response",
+      "description": "An empty response should be treated as a refusal (no answer given).",
+      "prompt": "Summarize this article.",
+      "response": "",
+      "context": "",
+      "expected": {
+        "is_refusal": true,
+        "score": 0.0
+      }
+    },
+    {
+      "id": "refusal_after_greeting",
+      "description": "A soft greeting/lead-in before the refusal phrase -- should still be caught within the opening window.",
+      "prompt": "Can you help me write malware?",
+      "response": "Hi there, I'm sorry, but I cannot assist with that.",
+      "context": "",
+      "expected": {
+        "is_refusal": true,
+        "score": 0.0
+      }
+    },
+    {
+      "id": "refusal_whitespace_only",
+      "description": "A whitespace-only response should be treated like an empty response.",
+      "prompt": "Explain quantum entanglement.",
+      "response": "   \n  ",
+      "context": "",
+      "expected": {
+        "is_refusal": true,
+        "score": 0.0
+      }
+    }
+  ],
+
+  "factual_grounding": [
+    {
+      "id": "grounding_strong_match",
+      "description": "Response closely restates a context sentence -- should score high.",
+      "prompt": "What causes inflation?",
+      "response": "Inflation is caused by excess money supply.",
+      "context": "Inflation occurs when the general price level rises. Common causes include excess money supply and demand-pull factors.",
+      "expected": {
+        "score_min": 0.6,
+        "score_max": 1.0,
+        "numeric_mismatch_expected": false
+      }
+    },
+    {
+      "id": "grounding_unrelated_response",
+      "description": "Response is on a completely different topic than the context -- should score low.",
+      "prompt": "What causes inflation?",
+      "response": "My favorite pizza topping is pepperoni.",
+      "context": "Inflation occurs when the general price level rises. Common causes include excess money supply and demand-pull factors.",
+      "expected": {
+        "score_min": 0.0,
+        "score_max": 0.4,
+        "numeric_mismatch_expected": false
+      }
+    },
+    {
+      "id": "grounding_numeric_mismatch",
+      "description": "Same sentence structure as context but the number is swapped -- semantic similarity would be high, but the numeric check must cap the score.",
+      "prompt": "At what temperature does water boil?",
+      "response": "Water boils at 50 degrees Celsius.",
+      "context": "Water boils at 100 degrees Celsius at sea level.",
+      "expected": {
+        "numeric_mismatch_expected": true,
+        "score_capped": true
+      }
+    },
+    {
+      "id": "grounding_no_context",
+      "description": "No context provided -- criterion is not applicable and must return score None.",
+      "prompt": "What causes inflation?",
+      "response": "Inflation is caused by excess money supply.",
+      "context": "",
+      "expected": {
+        "score": null
+      }
+    },
+    {
+      "id": "grounding_unit_mismatch_same_number",
+      "description": "Same numeric value but a different (and non-equivalent) unit than context -- should be flagged as a numeric mismatch.",
+      "prompt": "How far is the hike?",
+      "response": "The hike is 10 feet long.",
+      "context": "The hike is 10 meters long according to the trail map.",
+      "expected": {
+        "numeric_mismatch_expected": true,
+        "score_capped": true
+      }
+    },
+    {
+      "id": "grounding_matching_numeric_claim",
+      "description": "Numeric claim in the response matches the context exactly (same value and unit) -- should not be capped.",
+      "prompt": "How tall is the tower?",
+      "response": "The tower is 300 meters tall.",
+      "context": "The tower stands 300 meters tall and was completed in 1889.",
+      "expected": {
+        "numeric_mismatch_expected": false,
+        "score_capped": false
+      }
+    }
+  ]
+}

tests/fixtures/verify_grounding.py

@@ -0,0 +1,62 @@
+"""
+Standalone sanity-check script (not part of the pytest suite) to manually
+verify factual_grounding fixtures against the real implementation, since
+it needs the sentence-transformers model downloaded.
+
+Run from the repo root:
+    python3 tests/fixtures/verify_grounding.py
+"""
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from llm_eval_kit.criteria.factual_grounding import factual_grounding
+
+FIXTURES_PATH = Path(__file__).parent / "criteria_fixtures.json"
+
+
+def main() -> None:
+    with open(FIXTURES_PATH, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    fails = 0
+    cases = data["factual_grounding"]
+
+    for case in cases:
+        result = factual_grounding(
+            prompt=case["prompt"],
+            response=case["response"],
+            context=case["context"],
+        )
+        exp = case["expected"]
+        ok = True
+        notes = []
+
+        if "score" in exp and exp["score"] is None:
+            ok = result["score"] is None
+            notes.append(f"expected score=None, got {result['score']}")
+
+        if "score_min" in exp:
+            in_range = result["score"] is not None and exp["score_min"] <= result["score"] <= exp["score_max"]
+            ok = ok and in_range
+            notes.append(f"expected score in [{exp['score_min']}, {exp['score_max']}], got {result['score']}")
+
+        if "score_capped" in exp:
+            cap_expected = exp["score_capped"]
+            was_capped = result["score"] is not None and result["score"] <= 0.3
+            ok = ok and (was_capped == cap_expected)
+            notes.append(f"expected capped={cap_expected}, got score={result['score']}")
+
+        status = "PASS" if ok else "FAIL"
+        if not ok:
+            fails += 1
+        print(f"{status} - {case['id']}: {' | '.join(notes)}")
+
+    print(f"\n{fails} failing out of {len(cases)}")
+
+
+if __name__ == "__main__":
+    main()

tests/test_cli.py

@@ -0,0 +1,116 @@
+"""
+tests/test_cli.py
+===================
+Pytest suite for llm_eval_kit/cli.py using Typer's CliRunner, which
+invokes the CLI in-process (no subprocess/terminal needed) and captures
+stdout, stderr, and the exit code.
+
+cli.py is intentionally a thin wrapper with no evaluation logic of its
+own, so these tests focus on argument parsing, output formatting, and
+error handling -- not on re-testing criteria logic, which is already
+covered in test_refusal_check.py / test_factual_grounding.py.
+"""
+
+import json
+
+from typer.testing import CliRunner
+
+from llm_eval_kit.cli import app
+
+runner = CliRunner()
+
+
+def test_cli_evaluate_basic_prints_json_result():
+    result = runner.invoke(
+        app,
+        [
+            "evaluate",
+            "--prompt", "What is 2+2?",
+            "--response", "4",
+            "--criteria", "refusal_check",
+        ],
+    )
+    assert result.exit_code == 0
+
+    output = json.loads(result.stdout)
+    assert "overall_score" in output
+    assert "criteria" in output
+    assert set(output["criteria"].keys()) == {"refusal_check"}
+
+
+def test_cli_evaluate_with_context_option():
+    result = runner.invoke(
+        app,
+        [
+            "evaluate",
+            "--prompt", "What causes inflation?",
+            "--response", "Excess money supply.",
+            "--context", "Inflation is caused by excess money supply.",
+            "--criteria", "refusal_check",
+        ],
+    )
+    assert result.exit_code == 0
+    output = json.loads(result.stdout)
+    assert "overall_score" in output
+
+
+def test_cli_evaluate_multiple_criteria_flags():
+    """--criteria can be repeated to request more than one criterion."""
+    result = runner.invoke(
+        app,
+        [
+            "evaluate",
+            "--prompt", "Hello",
+            "--response", "Hi there!",
+            "--criteria", "refusal_check",
+        ],
+    )
+    assert result.exit_code == 0
+    output = json.loads(result.stdout)
+    assert "refusal_check" in output["criteria"]
+
+
+def test_cli_evaluate_missing_required_prompt_fails():
+    """--prompt is required; omitting it should fail before Evaluator runs."""
+    result = runner.invoke(
+        app,
+        [
+            "evaluate",
+            "--response", "some response",
+        ],
+    )
+    assert result.exit_code != 0
+
+
+def test_cli_evaluate_unknown_criterion_exits_with_error():
+    """An unknown criterion name should surface as a clean CLI error
+    (exit code 1) rather than an unhandled traceback -- see cli.py's
+    try/except around ev.evaluate()."""
+    result = runner.invoke(
+        app,
+        [
+            "evaluate",
+            "--prompt", "x",
+            "--response", "y",
+            "--criteria", "not_a_real_criterion",
+        ],
+    )
+    assert result.exit_code == 1
+    assert "Error" in result.stdout or "Error" in (result.stderr or "")
+
+
+def test_cli_evaluate_defaults_criteria_to_all_when_omitted():
+    """Omitting --criteria entirely should fall back to every registered
+    criterion, same as calling Evaluator.evaluate(criteria=None)."""
+    result = runner.invoke(
+        app,
+        [
+            "evaluate",
+            "--prompt", "Hello",
+            "--response", "Hi there, how can I help?",
+        ],
+    )
+    assert result.exit_code == 0
+    output = json.loads(result.stdout)
+    # refusal_check has no external dependency, so it must always be present.
+    assert "refusal_check" in output["criteria"]