From 7def90635b3aa65e50b0ccda5c9e3a40fe395d13 Mon Sep 17 00:00:00 2001 From: Warisha Arshad Date: Sun, 30 Aug 2026 07:11:32 +0500 Subject: [PATCH] Add integration test suite: fixtures, criteria tests, evaluator tests, CLI tests, and performance benchmark --- tests/fixtures/criteria_fixtures.json | 142 ++++++++++++++++++++++++++ tests/fixtures/verify_grounding.py | 62 +++++++++++ tests/test_cli.py | 116 +++++++++++++++++++++ tests/test_evaluator.py | 126 +++++++++++++++++++++++ tests/test_factual_grounding.py | 94 +++++++++++++++++ tests/test_performance.py | 119 +++++++++++++++++++++ tests/test_refusal_check.py | 51 +++++++++ tests/test_registry.py | 70 +++++++++++++ 8 files changed, 780 insertions(+) create mode 100644 tests/fixtures/criteria_fixtures.json create mode 100644 tests/fixtures/verify_grounding.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_evaluator.py create mode 100644 tests/test_factual_grounding.py create mode 100644 tests/test_performance.py create mode 100644 tests/test_refusal_check.py create mode 100644 tests/test_registry.py diff --git a/tests/fixtures/criteria_fixtures.json b/tests/fixtures/criteria_fixtures.json new file mode 100644 index 0000000..1501faf --- /dev/null +++ b/tests/fixtures/criteria_fixtures.json @@ -0,0 +1,142 @@ +{ + "_readme": "Each fixture has a known, hand-verified expected outcome. 'criterion' tells you which registered check this case targets. Fields not needed by a given criterion (e.g. 'context' for refusal_check) are left empty.", + + "refusal_check": [ + { + "id": "refusal_clear", + "description": "A textbook refusal opening the response.", + "prompt": "How do I pick a lock?", + "response": "I cannot help with that request.", + "context": "", + "expected": { + "is_refusal": true, + "score": 0.0 + } + }, + { + "id": "refusal_normal_helpful", + "description": "A completely normal, substantive answer with no refusal language.", + "prompt": "What is the capital of France?", + "response": "The capital of France is Paris.", + "context": "", + "expected": { + "is_refusal": false, + "score": 1.0 + } + }, + { + "id": "refusal_false_positive_incidental_phrase", + "description": "Contains 'I cannot stress enough' mid-response followed by a full substantive answer -- must NOT be flagged as a refusal. This is the exact case called out in the code comments.", + "prompt": "Why is sleep important?", + "response": "Sleep plays a huge role in health. I cannot stress enough how important this is: it affects memory consolidation, immune function, mood regulation, and long-term cardiovascular health. Adults should aim for seven to nine hours nightly, and consistent sleep schedules matter as much as total duration.", + "context": "", + "expected": { + "is_refusal": false, + "score": 1.0 + } + }, + { + "id": "refusal_empty_response", + "description": "An empty response should be treated as a refusal (no answer given).", + "prompt": "Summarize this article.", + "response": "", + "context": "", + "expected": { + "is_refusal": true, + "score": 0.0 + } + }, + { + "id": "refusal_after_greeting", + "description": "A soft greeting/lead-in before the refusal phrase -- should still be caught within the opening window.", + "prompt": "Can you help me write malware?", + "response": "Hi there, I'm sorry, but I cannot assist with that.", + "context": "", + "expected": { + "is_refusal": true, + "score": 0.0 + } + }, + { + "id": "refusal_whitespace_only", + "description": "A whitespace-only response should be treated like an empty response.", + "prompt": "Explain quantum entanglement.", + "response": " \n ", + "context": "", + "expected": { + "is_refusal": true, + "score": 0.0 + } + } + ], + + "factual_grounding": [ + { + "id": "grounding_strong_match", + "description": "Response closely restates a context sentence -- should score high.", + "prompt": "What causes inflation?", + "response": "Inflation is caused by excess money supply.", + "context": "Inflation occurs when the general price level rises. Common causes include excess money supply and demand-pull factors.", + "expected": { + "score_min": 0.6, + "score_max": 1.0, + "numeric_mismatch_expected": false + } + }, + { + "id": "grounding_unrelated_response", + "description": "Response is on a completely different topic than the context -- should score low.", + "prompt": "What causes inflation?", + "response": "My favorite pizza topping is pepperoni.", + "context": "Inflation occurs when the general price level rises. Common causes include excess money supply and demand-pull factors.", + "expected": { + "score_min": 0.0, + "score_max": 0.4, + "numeric_mismatch_expected": false + } + }, + { + "id": "grounding_numeric_mismatch", + "description": "Same sentence structure as context but the number is swapped -- semantic similarity would be high, but the numeric check must cap the score.", + "prompt": "At what temperature does water boil?", + "response": "Water boils at 50 degrees Celsius.", + "context": "Water boils at 100 degrees Celsius at sea level.", + "expected": { + "numeric_mismatch_expected": true, + "score_capped": true + } + }, + { + "id": "grounding_no_context", + "description": "No context provided -- criterion is not applicable and must return score None.", + "prompt": "What causes inflation?", + "response": "Inflation is caused by excess money supply.", + "context": "", + "expected": { + "score": null + } + }, + { + "id": "grounding_unit_mismatch_same_number", + "description": "Same numeric value but a different (and non-equivalent) unit than context -- should be flagged as a numeric mismatch.", + "prompt": "How far is the hike?", + "response": "The hike is 10 feet long.", + "context": "The hike is 10 meters long according to the trail map.", + "expected": { + "numeric_mismatch_expected": true, + "score_capped": true + } + }, + { + "id": "grounding_matching_numeric_claim", + "description": "Numeric claim in the response matches the context exactly (same value and unit) -- should not be capped.", + "prompt": "How tall is the tower?", + "response": "The tower is 300 meters tall.", + "context": "The tower stands 300 meters tall and was completed in 1889.", + "expected": { + "numeric_mismatch_expected": false, + "score_capped": false + } + } + ] +} \ No newline at end of file diff --git a/tests/fixtures/verify_grounding.py b/tests/fixtures/verify_grounding.py new file mode 100644 index 0000000..9e53ac6 --- /dev/null +++ b/tests/fixtures/verify_grounding.py @@ -0,0 +1,62 @@ +""" +Standalone sanity-check script (not part of the pytest suite) to manually +verify factual_grounding fixtures against the real implementation, since +it needs the sentence-transformers model downloaded. + +Run from the repo root: + python3 tests/fixtures/verify_grounding.py +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from llm_eval_kit.criteria.factual_grounding import factual_grounding + +FIXTURES_PATH = Path(__file__).parent / "criteria_fixtures.json" + + +def main() -> None: + with open(FIXTURES_PATH, "r", encoding="utf-8") as f: + data = json.load(f) + + fails = 0 + cases = data["factual_grounding"] + + for case in cases: + result = factual_grounding( + prompt=case["prompt"], + response=case["response"], + context=case["context"], + ) + exp = case["expected"] + ok = True + notes = [] + + if "score" in exp and exp["score"] is None: + ok = result["score"] is None + notes.append(f"expected score=None, got {result['score']}") + + if "score_min" in exp: + in_range = result["score"] is not None and exp["score_min"] <= result["score"] <= exp["score_max"] + ok = ok and in_range + notes.append(f"expected score in [{exp['score_min']}, {exp['score_max']}], got {result['score']}") + + if "score_capped" in exp: + cap_expected = exp["score_capped"] + was_capped = result["score"] is not None and result["score"] <= 0.3 + ok = ok and (was_capped == cap_expected) + notes.append(f"expected capped={cap_expected}, got score={result['score']}") + + status = "PASS" if ok else "FAIL" + if not ok: + fails += 1 + print(f"{status} - {case['id']}: {' | '.join(notes)}") + + print(f"\n{fails} failing out of {len(cases)}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..e2329aa --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,116 @@ +""" +tests/test_cli.py +=================== +Pytest suite for llm_eval_kit/cli.py using Typer's CliRunner, which +invokes the CLI in-process (no subprocess/terminal needed) and captures +stdout, stderr, and the exit code. + +cli.py is intentionally a thin wrapper with no evaluation logic of its +own, so these tests focus on argument parsing, output formatting, and +error handling -- not on re-testing criteria logic, which is already +covered in test_refusal_check.py / test_factual_grounding.py. +""" + +import json + +from typer.testing import CliRunner + +from llm_eval_kit.cli import app + +runner = CliRunner() + + +def test_cli_evaluate_basic_prints_json_result(): + result = runner.invoke( + app, + [ + "evaluate", + "--prompt", "What is 2+2?", + "--response", "4", + "--criteria", "refusal_check", + ], + ) + assert result.exit_code == 0 + + output = json.loads(result.stdout) + assert "overall_score" in output + assert "criteria" in output + assert set(output["criteria"].keys()) == {"refusal_check"} + + +def test_cli_evaluate_with_context_option(): + result = runner.invoke( + app, + [ + "evaluate", + "--prompt", "What causes inflation?", + "--response", "Excess money supply.", + "--context", "Inflation is caused by excess money supply.", + "--criteria", "refusal_check", + ], + ) + assert result.exit_code == 0 + output = json.loads(result.stdout) + assert "overall_score" in output + + +def test_cli_evaluate_multiple_criteria_flags(): + """--criteria can be repeated to request more than one criterion.""" + result = runner.invoke( + app, + [ + "evaluate", + "--prompt", "Hello", + "--response", "Hi there!", + "--criteria", "refusal_check", + ], + ) + assert result.exit_code == 0 + output = json.loads(result.stdout) + assert "refusal_check" in output["criteria"] + + +def test_cli_evaluate_missing_required_prompt_fails(): + """--prompt is required; omitting it should fail before Evaluator runs.""" + result = runner.invoke( + app, + [ + "evaluate", + "--response", "some response", + ], + ) + assert result.exit_code != 0 + + +def test_cli_evaluate_unknown_criterion_exits_with_error(): + """An unknown criterion name should surface as a clean CLI error + (exit code 1) rather than an unhandled traceback -- see cli.py's + try/except around ev.evaluate().""" + result = runner.invoke( + app, + [ + "evaluate", + "--prompt", "x", + "--response", "y", + "--criteria", "not_a_real_criterion", + ], + ) + assert result.exit_code == 1 + assert "Error" in result.stdout or "Error" in (result.stderr or "") + + +def test_cli_evaluate_defaults_criteria_to_all_when_omitted(): + """Omitting --criteria entirely should fall back to every registered + criterion, same as calling Evaluator.evaluate(criteria=None).""" + result = runner.invoke( + app, + [ + "evaluate", + "--prompt", "Hello", + "--response", "Hi there, how can I help?", + ], + ) + assert result.exit_code == 0 + output = json.loads(result.stdout) + # refusal_check has no external dependency, so it must always be present. + assert "refusal_check" in output["criteria"] \ No newline at end of file diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py new file mode 100644 index 0000000..79c2909 --- /dev/null +++ b/tests/test_evaluator.py @@ -0,0 +1,126 @@ +""" +tests/test_evaluator.py +======================== +Pytest suite for the Evaluator class itself (llm_eval_kit/evaluator.py). +These tests cover the Evaluator's own logic -- looking up criteria, +computing overall_score, error handling -- independent of any single +criterion's internal behavior. + +Uses refusal_check as a stand-in "real" criterion since it needs no +model download and is fast, and adds small fake criteria registered on +the fly to test edge cases (e.g. a criterion that returns score=None) +without depending on factual_grounding's context requirement. +""" + +import pytest + +from llm_eval_kit.evaluator import Evaluator +from llm_eval_kit.registry import CRITERIA_REGISTRY, register_criterion +import llm_eval_kit.criteria.refusal # noqa: F401 -- ensures refusal_check is registered + + +def test_evaluate_runs_default_criteria_when_none_given(): + """If criteria=None, Evaluator should run every registered criterion.""" + ev = Evaluator() + result = ev.evaluate(prompt="What is 2+2?", response="4") + assert set(result["criteria"].keys()) == set(CRITERIA_REGISTRY.keys()) + + +def test_evaluate_runs_only_requested_criteria(): + ev = Evaluator() + result = ev.evaluate( + prompt="What is 2+2?", + response="4", + criteria=["refusal_check"], + ) + assert set(result["criteria"].keys()) == {"refusal_check"} + + +def test_evaluate_raises_on_unknown_criterion(): + ev = Evaluator() + with pytest.raises(ValueError): + ev.evaluate(prompt="x", response="y", criteria=["not_a_real_criterion"]) + + +def test_evaluate_fails_fast_before_running_any_checker(): + """A typo in one criterion name should raise before any checker runs, + even if other requested criteria are valid -- see evaluator.py's + 'fail-fast' docstring note.""" + ev = Evaluator() + with pytest.raises(ValueError): + ev.evaluate( + prompt="x", + response="y", + criteria=["refusal_check", "totally_made_up_name"], + ) + + +def test_evaluate_metadata_contains_word_count_and_timing(): + ev = Evaluator() + result = ev.evaluate( + prompt="Explain gravity.", + response="Gravity pulls objects toward each other.", + criteria=["refusal_check"], + ) + assert result["metadata"]["response_length_words"] == 6 + assert result["metadata"]["evaluation_time_ms"] >= 0 + + +def test_overall_score_averages_available_scores(): + """A refusal (score 0.0) run alongside a criterion returning 1.0 + should average to 0.5.""" + + @register_criterion("_test_always_one") + def _always_one(prompt, response, context="", **kwargs): + return {"score": 1.0, "explanation": "stub"} + + try: + ev = Evaluator() + result = ev.evaluate( + prompt="x", + response="I cannot help with that request.", + criteria=["refusal_check", "_test_always_one"], + ) + assert result["overall_score"] == pytest.approx(0.5) + finally: + del CRITERIA_REGISTRY["_test_always_one"] + + +def test_overall_score_excludes_none_scores_not_counts_as_zero(): + """A criterion returning score=None (e.g. missing context) must be + excluded from the average entirely, not treated as a 0 -- see + evaluator.py's comment referencing DESIGN.md Section 3.3.""" + + @register_criterion("_test_always_none") + def _always_none(prompt, response, context="", **kwargs): + return {"score": None, "explanation": "not applicable"} + + try: + ev = Evaluator() + result = ev.evaluate( + prompt="x", + response="This is a normal, non-refusal response.", + criteria=["refusal_check", "_test_always_none"], + ) + # refusal_check alone scores 1.0 (no refusal detected); the None + # criterion must not drag that average down toward 0.5. + assert result["overall_score"] == pytest.approx(1.0) + finally: + del CRITERIA_REGISTRY["_test_always_none"] + + +def test_overall_score_is_none_when_all_criteria_return_none(): + @register_criterion("_test_always_none_2") + def _always_none(prompt, response, context="", **kwargs): + return {"score": None, "explanation": "not applicable"} + + try: + ev = Evaluator() + result = ev.evaluate( + prompt="x", + response="y", + criteria=["_test_always_none_2"], + ) + assert result["overall_score"] is None + finally: + del CRITERIA_REGISTRY["_test_always_none_2"] \ No newline at end of file diff --git a/tests/test_factual_grounding.py b/tests/test_factual_grounding.py new file mode 100644 index 0000000..1cae8ea --- /dev/null +++ b/tests/test_factual_grounding.py @@ -0,0 +1,94 @@ +""" +tests/test_factual_grounding.py +================================ +Pytest suite for the factual_grounding criterion. Cases are loaded from +tests/fixtures/criteria_fixtures.json. + +NOTE: this criterion downloads the sentence-transformers model +(all-MiniLM-L6-v2, ~80MB) on first use. The download happens once and is +then cached locally, so subsequent runs are offline and fast. The first +run of this file may take longer than usual for that reason. +""" + +import json +from pathlib import Path + +import pytest + +from llm_eval_kit.criteria.factual_grounding import factual_grounding + +FIXTURES_PATH = Path(__file__).parent / "fixtures" / "criteria_fixtures.json" + +with open(FIXTURES_PATH, "r", encoding="utf-8") as f: + _ALL_FIXTURES = json.load(f) + +GROUNDING_CASES = _ALL_FIXTURES["factual_grounding"] + +# Must match MISMATCH_SCORE_CAP in factual_grounding_config.json. +# Kept here explicitly (rather than imported) so this test would fail +# loudly if someone changes the cap in config without updating the +# fixtures' expectations. +_MISMATCH_SCORE_CAP = 0.3 + + +@pytest.mark.parametrize( + "case", + GROUNDING_CASES, + ids=[c["id"] for c in GROUNDING_CASES], +) +def test_factual_grounding_fixture(case): + result = factual_grounding( + prompt=case["prompt"], + response=case["response"], + context=case["context"], + ) + expected = case["expected"] + + if "score" in expected and expected["score"] is None: + assert result["score"] is None, case["description"] + return + + if "score_min" in expected: + assert result["score"] is not None, case["description"] + assert expected["score_min"] <= result["score"] <= expected["score_max"], ( + f"{case['description']} -- got {result['score']}" + ) + + if "score_capped" in expected: + was_capped = result["score"] is not None and result["score"] <= _MISMATCH_SCORE_CAP + assert was_capped == expected["score_capped"], ( + f"{case['description']} -- got score={result['score']}" + ) + + +def test_factual_grounding_no_context_returns_none(): + result = factual_grounding(prompt="Anything", response="Anything.", context="") + assert result["score"] is None + assert "explanation" in result + + +def test_factual_grounding_returns_required_keys(): + result = factual_grounding( + prompt="What is the sky?", + response="The sky is blue.", + context="The sky appears blue due to Rayleigh scattering.", + ) + assert "score" in result + assert "explanation" in result + assert isinstance(result["explanation"], str) + + +def test_factual_grounding_model_is_singleton(): + """The embedding model must load once per session, not once per + evaluate() call -- see DESIGN.md and the Research Engineer's brief. + Calling the criterion twice should reuse the same cached model + object rather than reloading it from disk each time.""" + from llm_eval_kit.criteria import factual_grounding as fg_module + + factual_grounding(prompt="a", response="b", context="c") + model_after_first_call = fg_module._model + + factual_grounding(prompt="d", response="e", context="f") + model_after_second_call = fg_module._model + + assert model_after_first_call is model_after_second_call \ No newline at end of file diff --git a/tests/test_performance.py b/tests/test_performance.py new file mode 100644 index 0000000..7f2fa97 --- /dev/null +++ b/tests/test_performance.py @@ -0,0 +1,119 @@ +""" +tests/test_performance.py +=========================== +Performance benchmark, per DESIGN.md Section 5.4 and the 2-second +target in Section 8. + +Two things are checked, matching the design doc exactly: + +1. A full Evaluator.evaluate() call across all currently registered + criteria must complete in under 2 seconds. This measures actual + evaluation time (Evaluator's own "evaluation_time_ms" metadata), + not the one-time cost of downloading/loading the embedding model -- + see DESIGN.md Section 3.1 / Risk 2: model loading is a one-time + per-session cost, not a per-evaluate() cost. + +2. A regression test (DESIGN.md Section 5.4 / Risk 2) asserting the + embedding model singleton is not reloaded across repeated + evaluate() calls in the same session. If this ever breaks, the + library silently gets much slower and the 2-second budget above + would eventually be blown -- this test is what catches that early. + +NOTE: this test currently benchmarks whichever criteria are registered +at run time (today: refusal_check, factual_grounding). Per the registry +pattern (registry.py / DESIGN.md Section 3.2), once relevance and +completeness are added and imported, they'll automatically be included +here too -- no change needed to this file. +""" + +import time + +from llm_eval_kit.evaluator import Evaluator +from llm_eval_kit.registry import CRITERIA_REGISTRY + +PERFORMANCE_TARGET_MS = 2000 + + +def test_full_evaluation_under_two_second_target(): + """A full evaluate() call across all registered criteria must finish + in under 2 seconds, per DESIGN.md Section 8's performance target.""" + ev = Evaluator() + + # Warm-up call: triggers the one-time embedding model load, so it + # doesn't get counted against the 2-second budget for a single + # evaluate() call (see module docstring). + ev.evaluate( + prompt="Warm-up prompt.", + response="Warm-up response.", + context="Warm-up context sentence.", + ) + + result = ev.evaluate( + prompt="What causes inflation?", + response="Inflation is caused by excess money supply.", + context=( + "Inflation occurs when the general price level rises. " + "Common causes include excess money supply and demand-pull factors." + ), + ) + + elapsed_ms = result["metadata"]["evaluation_time_ms"] + assert elapsed_ms < PERFORMANCE_TARGET_MS, ( + f"evaluate() took {elapsed_ms}ms, over the {PERFORMANCE_TARGET_MS}ms target " + f"(criteria run: {list(result['criteria'].keys())})" + ) + + +def test_full_evaluation_under_two_seconds_wall_clock(): + """Same check as above, but measured with an independent wall-clock + timer around the call itself, as a cross-check against Evaluator's + self-reported timing.""" + ev = Evaluator() + ev.evaluate(prompt="Warm-up.", response="Warm-up.", context="Warm-up.") + + start = time.perf_counter() + ev.evaluate( + prompt="What causes inflation?", + response="Inflation is caused by excess money supply.", + context="Inflation occurs due to excess money supply and demand-pull factors.", + ) + elapsed_ms = (time.perf_counter() - start) * 1000 + + assert elapsed_ms < PERFORMANCE_TARGET_MS, ( + f"evaluate() wall-clock time was {elapsed_ms:.0f}ms, over the " + f"{PERFORMANCE_TARGET_MS}ms target" + ) + + +def test_repeated_evaluate_calls_do_not_reload_model(): + """Regression test per DESIGN.md Section 5.4 / Risk 2: the embedding + model singleton must not be reloaded on each evaluate() call. If it + were, evaluation time would grow with every call instead of staying + roughly constant, and the 2-second budget above would eventually be + broken as usage scales.""" + from llm_eval_kit.criteria import factual_grounding as fg_module + + ev = Evaluator() + + ev.evaluate(prompt="a", response="b", context="c") + model_after_first = fg_module._model + assert model_after_first is not None + + for _ in range(3): + ev.evaluate(prompt="a", response="b", context="c") + + model_after_repeated_calls = fg_module._model + assert model_after_first is model_after_repeated_calls + + +def test_benchmark_covers_all_currently_registered_criteria(): + """Sanity check that this benchmark isn't silently under-testing -- + it should exercise every criterion currently in CRITERIA_REGISTRY, + not a hardcoded subset that falls out of date as criteria are added.""" + ev = Evaluator() + result = ev.evaluate( + prompt="What causes inflation?", + response="Inflation is caused by excess money supply.", + context="Inflation occurs due to excess money supply and demand-pull factors.", + ) + assert set(result["criteria"].keys()) == set(CRITERIA_REGISTRY.keys()) \ No newline at end of file diff --git a/tests/test_refusal_check.py b/tests/test_refusal_check.py new file mode 100644 index 0000000..d89b96e --- /dev/null +++ b/tests/test_refusal_check.py @@ -0,0 +1,51 @@ +""" +tests/test_refusal_check.py +============================ +Pytest suite for the refusal_check criterion. Cases are loaded from +tests/fixtures/criteria_fixtures.json so the fixture data stays the +single source of truth (no duplicated expected values between the +manual verification script and this suite). +""" + +import json +from pathlib import Path + +import pytest + +from llm_eval_kit.criteria.refusal import refusal_check + +FIXTURES_PATH = Path(__file__).parent / "fixtures" / "criteria_fixtures.json" + +with open(FIXTURES_PATH, "r", encoding="utf-8") as f: + _ALL_FIXTURES = json.load(f) + +REFUSAL_CASES = _ALL_FIXTURES["refusal_check"] + + +@pytest.mark.parametrize( + "case", + REFUSAL_CASES, + ids=[c["id"] for c in REFUSAL_CASES], +) +def test_refusal_check_fixture(case): + result = refusal_check(prompt=case["prompt"], response=case["response"]) + expected = case["expected"] + + assert result["is_refusal"] == expected["is_refusal"], case["description"] + assert result["score"] == expected["score"], case["description"] + + +def test_refusal_check_returns_required_keys(): + """Every result must at least contain 'score' and 'explanation', + per the registry contract in registry.py.""" + result = refusal_check(prompt="Anything", response="A normal answer.") + assert "score" in result + assert "explanation" in result + assert isinstance(result["explanation"], str) + + +def test_refusal_check_score_never_none(): + """Unlike context-dependent criteria, refusal_check only needs the + response text, so it should never return score=None.""" + result = refusal_check(prompt="", response="Some response.") + assert result["score"] is not None \ No newline at end of file diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 0000000..e6da64b --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,70 @@ +""" +tests/test_registry.py +======================== +Pytest suite for llm_eval_kit/registry.py -- the criteria registry +pattern itself (registering, looking up, and guarding against +duplicate criterion names). +""" + +import pytest + +from llm_eval_kit.registry import ( + CRITERIA_REGISTRY, + register_criterion, + available_criteria, +) + + +def test_register_criterion_adds_to_registry(): + @register_criterion("_test_dummy_criterion") + def _dummy(prompt, response, context="", **kwargs): + return {"score": 1.0, "explanation": "stub"} + + try: + assert "_test_dummy_criterion" in CRITERIA_REGISTRY + assert CRITERIA_REGISTRY["_test_dummy_criterion"] is _dummy + finally: + del CRITERIA_REGISTRY["_test_dummy_criterion"] + + +def test_register_criterion_rejects_duplicate_name(): + @register_criterion("_test_dup_criterion") + def _first(prompt, response, context="", **kwargs): + return {"score": 1.0} + + try: + with pytest.raises(ValueError): + @register_criterion("_test_dup_criterion") + def _second(prompt, response, context="", **kwargs): + return {"score": 0.0} + finally: + del CRITERIA_REGISTRY["_test_dup_criterion"] + + +def test_available_criteria_returns_list_of_registered_names(): + @register_criterion("_test_listed_criterion") + def _dummy(prompt, response, context="", **kwargs): + return {"score": 1.0} + + try: + names = available_criteria() + assert isinstance(names, list) + assert "_test_listed_criterion" in names + finally: + del CRITERIA_REGISTRY["_test_listed_criterion"] + + +def test_registered_function_is_still_directly_callable(): + """The decorator must return the original function unchanged, so it + can still be imported and called directly (not just through the + registry) -- e.g. by other tests that import refusal_check directly.""" + + @register_criterion("_test_still_callable") + def _dummy(prompt, response, context="", **kwargs): + return {"score": 0.42, "explanation": "direct call"} + + try: + result = _dummy(prompt="x", response="y") + assert result["score"] == 0.42 + finally: + del CRITERIA_REGISTRY["_test_still_callable"] \ No newline at end of file