Feat/final release

.gitignore

@@ -1,3 +1,10 @@
 *.docx
 __pycache__/
 .vscode/
+.pytest_cache/
+*.egg-info/
+dist/
+build/
+*env/
+*.venv/
+

README.md

@@ -37,7 +37,7 @@ pip install llm-eval-kit
 Here is a 4-line example demonstrating how to run the evaluator:
 
 ```python
-from llm_eval_kit.evaluator import Evaluator
+from llm_eval_kit import Evaluator
 
 # Initialize the evaluator
 evaluator = Evaluator()
@@ -47,33 +47,46 @@ prompt = "What is the boiling point of water at sea level?"
 response = "Water boils at 100 degrees Celsius at sea level."
 context = "At sea level, the boiling point of water is 100°C (212°F)."
 
-# Run the evaluation
+# Run the evaluation (specify criteria or omit to run all registered criteria)
 result = evaluator.evaluate(
     prompt=prompt,
     response=response,
     context=context,
-    criteria=["refusal_check", "factual_grounding"]
+    criteria=["refusal_check", "factual_grounding", "relevance", "completeness"]
 )
 
 # Output is a structured Python dictionary
 import json
 print(json.dumps(result, indent=2))
 ```
 
+> **Note on Evaluation Cost & Latency:**
+> Omitting the `criteria` argument (or passing `criteria=None`) executes all registered criteria in `CRITERIA_REGISTRY`. As additional criteria are added in the future (especially compute-heavy transformer models or multi-aspect passes), evaluating all criteria by default will proportionally increase the evaluation latency and resource cost. In production or latency-critical pipelines, it is best practice to explicitly specify only the criteria you need.
+
 ### Example Output JSON
 ```json
 {
-  "overall_score": 1.0,
+  "overall_score": 0.95,
   "criteria": {
     "refusal_check": {
       "score": 1.0,
       "is_refusal": false,
       "explanation": "No refusal phrase detected near the start of the response."
     },
     "factual_grounding": {
-      "score": 0.985,
+      "score": 0.88,
       "explanation": "Score reflects semantic similarity with a numeric-consistency penalty applied, not calibrated factual accuracy. Response numeric claims match or are supported by the context.",
       "best_matching_sentence": "At sea level, the boiling point of water is 100°C (212°F)."
+    },
+    "relevance": {
+      "score": 0.92,
+      "explanation": "Response is highly relevant and directly addresses the prompt topic."
+    },
+    "completeness": {
+      "score": 1.0,
+      "explanation": "Response covers all key aspects requested in the prompt.",
+      "covered_aspects": ["boiling point of water at sea level"],
+      "total_aspects": 1
     }
   },
   "metadata": {
@@ -102,7 +115,7 @@ llm-eval evaluate \
 - `--prompt` (Required): The original prompt sent to the LLM.
 - `--response` (Required): The LLM response to evaluate.
 - `--context` (Optional): Reference text for grounding.
-- `--criteria` (Optional): Repeating option to choose specific criteria (defaults to all registered criteria).
+- `--criteria` (Optional): Repeating option to choose specific criteria (defaults to all registered criteria). *Note: Omitting this option runs all criteria in the registry. As new criteria are added in the future, evaluating all criteria by default will increase latency and compute cost.*
 
 ---
 
@@ -116,11 +129,17 @@ Measures whether the response's claims are semantically aligned and supported by
 ### 2. Refusal Detection (`refusal_check`)
 Identifies if the model declined to answer the prompt.
 - **Smart Tail Filtering:** If a refusal-like phrase is detected at the start of a response but is followed by a substantial tail of actual content (exceeding `120` characters), it is treated as incidental language rather than a refusal.
-- **Configurable:** Refusal phrase lists, checking windows, and tail thresholds are loaded from `refusal_config.json`.
+- **Configurable:** Refusal phrase lists, checking windows, and tail thresholds are loaded from `refusal_config.json`. Supports multilingual variants (including Urdu refusal heuristics).
+
+### 3. Relevance (`relevance`)
+Evaluates semantic relevance between the prompt and the response.
+- **Dense Cosine Similarity:** Encodes the prompt and response with `all-MiniLM-L6-v2` and computes cosine similarity.
+- **Defensive Pre-Encoding:** Handles empty or whitespace prompts/responses safely without unnecessary transformer calls.
 
-### 3. Relevance (`relevance`) & Completeness (`completeness`) *(Under Development)*
-- **Relevance:** Computes cosine similarity between the prompt and the response.
-- **Completeness:** Heuristically splits the prompt into aspects (using delimiters like "and", commas, etc.) and calculates whether the response covers each aspect.
+### 4. Completeness (`completeness`)
+Evaluates whether all sub-questions or clauses in the user's prompt were addressed by the response.
+- **Aspect Extraction:** Decomposes complex prompts into semantic aspects and checks coverage against response sentences.
+- **Granular Accounting:** Returns detailed breakdowns (`covered_aspects`, `total_aspects`, and explanation).
 
 ---
 
@@ -165,6 +184,9 @@ def my_check(prompt: str, response: str, context: str = "", **kwargs) -> dict:
 from llm_eval_kit.criteria import my_check
 ```
 
+#### Evaluation Latency & Future Criteria
+Because `Evaluator.evaluate()` defaults to executing all registered criteria when `criteria` is omitted, adding future criteria (particularly those requiring deep embedding models or multi-aspect passes) will increase evaluation latency and resource consumption for default runs. In production workflows or CI environments with strict latency budgets, callers should pass an explicit list of required criteria (e.g., `criteria=["refusal_check"]`).
+
 ---
 
 ## Known Limitations

lab_note_lead_engineer.md

@@ -0,0 +1,15 @@
+# Architecting an Extensible, Deterministic Offline LLM Evaluation Engine
+
+Modern LLM evaluation workflows routinely depend on third-party "LLM-as-a-judge" APIs or human review teams. While flexible, this paradigm introduces severe latency bottlenecks, non-deterministic scores, prohibitive costs, and privacy vulnerabilities that prevent continuous evaluation within automated CI/CD pipelines. As Lead Engineer on `llm-eval-kit`, my objective was to design and build a clean, deterministic, and fully offline Python library capable of evaluating model outputs in milliseconds without cloud dependencies, subscriptions, or external network calls.
+
+**System Architecture and the Registry Pattern.** The core architectural challenge was designing an evaluation engine that could scale across arbitrary criteria without creating a fragile monolithic codebase. Traditional evaluation pipelines rely on hardcoded conditional branches that require modifying the central orchestrator whenever a new metric is introduced. To eliminate this coupling, I designed a decorator-based criteria registry (`registry.py`) that maps criterion identifiers to their respective checking functions at import time. The central `Evaluator` class (`evaluator.py`) never interacts with criterion implementations directly; it solely inspects `CRITERIA_REGISTRY`. This guarantees that adding future criteria requires writing only an isolated function decorated with `@register_criterion`, without touching a single line of core orchestration code.
+
+**Orchestration, Fail-Fast Guarantees, and Developer Experience.** In production pipelines, failing late after consuming heavy compute cycles is unacceptable. I implemented a strict fail-fast validation layer in `Evaluator.evaluate()`: before invoking any embedding models or compute-heavy heuristics, the engine cross-references all requested criteria against the registry. If an unknown or misspelled name is detected, execution halts immediately with a descriptive `ValueError`. Furthermore, I established a deliberate scoring convention: criteria that cannot run due to omitted inputs (e.g., factual grounding when reference context is absent) return `score: None` and are cleanly excluded from the overall average rather than penalized as zeros. To make the tool immediately accessible across CI workflows, I engineered a zero-overhead CLI wrapper (`cli.py`) powered by Typer, allowing developers to pipe model inputs and receive structured, machine-readable JSON outputs directly in their terminal.
+
+**Packaging, Integration, and Engineering Outcomes.** Delivering a truly offline library required addressing subtle distribution pitfalls. I structured the project under `pyproject.toml` with explicit package data inclusion rules (`[tool.setuptools.package-data]`) to guarantee that static criterion configuration JSON files are bundled into distribution wheels, avoiding runtime `FileNotFoundError` exceptions in clean environments. Coordinating with Muhammad Maaz (Research Engineer) and Warisha Arshad (Integration Engineer), we established clear interface contracts that enabled independent workstreams across criteria development, fixture authoring, and integration testing. The combined test suite spans unit tests, regression tests, edge-case validation, and performance benchmarks—verifying sub-second sentence segmentation, memory-safe embedding singletons, and overall evaluation well within our 2,000ms SLA.
+
+**What I Learned.** Leading this project reinforced the primacy of architectural minimalism. Establishing explicit interface contracts and single-responsibility boundaries early saved dozens of hours of integration churn. Designing for offline-first reliability proved that intelligent combination of dense semantic representations, symbolic parsing, and rule-based heuristics can match or exceed expensive LLM judges for targeted regression testing in continuous delivery.
+
+Mahrukh Baig  
+Lead Engineer  
+INFERENCE Lab Engineering Fellowship, Cohort 01

llm_eval_kit/__init__.py

@@ -1,6 +1,13 @@
-from llm_eval_kit.criteria import (
-    refusal,
-    factual_grounding,
-    relevance,
-    completeness,
-)
+from llm_eval_kit.evaluator import Evaluator
+from llm_eval_kit.registry import available_criteria
+from llm_eval_kit.criteria import (
+    refusal,
+    factual_grounding,
+    relevance,
+    completeness,
+)
+
+__all__ = [
+    "Evaluator",
+    "available_criteria",
+]

pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "llm-eval-kit"
+version = "0.1.0"
+authors = [
+    { name = "Mahrukh Baig", email = "mahrukh@example.com" }
+]
+description = "A lightweight, offline Python library for scoring LLM response quality."
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "sentence-transformers>=2.2.0",
+    "scikit-learn>=1.0.0",
+    "typer>=0.9.0",
+]
+
+[project.scripts]
+llm-eval = "llm_eval_kit.cli:app"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["llm_eval_kit*"]
+
+[tool.setuptools.package-data]
+"*" = ["*.json"]
+
+[tool.pytest.ini_options]
+markers = [
+    "integration: marks tests as integration tests (requiring model downloads or pre-cached models)",
+    "performance: marks tests as performance benchmarks (measuring latency or resource usage)",
+]
+

tests/test_cli.py

@@ -60,14 +60,17 @@ def test_cli_evaluate_multiple_criteria_flags():
         app,
         [
             "evaluate",
-            "--prompt", "Hello",
-            "--response", "Hi there!",
+            "--prompt", "What is the capital of France?",
+            "--response", "Paris is the capital of France.",
+            "--context", "Paris is the capital and most populous city of France.",
             "--criteria", "refusal_check",
+            "--criteria", "factual_grounding",
         ],
     )
     assert result.exit_code == 0
     output = json.loads(result.stdout)
     assert "refusal_check" in output["criteria"]
+    assert "factual_grounding" in output["criteria"]
 
 
 def test_cli_evaluate_missing_required_prompt_fails():

tests/test_evaluator.py

@@ -19,6 +19,20 @@
 import llm_eval_kit.criteria.refusal  # noqa: F401 -- ensures refusal_check is registered
 
 
+@pytest.fixture(autouse=True)
+def restore_criteria_registry():
+    """Takes a snapshot of the global CRITERIA_REGISTRY before each test
+    and restores it upon completion.
+
+    This ensures tests registering ad-hoc or mock criteria do not leak
+    state into subsequent tests, making the suite order-independent.
+    """
+    original_registry = dict(CRITERIA_REGISTRY)
+    yield
+    CRITERIA_REGISTRY.clear()
+    CRITERIA_REGISTRY.update(original_registry)
+
+
 def test_evaluate_runs_default_criteria_when_none_given():
     """If criteria=None, Evaluator should run every registered criterion."""
     ev = Evaluator()

tests/test_factual_grounding.py

@@ -17,6 +17,12 @@
 
 from llm_eval_kit.criteria.factual_grounding import factual_grounding
 
+# Mark all tests in this file as integration tests since factual_grounding
+# requires loading the sentence-transformers model (all-MiniLM-L6-v2).
+# Regular test suites can exclude these using `pytest -m "not integration"`
+# to run fast and offline without downloading models.
+pytestmark = pytest.mark.integration
+
 FIXTURES_PATH = Path(__file__).parent / "fixtures" / "criteria_fixtures.json"
 
 with open(FIXTURES_PATH, "r", encoding="utf-8") as f:

tests/test_performance.py

@@ -26,12 +26,22 @@
 here too -- no change needed to this file.
 """
 
+import os
 import time
 
+import pytest
+
 from llm_eval_kit.evaluator import Evaluator
 from llm_eval_kit.registry import CRITERIA_REGISTRY
 
-PERFORMANCE_TARGET_MS = 2000
+# Mark all tests in this module as performance benchmarks and integration tests.
+# CI runners can skip these using `pytest -m "not performance"` or `pytest -m "not integration"`.
+pytestmark = [pytest.mark.performance, pytest.mark.integration]
+
+# Target evaluation latency per DESIGN.md Section 8 (default: 2000ms).
+# Can be overridden via the PERFORMANCE_TARGET_MS environment variable to prevent
+# flakiness across varied or resource-constrained CI runner environments.
+PERFORMANCE_TARGET_MS = int(os.getenv("PERFORMANCE_TARGET_MS", "2000"))
 
 
 def test_full_evaluation_under_two_second_target():