From f5ce67bea2e55631fd0e112cc06a5ac97c50029b Mon Sep 17 00:00:00 2001 From: MahrukhBaig Date: Sun, 6 Sep 2026 00:08:51 +0500 Subject: [PATCH 1/2] Complete v0.1.0 release: export Evaluator, package JSON data, polish README, and add Lead Engineer lab note --- .gitignore | 7 +++++++ README.md | 34 +++++++++++++++++++++++++--------- lab_note_lead_engineer.md | 15 +++++++++++++++ llm_eval_kit/__init__.py | 19 +++++++++++++------ pyproject.toml | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 15 deletions(-) create mode 100644 lab_note_lead_engineer.md create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index 9491ebd..0a52245 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ *.docx __pycache__/ .vscode/ +.pytest_cache/ +*.egg-info/ +dist/ +build/ +*env/ +*.venv/ + diff --git a/README.md b/README.md index 5ee65ee..31fabc8 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ pip install llm-eval-kit Here is a 4-line example demonstrating how to run the evaluator: ```python -from llm_eval_kit.evaluator import Evaluator +from llm_eval_kit import Evaluator # Initialize the evaluator evaluator = Evaluator() @@ -47,12 +47,12 @@ prompt = "What is the boiling point of water at sea level?" response = "Water boils at 100 degrees Celsius at sea level." context = "At sea level, the boiling point of water is 100°C (212°F)." -# Run the evaluation +# Run the evaluation (specify criteria or omit to run all registered criteria) result = evaluator.evaluate( prompt=prompt, response=response, context=context, - criteria=["refusal_check", "factual_grounding"] + criteria=["refusal_check", "factual_grounding", "relevance", "completeness"] ) # Output is a structured Python dictionary @@ -63,7 +63,7 @@ print(json.dumps(result, indent=2)) ### Example Output JSON ```json { - "overall_score": 1.0, + "overall_score": 0.95, "criteria": { "refusal_check": { "score": 1.0, @@ -71,9 +71,19 @@ print(json.dumps(result, indent=2)) "explanation": "No refusal phrase detected near the start of the response." }, "factual_grounding": { - "score": 0.985, + "score": 0.88, "explanation": "Score reflects semantic similarity with a numeric-consistency penalty applied, not calibrated factual accuracy. Response numeric claims match or are supported by the context.", "best_matching_sentence": "At sea level, the boiling point of water is 100°C (212°F)." + }, + "relevance": { + "score": 0.92, + "explanation": "Response is highly relevant and directly addresses the prompt topic." + }, + "completeness": { + "score": 1.0, + "explanation": "Response covers all key aspects requested in the prompt.", + "covered_aspects": ["boiling point of water at sea level"], + "total_aspects": 1 } }, "metadata": { @@ -116,11 +126,17 @@ Measures whether the response's claims are semantically aligned and supported by ### 2. Refusal Detection (`refusal_check`) Identifies if the model declined to answer the prompt. - **Smart Tail Filtering:** If a refusal-like phrase is detected at the start of a response but is followed by a substantial tail of actual content (exceeding `120` characters), it is treated as incidental language rather than a refusal. -- **Configurable:** Refusal phrase lists, checking windows, and tail thresholds are loaded from `refusal_config.json`. +- **Configurable:** Refusal phrase lists, checking windows, and tail thresholds are loaded from `refusal_config.json`. Supports multilingual variants (including Urdu refusal heuristics). + +### 3. Relevance (`relevance`) +Evaluates semantic relevance between the prompt and the response. +- **Dense Cosine Similarity:** Encodes the prompt and response with `all-MiniLM-L6-v2` and computes cosine similarity. +- **Defensive Pre-Encoding:** Handles empty or whitespace prompts/responses safely without unnecessary transformer calls. -### 3. Relevance (`relevance`) & Completeness (`completeness`) *(Under Development)* -- **Relevance:** Computes cosine similarity between the prompt and the response. -- **Completeness:** Heuristically splits the prompt into aspects (using delimiters like "and", commas, etc.) and calculates whether the response covers each aspect. +### 4. Completeness (`completeness`) +Evaluates whether all sub-questions or clauses in the user's prompt were addressed by the response. +- **Aspect Extraction:** Decomposes complex prompts into semantic aspects and checks coverage against response sentences. +- **Granular Accounting:** Returns detailed breakdowns (`covered_aspects`, `total_aspects`, and explanation). --- diff --git a/lab_note_lead_engineer.md b/lab_note_lead_engineer.md new file mode 100644 index 0000000..927b043 --- /dev/null +++ b/lab_note_lead_engineer.md @@ -0,0 +1,15 @@ +# Architecting an Extensible, Deterministic Offline LLM Evaluation Engine + +Modern LLM evaluation workflows routinely depend on third-party "LLM-as-a-judge" APIs or human review teams. While flexible, this paradigm introduces severe latency bottlenecks, non-deterministic scores, prohibitive costs, and privacy vulnerabilities that prevent continuous evaluation within automated CI/CD pipelines. As Lead Engineer on `llm-eval-kit`, my objective was to design and build a clean, deterministic, and fully offline Python library capable of evaluating model outputs in milliseconds without cloud dependencies, subscriptions, or external network calls. + +**System Architecture and the Registry Pattern.** The core architectural challenge was designing an evaluation engine that could scale across arbitrary criteria without creating a fragile monolithic codebase. Traditional evaluation pipelines rely on hardcoded conditional branches that require modifying the central orchestrator whenever a new metric is introduced. To eliminate this coupling, I designed a decorator-based criteria registry (`registry.py`) that maps criterion identifiers to their respective checking functions at import time. The central `Evaluator` class (`evaluator.py`) never interacts with criterion implementations directly; it solely inspects `CRITERIA_REGISTRY`. This guarantees that adding future criteria requires writing only an isolated function decorated with `@register_criterion`, without touching a single line of core orchestration code. + +**Orchestration, Fail-Fast Guarantees, and Developer Experience.** In production pipelines, failing late after consuming heavy compute cycles is unacceptable. I implemented a strict fail-fast validation layer in `Evaluator.evaluate()`: before invoking any embedding models or compute-heavy heuristics, the engine cross-references all requested criteria against the registry. If an unknown or misspelled name is detected, execution halts immediately with a descriptive `ValueError`. Furthermore, I established a deliberate scoring convention: criteria that cannot run due to omitted inputs (e.g., factual grounding when reference context is absent) return `score: None` and are cleanly excluded from the overall average rather than penalized as zeros. To make the tool immediately accessible across CI workflows, I engineered a zero-overhead CLI wrapper (`cli.py`) powered by Typer, allowing developers to pipe model inputs and receive structured, machine-readable JSON outputs directly in their terminal. + +**Packaging, Integration, and Engineering Outcomes.** Delivering a truly offline library required addressing subtle distribution pitfalls. I structured the project under `pyproject.toml` with explicit package data inclusion rules (`[tool.setuptools.package-data]`) to guarantee that static criterion configuration JSON files are bundled into distribution wheels, avoiding runtime `FileNotFoundError` exceptions in clean environments. Coordinating with Muhammad Maaz (Research Engineer) and Warisha Arshad (Integration Engineer), we established clear interface contracts that enabled independent workstreams across criteria development, fixture authoring, and integration testing. The combined test suite spans unit tests, regression tests, edge-case validation, and performance benchmarks—verifying sub-second sentence segmentation, memory-safe embedding singletons, and overall evaluation well within our 2,000ms SLA. + +**What I Learned.** Leading this project reinforced the primacy of architectural minimalism. Establishing explicit interface contracts and single-responsibility boundaries early saved dozens of hours of integration churn. Designing for offline-first reliability proved that intelligent combination of dense semantic representations, symbolic parsing, and rule-based heuristics can match or exceed expensive LLM judges for targeted regression testing in continuous delivery. + +Mahrukh Baig +Lead Engineer + diff --git a/llm_eval_kit/__init__.py b/llm_eval_kit/__init__.py index 17254e1..541897d 100644 --- a/llm_eval_kit/__init__.py +++ b/llm_eval_kit/__init__.py @@ -1,6 +1,13 @@ -from llm_eval_kit.criteria import ( - refusal, - factual_grounding, - relevance, - completeness, -) \ No newline at end of file +from llm_eval_kit.evaluator import Evaluator +from llm_eval_kit.registry import available_criteria +from llm_eval_kit.criteria import ( + refusal, + factual_grounding, + relevance, + completeness, +) + +__all__ = [ + "Evaluator", + "available_criteria", +] \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b506e1d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "llm-eval-kit" +version = "0.1.0" +authors = [ + { name = "Mahrukh Baig", email = "mahrukh@example.com" } +] +description = "A lightweight, offline Python library for scoring LLM response quality." +readme = "README.md" +requires-python = ">=3.9" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = [ + "sentence-transformers>=2.2.0", + "scikit-learn>=1.0.0", + "typer>=0.9.0", +] + +[project.scripts] +llm-eval = "llm_eval_kit.cli:app" + +[tool.setuptools.packages.find] +where = ["."] +include = ["llm_eval_kit*"] + +[tool.setuptools.package-data] +"*" = ["*.json"] + From 57a4cb6f1edabfc7c5b550cf8d7d485ba17d4eb4 Mon Sep 17 00:00:00 2001 From: MahrukhBaig Date: Mon, 7 Sep 2026 00:44:37 +0500 Subject: [PATCH 2/2] Update test suite reliability, registry isolation, and documentation --- README.md | 8 +++++++- lab_note_lead_engineer.md | 2 +- pyproject.toml | 6 ++++++ tests/test_cli.py | 7 +++++-- tests/test_evaluator.py | 14 ++++++++++++++ tests/test_factual_grounding.py | 6 ++++++ tests/test_performance.py | 12 +++++++++++- 7 files changed, 50 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 31fabc8..eda2881 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,9 @@ import json print(json.dumps(result, indent=2)) ``` +> **Note on Evaluation Cost & Latency:** +> Omitting the `criteria` argument (or passing `criteria=None`) executes all registered criteria in `CRITERIA_REGISTRY`. As additional criteria are added in the future (especially compute-heavy transformer models or multi-aspect passes), evaluating all criteria by default will proportionally increase the evaluation latency and resource cost. In production or latency-critical pipelines, it is best practice to explicitly specify only the criteria you need. + ### Example Output JSON ```json { @@ -112,7 +115,7 @@ llm-eval evaluate \ - `--prompt` (Required): The original prompt sent to the LLM. - `--response` (Required): The LLM response to evaluate. - `--context` (Optional): Reference text for grounding. -- `--criteria` (Optional): Repeating option to choose specific criteria (defaults to all registered criteria). +- `--criteria` (Optional): Repeating option to choose specific criteria (defaults to all registered criteria). *Note: Omitting this option runs all criteria in the registry. As new criteria are added in the future, evaluating all criteria by default will increase latency and compute cost.* --- @@ -181,6 +184,9 @@ def my_check(prompt: str, response: str, context: str = "", **kwargs) -> dict: from llm_eval_kit.criteria import my_check ``` +#### Evaluation Latency & Future Criteria +Because `Evaluator.evaluate()` defaults to executing all registered criteria when `criteria` is omitted, adding future criteria (particularly those requiring deep embedding models or multi-aspect passes) will increase evaluation latency and resource consumption for default runs. In production workflows or CI environments with strict latency budgets, callers should pass an explicit list of required criteria (e.g., `criteria=["refusal_check"]`). + --- ## Known Limitations diff --git a/lab_note_lead_engineer.md b/lab_note_lead_engineer.md index 927b043..2f1c231 100644 --- a/lab_note_lead_engineer.md +++ b/lab_note_lead_engineer.md @@ -12,4 +12,4 @@ Modern LLM evaluation workflows routinely depend on third-party "LLM-as-a-judge" Mahrukh Baig Lead Engineer - +INFERENCE Lab Engineering Fellowship, Cohort 01 diff --git a/pyproject.toml b/pyproject.toml index b506e1d..d1e0eb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,3 +32,9 @@ include = ["llm_eval_kit*"] [tool.setuptools.package-data] "*" = ["*.json"] +[tool.pytest.ini_options] +markers = [ + "integration: marks tests as integration tests (requiring model downloads or pre-cached models)", + "performance: marks tests as performance benchmarks (measuring latency or resource usage)", +] + diff --git a/tests/test_cli.py b/tests/test_cli.py index e2329aa..9e24755 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -60,14 +60,17 @@ def test_cli_evaluate_multiple_criteria_flags(): app, [ "evaluate", - "--prompt", "Hello", - "--response", "Hi there!", + "--prompt", "What is the capital of France?", + "--response", "Paris is the capital of France.", + "--context", "Paris is the capital and most populous city of France.", "--criteria", "refusal_check", + "--criteria", "factual_grounding", ], ) assert result.exit_code == 0 output = json.loads(result.stdout) assert "refusal_check" in output["criteria"] + assert "factual_grounding" in output["criteria"] def test_cli_evaluate_missing_required_prompt_fails(): diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 79c2909..64d296b 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -19,6 +19,20 @@ import llm_eval_kit.criteria.refusal # noqa: F401 -- ensures refusal_check is registered +@pytest.fixture(autouse=True) +def restore_criteria_registry(): + """Takes a snapshot of the global CRITERIA_REGISTRY before each test + and restores it upon completion. + + This ensures tests registering ad-hoc or mock criteria do not leak + state into subsequent tests, making the suite order-independent. + """ + original_registry = dict(CRITERIA_REGISTRY) + yield + CRITERIA_REGISTRY.clear() + CRITERIA_REGISTRY.update(original_registry) + + def test_evaluate_runs_default_criteria_when_none_given(): """If criteria=None, Evaluator should run every registered criterion.""" ev = Evaluator() diff --git a/tests/test_factual_grounding.py b/tests/test_factual_grounding.py index 1cae8ea..7e90a73 100644 --- a/tests/test_factual_grounding.py +++ b/tests/test_factual_grounding.py @@ -17,6 +17,12 @@ from llm_eval_kit.criteria.factual_grounding import factual_grounding +# Mark all tests in this file as integration tests since factual_grounding +# requires loading the sentence-transformers model (all-MiniLM-L6-v2). +# Regular test suites can exclude these using `pytest -m "not integration"` +# to run fast and offline without downloading models. +pytestmark = pytest.mark.integration + FIXTURES_PATH = Path(__file__).parent / "fixtures" / "criteria_fixtures.json" with open(FIXTURES_PATH, "r", encoding="utf-8") as f: diff --git a/tests/test_performance.py b/tests/test_performance.py index 7f2fa97..dd0900e 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -26,12 +26,22 @@ here too -- no change needed to this file. """ +import os import time +import pytest + from llm_eval_kit.evaluator import Evaluator from llm_eval_kit.registry import CRITERIA_REGISTRY -PERFORMANCE_TARGET_MS = 2000 +# Mark all tests in this module as performance benchmarks and integration tests. +# CI runners can skip these using `pytest -m "not performance"` or `pytest -m "not integration"`. +pytestmark = [pytest.mark.performance, pytest.mark.integration] + +# Target evaluation latency per DESIGN.md Section 8 (default: 2000ms). +# Can be overridden via the PERFORMANCE_TARGET_MS environment variable to prevent +# flakiness across varied or resource-constrained CI runner environments. +PERFORMANCE_TARGET_MS = int(os.getenv("PERFORMANCE_TARGET_MS", "2000")) def test_full_evaluation_under_two_second_target():