diff --git a/packages/evaluator/src/rag_forge_evaluator/engines/ragas_adapters.py b/packages/evaluator/src/rag_forge_evaluator/engines/ragas_adapters.py index a2a0033..c315aac 100644 --- a/packages/evaluator/src/rag_forge_evaluator/engines/ragas_adapters.py +++ b/packages/evaluator/src/rag_forge_evaluator/engines/ragas_adapters.py @@ -7,28 +7,50 @@ we honor --judge claude end-to-end and stay version-stable across ragas releases. -ragas 0.4.x contract (from src/ragas/llms/base.py):: +ragas 0.4.x contract (from src/ragas/llms/base.py), full public surface +the wrapper must honour: class BaseRagasLLM(ABC): - def generate_text( - self, - prompt: PromptValue, - n: int = 1, - temperature: float | None = 0.01, - stop: list[str] | None = None, - callbacks: Callbacks = None, - ) -> LLMResult: ... - - async def agenerate_text(...) -> LLMResult: ... - -Our wrappers implement the full signature so ragas receives exactly what -its internal metric implementations expect, rather than duck-typing a -narrow subset that may break on minor version bumps. ``n``, ``temperature``, -``stop``, and ``callbacks`` are accepted for contract compatibility but -currently forwarded to the underlying rag-forge Judge only insofar as the -Judge interface supports them — the Judge protocol is a simple -``(system_prompt, user_prompt) -> str`` call, so extra kwargs are -gracefully ignored. + run_config: RunConfig + + # abstract — must be implemented + def generate_text(self, prompt, n, temperature, stop, callbacks) -> LLMResult: ... + async def agenerate_text(self, prompt, n, temperature, stop, callbacks) -> LLMResult: ... + def is_finished(self, response: LLMResult) -> bool: ... + + # concrete helpers on the base class that ragas's metric code CALLS + # on every LLM (not just ones that subclass BaseRagasLLM). A duck- + # typed wrapper must reimplement these or it will crash the moment + # a metric invokes them: + async def generate(self, prompt, n, temperature, stop, callbacks) -> LLMResult: ... + def get_temperature(self, n: int) -> float: ... + def set_run_config(self, run_config: RunConfig) -> None: ... + + class BaseRagasEmbeddings(ABC): + run_config: RunConfig + def embed_query(self, text) -> list[float]: ... + def embed_documents(self, texts) -> list[list[float]]: ... + async def aembed_query(self, text) -> list[float]: ... + async def aembed_documents(self, texts) -> list[list[float]]: ... + def set_run_config(self, run_config: RunConfig) -> None: ... + +``n``, ``temperature``, ``stop``, and ``callbacks`` are accepted for +contract compatibility but currently forwarded to the underlying rag-forge +Judge only insofar as the Judge interface supports them — the Judge +protocol is a simple ``(system_prompt, user_prompt) -> str`` call, so +extra kwargs are gracefully ignored. + +**History:** v0.2.0 shipped only ``generate_text`` / ``agenerate_text`` / +``model_name`` under the assumption those were the only methods ragas +called on an LLM. The Cycle 3 PearMedica audit (2026-04-15) proved that +assumption wrong — ragas's metric code calls the concrete +``BaseRagasLLM.generate`` wrapper, which in turn calls ``agenerate_text`` +with retry. v0.2.2 adds the missing ``generate`` / ``get_temperature`` / +``is_finished`` / ``set_run_config`` shims so the duck-typed wrapper +matches the full base-class surface. A contract test in +``tests/test_ragas_adapters_contract.py`` now iterates every public method +on ``BaseRagasLLM`` and ``BaseRagasEmbeddings`` and asserts each is +implemented, so this class of bug cannot silently ship again. ragas and langchain imports are deferred into method bodies so this module stays importable without either installed — unit tests run @@ -94,6 +116,60 @@ def _wrap_as_llm_result(text: str) -> LLMResult: return _LLMResult(generations=[[Generation(text=text)]]) +def _fuse_llm_results(results: list[Any]) -> Any: + """Fuse N single-generation result objects into one + ``[[gen1..genN]]``-shaped result. + + ragas expects ``LLMResult.generations`` to be a list whose outer + length is the number of prompts (1 for a single prompt) and whose + inner length is the number of samples per prompt (N for ``n>1``). + Our ``agenerate_text`` path produces one generation per call, so + to honour the ``n>1`` contract we fan out ``n`` calls and reshape + them here. + + Handles both of the result shapes ``agenerate_text`` may return: + + - The real ``langchain_core.outputs.LLMResult`` (production path + with the ``[ragas]`` extra installed). + - The ``_StringLLMResult`` stub (unit-test path without langchain + — shipped by this module for environments that don't install + the optional extras). + + Flattens the ``[[gen]]`` shape of each input into a single + ``[[gen, gen, ..., gen]]`` shape on the output. **Raises + ``ValueError`` on malformed input** — e.g. an item that is not + an LLMResult, a missing ``.generations`` attribute, an empty + outer list, or a non-iterable inner element. CodeRabbit on PR + #36 round 2 flagged an earlier silent ``results[0]`` fallback + as a real correctness bug: collapsing a requested ``n>3`` into + a single sample skews downstream ragas scoring without any + signal. Fail loud instead. + """ + if not results: + msg = "_fuse_llm_results called with empty results list" + raise ValueError(msg) + if len(results) == 1: + return results[0] + try: + fused_generations = [gen for r in results for gen in r.generations[0]] + except (AttributeError, IndexError, TypeError) as exc: + msg = ( + f"_fuse_llm_results received a malformed result: {exc!r}. " + f"Every item must expose .generations[0] as a non-empty " + f"iterable of Generation objects. Got result types: " + f"{[type(r).__name__ for r in results]}" + ) + raise ValueError(msg) from exc + # Prefer the real langchain LLMResult when langchain is installed; + # otherwise construct a _StringLLMResult carrying the fused list. + try: + from langchain_core.outputs import LLMResult as _LLMResult + + return _LLMResult(generations=[fused_generations]) + except ImportError: + return _StringLLMResult._from_generations(fused_generations) + + class RagForgeRagasLLM: """ragas LLM wrapper that forwards to a rag-forge Judge. @@ -114,9 +190,10 @@ class RagForgeRagasLLM: The class does NOT subclass ``BaseRagasLLM`` because that would force a hard import of ragas at module load time and break unit tests on machines without the ``[ragas]`` extra installed. It implements the - duck-typed contract — ragas only calls ``generate_text`` / - ``agenerate_text`` / ``model_name``, all of which we provide with - correct signatures and return types. + duck-typed contract by re-declaring every public method ragas calls — + see the module docstring for the full surface. The contract test at + ``tests/test_ragas_adapters_contract.py`` is the tripwire that keeps + this in sync with future ragas releases. """ def __init__( @@ -128,6 +205,7 @@ def __init__( self._judge = judge self._system_prompt = system_prompt self._refusal_aware = refusal_aware + self.run_config: Any = None def _complete(self, prompt: Any) -> str: """Internal: extract text, inject refusal note, call judge.""" @@ -186,6 +264,118 @@ def model_name(self) -> str: """Return the model name from the underlying judge.""" return self._judge.model_name() + async def generate( + self, + prompt: Any, + n: int = 1, + temperature: float | None = 0.01, + stop: list[str] | None = None, + callbacks: Callbacks | None = None, + ) -> Any: + """Async concrete shim that mirrors ``BaseRagasLLM.generate``. + + ragas's metric code calls ``await llm.generate(prompt, ...)`` on + every LLM, regardless of whether the LLM subclasses + ``BaseRagasLLM``. Because this wrapper is duck-typed (see class + docstring for why we do not subclass), we must re-declare + ``generate`` here or every RAGAS evaluation crashes on first job + with ``AttributeError: no attribute 'generate'`` — the Cycle 3 + regression that motivated v0.2.2. + + Default temperature 0.01 matches ``BaseRagasLLM.generate``. + Passing ``None`` explicitly still triggers the + ``get_temperature(n)`` fallback, same as the base class. + + **Sample diversity (``n > 1``)** — ragas uses ``n > 1`` for + metrics that need multiple samples (e.g. ``answer_correctness`` + consistency checks). ``LLMResult.generations`` must be shaped + ``[[gen1, gen2, ..., genN]]`` — a single prompt run containing + N candidate generations. The rag-forge ``Judge`` protocol is + synchronous and stateless, so we invoke the judge ``n`` times + in parallel via ``asyncio.gather`` and fuse the per-call + results into the expected shape. For ``n == 1`` (the common + case) this is a single call, same cost as before. + + We intentionally skip the base class's ``add_async_retry`` + wrapper — our underlying Judge implementations already own + their own retry policy (see e.g. ClaudeJudge's 529 handling) + and double-wrapping leads to retry storms. + """ + if temperature is None: + temperature = self.get_temperature(n) + if n <= 1: + result = await self.agenerate_text( + prompt, + n=n, + temperature=temperature, + stop=stop, + callbacks=callbacks, + ) + else: + # Fan out n independent calls and fuse into a single + # [[gen1..genN]] LLMResult so ragas's multi-sample metrics + # see the shape they expect. asyncio.gather runs the + # per-call agenerate_text invocations concurrently via + # the worker-thread path. + per_call_results = await asyncio.gather( + *( + self.agenerate_text( + prompt, + n=1, + temperature=temperature, + stop=stop, + callbacks=callbacks, + ) + for _ in range(n) + ) + ) + result = _fuse_llm_results(per_call_results) + if not self.is_finished(result): + msg = ( + "RagForgeRagasLLM.generate: underlying judge response did " + "not finish cleanly (is_finished == False). This is a " + "defensive check; override is_finished if your Judge " + "exposes a real finish signal." + ) + raise RuntimeError(msg) + return result + + def is_finished(self, response: Any) -> bool: + """Report whether a generation finished cleanly. + + ragas uses this to decide whether to retry on truncation. Our + Judge protocol does not currently expose a finish-reason, so we + conservatively report True — the underlying Judge implementation + is responsible for retrying on truncation (see Cycle 2 Finding + #5 and the ClaudeJudge ``max_tokens`` raise). + + If a future Judge exposes a finish-reason signal, this method is + the place to propagate it. + """ + _ = response + return True + + def get_temperature(self, n: int) -> float: + """Return a sampling temperature given the requested ``n``. + + Matches ragas's own convention: near-deterministic for a single + sample, warmer when the caller asks for multiple. ragas uses + this to diversify samples for metrics that rely on sampling + stability (e.g. ``answer_correctness``). + """ + return 0.01 if n <= 1 else 0.3 + + def set_run_config(self, run_config: Any) -> None: + """Store a ragas ``RunConfig`` for later use. + + ragas's evaluation runner injects its ``RunConfig`` (timeouts, + retry budgets, worker counts) into every LLM and embeddings + wrapper before kicking off metric jobs. We store it for contract + compatibility; our actual execution path honours the Judge's + own timeout/retry configuration rather than ragas's. + """ + self.run_config = run_config + class _StringLLMResult: """Minimal duck-typed stand-in for ``langchain_core.outputs.LLMResult``. @@ -200,6 +390,23 @@ def __init__(self, text: str) -> None: self._text = text self.generations = [[_StringGeneration(text)]] + @classmethod + def _from_generations(cls, generations: list[Any]) -> _StringLLMResult: + """Build a stub carrying a pre-fused ``[[gen1..genN]]`` list. + + Used by ``_fuse_llm_results`` on the no-langchain code path + when ``n > 1`` in ``RagForgeRagasLLM.generate`` fans out + multiple calls and needs to collapse them into one result + object. The ``_text`` property falls back to the first + generation's text so ``str(result)`` and ``result == "x"`` + remain useful on the fused object. + """ + instance = cls.__new__(cls) + first_text = generations[0].text if generations else "" + instance._text = first_text + instance.generations = [generations] + return instance + @property def text(self) -> str: return self._text @@ -275,6 +482,7 @@ def __init__( raise ValueError(msg) self._provider = provider self._model = model + self.run_config: Any = None def embed_query(self, text: str) -> list[float]: if self._provider == "mock": @@ -326,6 +534,42 @@ async def aembed_documents(self, texts: list[str]) -> list[list[float]]: """Async variant of ``embed_documents``.""" return await asyncio.to_thread(self.embed_documents, texts) + def set_run_config(self, run_config: Any) -> None: + """Store a ragas ``RunConfig`` for contract compatibility. + + See ``RagForgeRagasLLM.set_run_config`` for rationale. + """ + self.run_config = run_config + + async def embed_text(self, text: str, is_async: bool = True) -> list[float]: + """Dispatch helper matching ``BaseRagasEmbeddings.embed_text``. + + **This method is ``async``, not ``def``.** ragas's real + ``BaseRagasEmbeddings.embed_text`` is an ``async`` coroutine — + metric code invokes it with ``await embeddings.embed_text(...)``. + v0.2.2 originally shipped this as a sync method that called + ``asyncio.run(self.aembed_query(text))``, which crashes with + ``RuntimeError: asyncio.run() cannot be called from a running + event loop`` because ragas's evaluation runner is itself + inside an event loop. Caught by CodeRabbit on PR #36. + + ``is_async`` is accepted for signature parity with the base + class but ignored — our underlying embedding clients are + synchronous and ``aembed_query`` already runs them in a worker + thread via ``asyncio.to_thread``, so both flag values land on + the same code path. + """ + _ = is_async + return await self.aembed_query(text) + + async def embed_texts(self, texts: list[str], is_async: bool = True) -> list[list[float]]: + """Batch variant of ``embed_text`` — same async contract. + + See ``embed_text`` for the async / ``asyncio.run()`` history. + """ + _ = is_async + return await self.aembed_documents(texts) + @classmethod def _mock_embed(cls, text: str) -> list[float]: """Deterministic fake embedding: SHA-256 the text, take 8 bytes, diff --git a/packages/evaluator/tests/test_ragas_adapters.py b/packages/evaluator/tests/test_ragas_adapters.py index 812390b..ff97246 100644 --- a/packages/evaluator/tests/test_ragas_adapters.py +++ b/packages/evaluator/tests/test_ragas_adapters.py @@ -1,6 +1,7 @@ """Tests for ragas adapter wrappers.""" import asyncio +from typing import Any from unittest.mock import MagicMock, patch import pytest @@ -11,6 +12,20 @@ ) +def _llm_result_text(result: Any) -> str: + """Extract the generated text from either a real langchain ``LLMResult`` + or the ``_StringLLMResult`` stub used when langchain is not installed. + + Tests need to assert against the judge's response regardless of which + environment they run in — langchain-core comes in with the ``[ragas]`` + extra, so on CI with the extra installed we get real LLMResults, and + without it we get the duck-typed stub. + """ + if isinstance(result, str): + return result + return str(result.generations[0][0].text) + + class FakeJudge: def __init__(self, response: str = '{"ok": true}') -> None: self._response = response @@ -28,7 +43,7 @@ def test_wrapper_forwards_generate_text_to_judge(): judge = FakeJudge(response="faithful") llm = RagForgeRagasLLM(judge=judge, refusal_aware=False) result = llm.generate_text("What is the capital of France?") - assert result == "faithful" + assert _llm_result_text(result) == "faithful" assert judge.calls[0][1] == "What is the capital of France?" @@ -49,10 +64,175 @@ def test_wrapper_async_generate_text(): judge = FakeJudge(response="async-ok") llm = RagForgeRagasLLM(judge=judge, refusal_aware=False) result = asyncio.run(llm.agenerate_text("ping")) - assert result == "async-ok" + assert _llm_result_text(result) == "async-ok" assert judge.calls[0][1] == "ping" +def test_wrapper_async_generate_forwards_to_agenerate_text(): + """The v0.2.2 ``generate`` shim. This is the method Cycle 3 caught + missing — ragas's metric code calls ``await llm.generate(prompt)`` + and v0.2.1 crashed with AttributeError. Verifies the shim exists, + is async, and threads the prompt all the way through to the judge.""" + judge = FakeJudge(response="generate-ok") + llm = RagForgeRagasLLM(judge=judge, refusal_aware=False) + result = asyncio.run(llm.generate("what is DKA?")) + assert _llm_result_text(result) == "generate-ok" + assert judge.calls[0][1] == "what is DKA?" + + +def test_wrapper_async_generate_resolves_default_temperature_when_none(): + """BaseRagasLLM.generate resolves ``temperature=None`` via + ``get_temperature(n)`` before calling ``agenerate_text``. Our shim + must match that behaviour so ragas callers that pass ``None`` don't + blow up with a type error in downstream judge code. + + Asserts the fallback path was actually taken — not just that the + call didn't raise. CodeRabbit on PR #36 pointed out the original + weaker test would have passed even if ``get_temperature`` was + never called, so we patch it and assert the invocation. + """ + judge = FakeJudge(response="ok") + llm = RagForgeRagasLLM(judge=judge, refusal_aware=False) + with patch.object( + llm, "get_temperature", wraps=llm.get_temperature + ) as get_temp: + result = asyncio.run(llm.generate("prompt", n=1, temperature=None)) + assert _llm_result_text(result) == "ok" + get_temp.assert_called_once_with(1) + + +def test_wrapper_async_generate_keeps_explicit_temperature(): + """Inverse of the fallback test: when a concrete temperature is + passed, the shim must NOT call ``get_temperature``. Guards against + over-eager fallback firing on valid caller-supplied 0.0 / 0.01. + """ + judge = FakeJudge(response="ok") + llm = RagForgeRagasLLM(judge=judge, refusal_aware=False) + with patch.object(llm, "get_temperature") as get_temp: + result = asyncio.run(llm.generate("prompt", n=1, temperature=0.7)) + assert _llm_result_text(result) == "ok" + get_temp.assert_not_called() + + +def test_wrapper_async_generate_n_greater_than_one_produces_fused_shape(): + """When ``n > 1``, ``generate`` must return an ``LLMResult`` with + exactly N generations inside ``generations[0]`` — the shape ragas + expects for multi-sample metrics (e.g. answer_correctness). CodeRabbit + on PR #36 caught that the original shim ignored ``n`` and returned + a single generation. + + Uses a thread-safe counting judge because + ``RagForgeRagasLLM.generate`` fans the N calls out via + ``asyncio.gather`` and each call runs in a separate worker thread + (``asyncio.to_thread`` under the hood). A naive non-atomic + ``counter += 1`` would race and produce duplicate sample labels + or an undercounted total, flaking the test. CodeRabbit on PR #36 + round 2 caught this. + """ + import threading as _threading + + counter = {"i": 0} + lock = _threading.Lock() + + class CountingJudge: + def judge(self, system_prompt: str, user_prompt: str) -> str: + _ = system_prompt, user_prompt + with lock: + counter["i"] += 1 + return f"sample-{counter['i']}" + + def model_name(self) -> str: + return "counting-judge" + + llm = RagForgeRagasLLM(judge=CountingJudge(), refusal_aware=False) + result = asyncio.run(llm.generate("prompt", n=3, temperature=0.7)) + + assert counter["i"] == 3, f"expected 3 judge calls, got {counter['i']}" + + assert hasattr(result, "generations"), "result missing .generations" + outer = result.generations + assert len(outer) == 1, f"outer length must be 1 (one prompt), got {len(outer)}" + inner = outer[0] + assert len(inner) == 3, f"inner length must be 3 (n=3 samples), got {len(inner)}" + texts = {gen.text for gen in inner} + assert texts == {"sample-1", "sample-2", "sample-3"}, ( + f"expected three distinct samples, got {texts}" + ) + + +def test_fuse_llm_results_raises_on_malformed_input(): + """``_fuse_llm_results`` must fail loud on malformed result shapes + instead of silently collapsing to ``results[0]``. CodeRabbit on + PR #36 round 2 pointed out that a silent fallback hides real + ``n > 1`` correctness bugs — a single returned sample skews + downstream ragas metrics with no signal. + """ + from rag_forge_evaluator.engines.ragas_adapters import _fuse_llm_results + + class NotAnLLMResult: + pass + + with pytest.raises(ValueError, match="malformed result"): + _fuse_llm_results([NotAnLLMResult(), NotAnLLMResult()]) + + class EmptyOuter: + def __init__(self) -> None: + self.generations: list[list[object]] = [] + + with pytest.raises(ValueError, match="malformed result"): + _fuse_llm_results([EmptyOuter(), EmptyOuter()]) + + with pytest.raises(ValueError, match="empty results list"): + _fuse_llm_results([]) + + +def test_embeddings_embed_text_is_async_and_awaitable(): + """``BaseRagasEmbeddings.embed_text`` is async in ragas 0.4.x. Our + shim must match so ragas's ``await embeddings.embed_text(...)`` + calls work inside the evaluation runner's event loop. + + v0.2.2 originally shipped ``embed_text`` as sync with + ``asyncio.run(self.aembed_query(...))`` — which crashes any caller + already inside an event loop with + ``RuntimeError: asyncio.run() cannot be called from a running + event loop``. CodeRabbit on PR #36 caught it. + """ + import inspect as _inspect + + embed = RagForgeRagasEmbeddings(provider="mock") + assert _inspect.iscoroutinefunction(embed.embed_text), ( + "embed_text must be async (coroutine function) to match " + "BaseRagasEmbeddings.embed_text" + ) + assert _inspect.iscoroutinefunction(embed.embed_texts), ( + "embed_texts must be async (coroutine function)" + ) + # Round-trip through the async path. + v = asyncio.run(embed.embed_text("hello")) + assert isinstance(v, list) + assert len(v) == 8 + vs = asyncio.run(embed.embed_texts(["a", "b"])) + assert len(vs) == 2 + assert all(len(x) == 8 for x in vs) + + +def test_embeddings_embed_text_callable_from_running_event_loop(): + """Regression: the sync ``asyncio.run()`` implementation crashed + with ``RuntimeError: asyncio.run() cannot be called from a running + event loop``. This test exercises the exact code path by calling + ``embed_text`` from inside a running loop (the setting ragas + evaluation creates). It must complete without raising. + """ + embed = RagForgeRagasEmbeddings(provider="mock") + + async def run_inside_loop() -> list[float]: + return await embed.embed_text("hello") + + v = asyncio.run(run_inside_loop()) + assert isinstance(v, list) + assert len(v) == 8 + + def test_embeddings_mock_provider_returns_deterministic_vector(): embed = RagForgeRagasEmbeddings(provider="mock") v1 = embed.embed_query("hello") diff --git a/packages/evaluator/tests/test_ragas_adapters_contract.py b/packages/evaluator/tests/test_ragas_adapters_contract.py new file mode 100644 index 0000000..6f1d8f7 --- /dev/null +++ b/packages/evaluator/tests/test_ragas_adapters_contract.py @@ -0,0 +1,254 @@ +"""Contract tests that enforce our ragas adapter wrappers implement every +public method on ragas's ``BaseRagasLLM`` and ``BaseRagasEmbeddings``. + +**Why this file exists:** v0.2.1 shipped ``RagForgeRagasLLM`` without a +``generate`` method because the original module docstring claimed +"ragas only calls generate_text / agenerate_text / model_name". That was +wrong — ragas's concrete ``BaseRagasLLM.generate`` calls ``agenerate_text`` +through a retry wrapper, and ragas's metric code invokes it on every +LLM regardless of subclass status. The wrapper crashed with +``AttributeError: 'RagForgeRagasLLM' object has no attribute 'generate'`` +on every metric job during the PearMedica Cycle 3 audit (2026-04-15). + +This file is the tripwire. It iterates **every public method** on the +ragas base classes and asserts our duck-typed wrappers declare a callable +of the same name with a compatible arity. When ragas adds new methods in +a future release, these tests fail fast and we find out at CI time — not +at user-audit time. + +Requires the ``[ragas]`` extra. Skipped if ragas is not installed, so +base test runs on machines without the extra keep working. +""" + +from __future__ import annotations + +import inspect + +import pytest + +from rag_forge_evaluator.engines.ragas_adapters import ( + RagForgeRagasEmbeddings, + RagForgeRagasLLM, +) + + +class _StubJudge: + """Minimal Judge-protocol implementation for wrapper instantiation.""" + + def judge(self, system_prompt: str, user_prompt: str) -> str: + _ = system_prompt, user_prompt + return "stub" + + def model_name(self) -> str: + return "stub-judge" + + +def _public_methods(cls: type) -> list[str]: + """Return the names of every public (non-underscore) callable on ``cls``.""" + return sorted( + name + for name in dir(cls) + if not name.startswith("_") and callable(getattr(cls, name)) + ) + + +@pytest.fixture +def ragas_base_llm() -> type: + base = pytest.importorskip("ragas.llms.base") + return base.BaseRagasLLM + + +@pytest.fixture +def ragas_base_embeddings() -> type: + base = pytest.importorskip("ragas.embeddings.base") + return base.BaseRagasEmbeddings + + +def test_llm_wrapper_declares_every_base_class_public_method( + ragas_base_llm: type, +) -> None: + """Every public callable on ``BaseRagasLLM`` must exist on our wrapper. + + The actionable failure message lists the *missing* methods so a + future contributor sees exactly what ragas grew and what we still + owe it. + """ + base_methods = set(_public_methods(ragas_base_llm)) + wrapper_methods = set(_public_methods(RagForgeRagasLLM)) + missing = base_methods - wrapper_methods + assert not missing, ( + f"RagForgeRagasLLM is missing {len(missing)} method(s) that " + f"ragas.llms.base.BaseRagasLLM exposes: {sorted(missing)}. " + f"Add duck-typed shims in ragas_adapters.py — see the v0.2.2 " + f"release notes for precedent." + ) + + +def test_embeddings_wrapper_declares_every_base_class_public_method( + ragas_base_embeddings: type, +) -> None: + """Every public callable on ``BaseRagasEmbeddings`` must exist on our wrapper.""" + base_methods = set(_public_methods(ragas_base_embeddings)) + wrapper_methods = set(_public_methods(RagForgeRagasEmbeddings)) + missing = base_methods - wrapper_methods + assert not missing, ( + f"RagForgeRagasEmbeddings is missing {len(missing)} method(s) " + f"that ragas.embeddings.base.BaseRagasEmbeddings exposes: " + f"{sorted(missing)}. Add duck-typed shims in ragas_adapters.py." + ) + + +def test_llm_wrapper_async_signature_matches_base(ragas_base_llm: type) -> None: + """For every method that is ``async`` on ``BaseRagasLLM``, our + wrapper's method of the same name must also be ``async`` — and vice + versa. CodeRabbit on PR #36 pointed out that checking names alone + lets sync-vs-async mismatches through. A sync wrapper of an async + base method crashes at runtime the moment ragas invokes it with + ``await``; a sync base method called with ``await`` crashes the + other direction. + """ + for name in _public_methods(ragas_base_llm): + wrapper_attr = getattr(RagForgeRagasLLM, name, None) + if wrapper_attr is None: + continue # name-parity test above catches missing methods + base_is_async = inspect.iscoroutinefunction(getattr(ragas_base_llm, name)) + wrapper_is_async = inspect.iscoroutinefunction(wrapper_attr) + assert wrapper_is_async == base_is_async, ( + f"RagForgeRagasLLM.{name} is " + f"{'async' if wrapper_is_async else 'sync'} but " + f"BaseRagasLLM.{name} is " + f"{'async' if base_is_async else 'sync'}. Fix one to match." + ) + + +def test_embeddings_wrapper_async_signature_matches_base( + ragas_base_embeddings: type, +) -> None: + """Same async/sync parity check as the LLM wrapper, for embeddings. + + This is the test that would have caught v0.2.2's first attempt at + sync ``embed_text`` / ``embed_texts`` before they shipped — both + are ``async`` on ragas's ``BaseRagasEmbeddings`` in 0.4.x. + """ + for name in _public_methods(ragas_base_embeddings): + wrapper_attr = getattr(RagForgeRagasEmbeddings, name, None) + if wrapper_attr is None: + continue + base_is_async = inspect.iscoroutinefunction( + getattr(ragas_base_embeddings, name) + ) + wrapper_is_async = inspect.iscoroutinefunction(wrapper_attr) + assert wrapper_is_async == base_is_async, ( + f"RagForgeRagasEmbeddings.{name} is " + f"{'async' if wrapper_is_async else 'sync'} but " + f"BaseRagasEmbeddings.{name} is " + f"{'async' if base_is_async else 'sync'}. Fix one to match." + ) + + +def _required_param_names(cls: type, method_name: str) -> set[str]: + """Return the set of named parameters on ``cls.method_name``. + + Excludes ``self`` / ``cls`` and variadic ``*args`` / ``**kwargs`` + catchalls. Used by the parameter-parity tests below to compare + the named parameters a wrapper accepts against what the base + class declares. Extra parameters on the wrapper are fine; + missing parameters mean ragas will hit a ``TypeError`` the + moment it passes that kwarg. + """ + try: + sig = inspect.signature(getattr(cls, method_name)) + except (ValueError, TypeError): + return set() + return { + name + for name, param in sig.parameters.items() + if name not in ("self", "cls") + and param.kind + not in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ) + } + + +def test_llm_wrapper_parameter_names_cover_base_class( + ragas_base_llm: type, +) -> None: + """For every method present on both the base class and our wrapper, + the wrapper must accept at least every named parameter the base + class declares. Missing a name means ragas will hit a TypeError + the moment it tries to pass that kwarg. + + CodeRabbit on PR #38 round 2 pointed out that checking async/sync + parity alone lets parameter drift through. A future ragas release + that adds ``max_tokens`` to ``generate_text`` would silently break + our wrapper until a user audit caught it. + """ + for name in _public_methods(ragas_base_llm): + if getattr(RagForgeRagasLLM, name, None) is None: + continue + base_params = _required_param_names(ragas_base_llm, name) + wrapper_params = _required_param_names(RagForgeRagasLLM, name) + missing = base_params - wrapper_params + assert not missing, ( + f"RagForgeRagasLLM.{name} is missing parameters that " + f"BaseRagasLLM.{name} declares: {sorted(missing)}. Either " + f"add them to the shim signature or accept **kwargs so " + f"ragas's caller can still pass them." + ) + + +def test_embeddings_wrapper_parameter_names_cover_base_class( + ragas_base_embeddings: type, +) -> None: + """Same parameter-parity check for the embeddings wrapper.""" + for name in _public_methods(ragas_base_embeddings): + if getattr(RagForgeRagasEmbeddings, name, None) is None: + continue + base_params = _required_param_names(ragas_base_embeddings, name) + wrapper_params = _required_param_names(RagForgeRagasEmbeddings, name) + missing = base_params - wrapper_params + assert not missing, ( + f"RagForgeRagasEmbeddings.{name} is missing parameters " + f"that BaseRagasEmbeddings.{name} declares: {sorted(missing)}." + ) + + +def test_llm_wrapper_generate_is_async_and_callable_on_instance() -> None: + """The specific shim that Cycle 3 caught missing — invoke it.""" + llm = RagForgeRagasLLM(judge=_StubJudge(), refusal_aware=False) + generate = getattr(llm, "generate", None) + assert generate is not None, "generate shim not declared on instance" + assert inspect.iscoroutinefunction(generate), ( + "generate must be async — ragas's BaseRagasLLM.generate is async " + "and is invoked via `await` in metric code" + ) + + +def test_llm_wrapper_is_finished_returns_true_by_default() -> None: + """``is_finished`` is abstract on the base class — our shim exists + and reports True so ragas's retry path doesn't trigger on every call.""" + llm = RagForgeRagasLLM(judge=_StubJudge(), refusal_aware=False) + assert llm.is_finished(response="ignored") is True + + +def test_llm_wrapper_get_temperature_matches_base_convention() -> None: + """n<=1 → deterministic, n>1 → warmer. Matches BaseRagasLLM.""" + llm = RagForgeRagasLLM(judge=_StubJudge(), refusal_aware=False) + assert llm.get_temperature(1) == 0.01 + assert llm.get_temperature(5) == 0.3 + + +def test_llm_wrapper_set_run_config_stores_the_value() -> None: + llm = RagForgeRagasLLM(judge=_StubJudge(), refusal_aware=False) + sentinel = object() + llm.set_run_config(sentinel) + assert llm.run_config is sentinel + + +def test_embeddings_wrapper_set_run_config_stores_the_value() -> None: + emb = RagForgeRagasEmbeddings(provider="mock") + sentinel = object() + emb.set_run_config(sentinel) + assert emb.run_config is sentinel diff --git a/packages/evaluator/tests/test_ragas_adapters_e2e.py b/packages/evaluator/tests/test_ragas_adapters_e2e.py new file mode 100644 index 0000000..4c9287f --- /dev/null +++ b/packages/evaluator/tests/test_ragas_adapters_e2e.py @@ -0,0 +1,164 @@ +"""End-to-end smoke test: run ``ragas.evaluate()`` against our wrappers. + +This is the test that would have caught the v0.2.1 C3-2 regression before +publish. It invokes ragas's own evaluation entrypoint on a tiny two-sample +dataset, passing ``RagForgeRagasLLM`` and ``RagForgeRagasEmbeddings`` as +the ``llm`` and ``embeddings`` arguments — the exact call path that +crashed with ``AttributeError: no attribute 'generate'`` on every metric +job during the PearMedica Cycle 3 audit. + +If this test passes, the wrappers implement enough of the ragas contract +for a real ``evaluate()`` call to complete. If it fails, something in +ragas's async job loop reached into an undeclared method — repeat the +contract-test audit, add the missing shim, re-run. + +Skipped gracefully if the ``[ragas]`` extra is not installed. +""" + +from __future__ import annotations + +import os +from typing import Any + +import pytest + + +class _DeterministicJudge: + """Judge stub that returns canned JSON responses ragas can parse. + + ragas's faithfulness metric goes through two stages: + 1. Ask the LLM to extract a list of "statements" from the answer. + 2. For each statement, ask the LLM to verdict it against the context. + + A smoke-test judge just needs to return valid-looking JSON in both + stages — we never try to measure actual quality here, only to prove + the wrapper's dispatch and return shape are compatible with ragas's + internal plumbing. + """ + + def __init__(self) -> None: + self.call_count = 0 + + def judge(self, system_prompt: str, user_prompt: str) -> str: + _ = system_prompt + self.call_count += 1 + text = user_prompt.lower() + # Statement-extraction stage + if "statement" in text and "extract" in text: + return '{"statements": ["Paris is the capital of France."]}' + # Per-statement verdict stage + if "verdict" in text or "faithful" in text: + return ( + '{"statements": [{"statement": "Paris is the capital of France.", ' + '"verdict": 1, "reason": "directly stated in context"}]}' + ) + # Fallback — some metrics emit a generic prompt + return '{"answer": "Paris", "score": 1}' + + def model_name(self) -> str: + return "deterministic-judge-stub" + + +def _make_dataset() -> Any: + datasets = pytest.importorskip("datasets") + return datasets.Dataset.from_dict( + { + "question": ["What is the capital of France?"], + "answer": ["Paris is the capital of France."], + "contexts": [["Paris is the capital city of France."]], + "ground_truth": ["Paris"], + } + ) + + +def test_ragas_evaluate_never_raises_attribute_error_on_wrapper() -> None: + """Invoke ragas.evaluate() with our wrappers on a 1-sample dataset. + + **What this test is proving, and what it is NOT proving.** + + Proving: ragas's async job loop can reach our ``RagForgeRagasLLM`` + and ``RagForgeRagasEmbeddings`` instances and call every method + they need without hitting an ``AttributeError``. This is the exact + failure mode the PearMedica Cycle 3 audit caught in v0.2.1: + ``AttributeError: 'RagForgeRagasLLM' object has no attribute 'generate'`` + — every metric job crashed identically before any real evaluation + logic ran. + + NOT proving: that the metric produces a meaningful score, or that + the output parser accepts our stub judge's return values. A stub + judge returning canned JSON cannot match every pydantic schema + ragas uses internally across versions — fighting that is a losing + game for a smoke test. If the call fails with + ``RagasOutputParserException`` or similar parser errors, the + wrapper dispatch worked and ragas is doing its own thing downstream. + We assert on exception **type** rather than demanding a clean + return. + + Run against real production telemetry (not a stub) when verifying + end-to-end evaluation quality — see the Cycle 3 RAGAS re-run + planned for Workstream G2. + """ + pytest.importorskip("ragas") + pytest.importorskip("langchain_core") + pytest.importorskip("datasets") + + from ragas import evaluate + from ragas.metrics import faithfulness + + from rag_forge_evaluator.engines.ragas_adapters import ( + RagForgeRagasEmbeddings, + RagForgeRagasLLM, + ) + + os.environ.setdefault("OPENAI_API_KEY", "sk-test-nonce") + + llm = RagForgeRagasLLM(judge=_DeterministicJudge(), refusal_aware=False) + embeddings = RagForgeRagasEmbeddings(provider="mock") + dataset = _make_dataset() + + # What we expect downstream of our wrappers when everything is + # shaped correctly: ragas's faithfulness metric reaches its + # pydantic output parser and fails to parse our stub judge's + # canned JSON. That raises a ragas-internal exception (either + # ``RagasOutputParserException`` or a ``RagasException`` + # subclass). **Any other exception shape** — TypeError from a bad + # shim signature, RuntimeError from a nested-loop bug, ValueError + # from embedding dimension mismatch — means something our wrapper + # is responsible for went wrong, and the test must fail. + # + # CodeRabbit on PR #36 pointed out the original broad + # ``except Exception`` swallowed real bugs as long as they + # weren't AttributeError. This tightening makes the regression + # guard meaningful. + allowed_exception_types: tuple[type[BaseException], ...] + try: + from ragas.exceptions import RagasOutputParserException # type: ignore[attr-defined] + + allowed_exception_types = (RagasOutputParserException,) + except ImportError: + allowed_exception_types = () + + try: + evaluate( + dataset=dataset, + metrics=[faithfulness], + llm=llm, + embeddings=embeddings, + raise_exceptions=True, + show_progress=False, + ) + except AttributeError as exc: + wrapper_names = ("RagForgeRagasLLM", "RagForgeRagasEmbeddings") + if any(name in str(exc) for name in wrapper_names): + pytest.fail( + f"ragas called a method on our wrapper that doesn't " + f"exist: {exc!r}. Add the missing shim in " + f"ragas_adapters.py and update the contract test." + ) + raise + except allowed_exception_types: + # Parser / schema failure downstream of our wrappers. The + # wrapper dispatch worked; ragas is now doing its own thing + # with a stub judge that returns JSON not matching its + # internal pydantic schemas. Treat as success. + pass