fix: report missing AI packages and failed model requests as evaluation errors in prompt()

README.md

@@ -75,7 +75,7 @@ _Data validation toolkit for assessing and monitoring data quality._
 
 [![Python Versions](https://img.shields.io/pypi/pyversions/pointblank.svg)](https://pypi.python.org/pypi/pointblank)
 [![PyPI](https://img.shields.io/pypi/v/pointblank)](https://pypi.org/project/pointblank/#history)
-[![PyPI Downloads](https://static.pepy.tech/badge/pointblank)](https://pepy.tech/projects/pointblank)
+[![PyPI Downloads](https://img.shields.io/pypi/dm/pointblank)](https://pypistats.org/packages/pointblank)
 [![Conda Version](https://img.shields.io/conda/vn/conda-forge/pointblank.svg)](https://anaconda.org/conda-forge/pointblank)
 [![License](https://img.shields.io/github/license/posit-dev/pointblank)](https://img.shields.io/github/license/posit-dev/pointblank)
 

pointblank/_interrogation.py

@@ -2478,54 +2478,16 @@ def interrogate_prompt(  # pragma: no cover
 
         return result_tbl
 
-    except ImportError as e:
-        logger.error(f"Missing dependencies for AI validation: {e}")
-        logger.error("Install required packages: pip install openai anthropic aiohttp")
-
-        # Return all False results as fallback (nw_tbl and total_rows defined at function start)
-        native_tbl = nw_tbl.to_native()
-        validation_results = [False] * total_rows
-
-        if hasattr(native_tbl, "with_columns"):  # Polars
-            import polars as pl
-
-            result_tbl = native_tbl.with_columns(pb_is_good_=pl.Series(validation_results))
-
-        elif hasattr(native_tbl, "assign"):  # Pandas
-            import pandas as pd
-
-            result_tbl = native_tbl.assign(pb_is_good_=pd.Series(validation_results, dtype=bool))
-
-        else:
-            # Fallback
-            result_tbl = native_tbl.copy() if hasattr(native_tbl, "copy") else native_tbl
-            result_tbl["pb_is_good_"] = validation_results
+    except StepEvaluationError:
+        raise
 
-        return result_tbl
+    except ImportError as e:
+        # A missing package (e.g., the provider's SDK); the message says how to install it
+        raise StepEvaluationError(f"The prompt() step couldn't be run: {e}") from e
 
     except Exception as e:
-        logger.error(f"AI validation failed: {e}")
-
-        # Return all False results as fallback (nw_tbl and total_rows defined at function start)
-        native_tbl = nw_tbl.to_native()
-        validation_results = [False] * total_rows
-
-        if hasattr(native_tbl, "with_columns"):  # Polars
-            import polars as pl
-
-            result_tbl = native_tbl.with_columns(pb_is_good_=pl.Series(validation_results))
-
-        elif hasattr(native_tbl, "assign"):  # Pandas
-            import pandas as pd
-
-            result_tbl = native_tbl.assign(pb_is_good_=pd.Series(validation_results, dtype=bool))
-
-        else:
-            # Fallback
-            result_tbl = native_tbl.copy() if hasattr(native_tbl, "copy") else native_tbl
-            result_tbl["pb_is_good_"] = validation_results
-
-        return result_tbl
+        # Anything else that stops the step as a whole (e.g., an invalid model configuration)
+        raise StepEvaluationError(f"The prompt() step couldn't be run ({_first_line(e)}).") from e
 
 
 def data_freshness(

pointblank/_utils_ai.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
 import hashlib
+import importlib
 import json
 import logging
 import pathlib
+import re
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -22,6 +24,61 @@
 # LLM Configuration and Chat Interface
 # ============================================================================
 
+# All of the AI features' dependencies are in the `generate` extra
+_AI_INSTALL_HINT = '`pip install "pointblank[generate]"`'
+
+# The packages each provider needs, besides chatlas (Bedrock goes through `anthropic[bedrock]`)
+_PROVIDER_PACKAGES = {
+    "anthropic": ["anthropic"],
+    "openai": ["openai"],
+    "ollama": ["openai"],
+    "bedrock": ["anthropic", "boto3"],
+    "azure-openai": ["openai"],
+}
+
+
+def _check_ai_package(package: str, needed_for: str) -> None:
+    """Raise an `ImportError` with an install hint if a package needed for an AI feature is missing."""
+    try:
+        importlib.import_module(package)
+    except ImportError as e:
+        raise ImportError(
+            f"The `{package}` package is required {needed_for}. Install Pointblank's AI "
+            f"dependencies with {_AI_INSTALL_HINT}."
+        ) from e
+
+
+def _check_provider_packages(provider: str) -> None:
+    """Check that the packages a model provider needs are installed."""
+    for package in _PROVIDER_PACKAGES.get(provider, []):
+        _check_ai_package(package, f"to use the `{provider}` provider")
+
+
+def _http_library(sdk: str) -> str:
+    """
+    Get the HTTP library a provider SDK is built on: `httpx2` or `httpx`.
+
+    `anthropic` (from 1.0) and `openai` (from 3.0) moved from `httpx` to `httpx2`, and they
+    reject (or fail later with) a client from the other library. Their package metadata says
+    which one they need.
+    """
+    from importlib.metadata import PackageNotFoundError, requires
+
+    try:
+        requirements = requires(sdk) or []
+    except PackageNotFoundError:  # pragma: no cover
+        requirements = []
+
+    names = {re.split(r"[\s;<>=!~\[]", r, maxsplit=1)[0].lower() for r in requirements}
+    return "httpx2" if "httpx2" in names else "httpx"
+
+
+def _async_http_client(sdk: str, verify_ssl: bool) -> Any:
+    """Create an async HTTP client (with the SSL setting) of the kind that a provider SDK accepts."""
+    library = _http_library(sdk)
+    _check_ai_package(library, "for SSL configuration")
+    return importlib.import_module(library).AsyncClient(verify=verify_ssl)
+
 
 @dataclass
 class _LLMConfig:
@@ -94,14 +151,7 @@ def _create_chat_instance(
     -------
     Chat instance from chatlas.
     """
-    # Check if chatlas is installed
-    try:
-        import chatlas  # noqa
-    except ImportError:  # pragma: no cover
-        raise ImportError(
-            "The `chatlas` package is required for AI validation. "
-            "Please install it using `pip install chatlas`."
-        )
+    _check_ai_package("chatlas", "for AI validation")
 
     # Validate provider
     if provider not in MODEL_PROVIDERS:
@@ -110,79 +160,43 @@ def _create_chat_instance(
             f"Supported providers: {', '.join(MODEL_PROVIDERS)}"
         )
 
+    _check_provider_packages(provider)
+
     # Use the row-validation system prompt unless a caller supplies its own
     if system_prompt is None:
         system_prompt = _ROW_VALIDATION_SYSTEM_PROMPT
 
-    # Create httpx client with SSL verification settings
-    try:
-        import httpx  # noqa
-    except ImportError:  # pragma: no cover
-        raise ImportError(  # pragma: no cover
-            "The `httpx` package is required for SSL configuration. "
-            "Please install it using `pip install httpx`."
-        )
-
-    http_client = httpx.AsyncClient(verify=verify_ssl)
-
     # Create provider-specific chat instance
-    if provider == "anthropic":  # pragma: no cover
-        # Check that the anthropic package is installed
-        try:
-            import anthropic  # noqa  # type: ignore[import-not-found]
-        except ImportError:
-            raise ImportError(
-                "The `anthropic` package is required to use AI validation with "
-                "`anthropic`. Please install it using `pip install anthropic`."
-            )
-
+    if provider == "anthropic":
         from chatlas import ChatAnthropic
 
         chat = ChatAnthropic(
             model=model_name,
             api_key=api_key,
             system_prompt=system_prompt,
-            kwargs={"http_client": http_client},
+            kwargs={"http_client": _async_http_client("anthropic", verify_ssl)},
         )
 
     elif provider == "openai":  # pragma: no cover
-        # Check that the openai package is installed
-        try:
-            import openai  # noqa
-        except ImportError:
-            raise ImportError(
-                "The `openai` package is required to use AI validation with "
-                "`openai`. Please install it using `pip install openai`."
-            )
-
         from chatlas import ChatOpenAI
 
         chat = ChatOpenAI(
             model=model_name,
             api_key=api_key,
             system_prompt=system_prompt,
-            kwargs={"http_client": http_client},
+            kwargs={"http_client": _async_http_client("openai", verify_ssl)},
         )
 
     elif provider == "ollama":  # pragma: no cover
-        # Check that the openai package is installed (required for Ollama)
-        try:
-            import openai  # noqa
-        except ImportError:
-            raise ImportError(
-                "The `openai` package is required to use AI validation with "
-                "`ollama`. Please install it using `pip install openai`."
-            )
-
         from chatlas import ChatOllama
 
         chat = ChatOllama(
             model=model_name,
             system_prompt=system_prompt,
-            kwargs={"http_client": http_client},
+            kwargs={"http_client": _async_http_client("openai", verify_ssl)},
         )
 
-    elif provider == "bedrock":  # pragma: no cover
+    elif provider == "bedrock":
         from chatlas import ChatBedrockAnthropic
 
         # ChatBedrockAnthropic creates both a sync and async client internally,
@@ -194,14 +208,6 @@ def _create_chat_instance(
         )
 
     elif provider == "azure-openai":  # pragma: no cover
-        try:
-            import openai  # noqa
-        except ImportError:
-            raise ImportError(
-                "The `openai` package is required to use AI validation with "
-                "`azure-openai`. Please install it using `pip install openai`."
-            )
-
         import os
 
         endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
@@ -225,7 +231,7 @@ def _create_chat_instance(
             api_version=api_version,
             api_key=api_key,
             system_prompt=system_prompt,
-            kwargs={"http_client": http_client},
+            kwargs={"http_client": _async_http_client("openai", verify_ssl)},
         )
 
     else:
@@ -252,18 +258,13 @@ def _prepare_attachments(attachments: Optional[List[Any]]) -> List[Any]:
     if not isinstance(attachments, (list, tuple)):
         raise TypeError(f"attachments must be a list or tuple, got {type(attachments).__name__}")
 
-    try:
-        from chatlas import (
-            content_image_file,
-            content_image_url,
-            content_pdf_file,
-            content_pdf_url,
-        )
-    except ImportError:  # pragma: no cover
-        raise ImportError(
-            "The `chatlas` package is required for attachments support. "
-            "Please install it using `pip install chatlas`."
-        )
+    _check_ai_package("chatlas", "for attachments support")
+    from chatlas import (
+        content_image_file,
+        content_image_url,
+        content_pdf_file,
+        content_pdf_url,
+    )
 
     prepared: List[Any] = []
     for item in attachments:
@@ -945,22 +946,32 @@ def validate_batch(batch: Dict[str, Any]) -> List[Dict[str, Any]]:
                 logger.info(f"Successfully validated batch {batch['batch_id']}")
                 return results
 
-            except Exception as e:  # pragma: no cover
-                logger.error(
-                    f"Failed to validate batch {batch['batch_id']}: {e}"
-                )  # pragma: no cover
+            except Exception as e:
+                logger.error(f"Failed to validate batch {batch['batch_id']}: {e}")
+                errors.append(e)
                 # Return default results (all False) for failed batches
-                default_results = []  # pragma: no cover
-                for i in range(batch["start_row"], batch["end_row"]):  # pragma: no cover
-                    default_results.append({"index": i, "result": False})  # pragma: no cover
-                return default_results  # pragma: no cover
+                default_results = []
+                for i in range(batch["start_row"], batch["end_row"]):
+                    default_results.append({"index": i, "result": False})
+                return default_results
 
         # Execute all batch validations sequentially (chatlas is synchronous)
+        errors: List[Exception] = []
         final_results = []
         for batch in batches:
             result = validate_batch(batch)
             final_results.append(result)
 
+        # If no batch could be validated (e.g., an invalid API key), every row would be reported as
+        # failing, so treat the step as one that couldn't be evaluated instead
+        if batches and len(errors) == len(batches):
+            from pointblank._interrogation import StepEvaluationError, _first_line
+
+            raise StepEvaluationError(
+                f"None of the {len(batches)} requests to the model succeeded "
+                f"(the first failed with {_first_line(errors[0])})."
+            ) from errors[0]
+
         return final_results
 
     def validate_single_batch(