fix(benchmark): treat a pcap engine's loader OSError as unmeasured, not fatal (#2129)

examples/benchmark/README.md

@@ -447,7 +447,9 @@ back to `pcapkit`'s parser refuses to report a number.
 - A row you expected is `*not measured*` -- read the reason. It comes from the
   engine's own `unsupported_reason()`, which names the actual cause rather than
   guessing at one -- except for `pypcap` on 3.12+, where the interpreter ceiling is
-  reported instead.
+  reported instead. A `pcap` module that is installed but will not load, such as
+  `pcap-ct`'s libpcap loader finding no library on macOS or Windows, is reported
+  with the loader's own error for both `pypcap` and `pcap_ct`.
 - A whole column is `--` -- read the note under the table, and
   `out/missing/<version>.txt`. The version was attempted; the build or the run
   failed, and the reason is recorded rather than the column dropped.

examples/benchmark/benchmark.py

@@ -269,6 +269,45 @@ def _declared_reason(engine: 'str') -> 'Optional[str]':
         return f'{type(exc).__name__}: {exc}'
 
 
+def _loader_failure(engine: 'str') -> 'Optional[str]':
+    """Why a ``pcap``-module engine cannot load its ``pcap`` module here (#2129).
+
+    ``pypcap`` and ``pcap_ct`` both import the ``pcap`` module, and ``pcap-ct``'s
+    import runs a :manpage:`libpcap(3)` loader that raises :exc:`OSError` when the
+    library it looks for is absent -- ``dlopen(libc.so.6 ...)`` on macOS,
+    ``Cannot find npcap/wpcap.dll`` on Windows. ``pcap_ct`` declares that through
+    its own ``unsupported_reason()``; ``pypcap`` does not, so the :exc:`OSError`
+    escapes the extraction and would end the whole environment.
+
+    Asked of the import itself, through the same probe the engines use, rather than
+    by matching the wording of an exception from the extraction. That keeps the
+    scope to the import: an :exc:`OSError` from opening or reading the capture is
+    still a real failure, and is re-raised.
+
+    Args:
+        engine: Engine name, as passed to ``pcapkit.extract``.
+
+    Returns:
+        The loader's error, or :data:`None` for an engine that does not import the
+        ``pcap`` module, when the module imports, or when it is simply not
+        installed -- which the extraction already reports.
+
+    """
+    from pcapkit.foundation.engines import _pcap_backend  # pylint: disable=import-outside-toplevel
+    from pcapkit.foundation.extraction import Extractor  # pylint: disable=import-outside-toplevel
+
+    registered = Extractor.__engine__.get(engine)
+    klass = getattr(registered, 'klass', registered)
+    if getattr(klass, '__engine_distribution__', None) not in _pcap_backend.DISTRIBUTIONS:
+        return None
+    # ``probe`` purges a half-imported ``pcap`` after a failure, so the next engine
+    # to ask sees the same loader error rather than the residue of this attempt.
+    found = _pcap_backend.probe()
+    if found.name is None and not found.missing:
+        return f'the `pcap` module is installed but unusable -- {found.failure}'
+    return None
+
+
 def _extract(engine: 'str', capture: 'str') -> 'Extractor':
     """One extraction, with the arguments the legacy timing script used.
 
@@ -292,8 +331,8 @@ def _extract(engine: 'str', capture: 'str') -> 'Extractor':
 def _unavailable(exc: 'Exception') -> 'Optional[str]':
     """Explain *exc* if it means the engine cannot run here.
 
-    A last resort behind :func:`_declared_reason`, for the failures no engine
-    declares in advance.
+    A last resort behind :func:`_declared_reason` and :func:`_loader_failure`, for
+    the failures no engine declares in advance.
 
     Args:
         exc: Exception the engine raised.
@@ -314,10 +353,9 @@ def _unavailable(exc: 'Exception') -> 'Optional[str]':
         version = '.'.join(str(part) for part in sys.version_info[:3])
         return (f'{exc} -- pyshark asks for an implicit asyncio event loop, which '
                 f'Python {version} no longer provides')
-    # ``pcap-ct`` with no system libpcap raises OSError from its loader, which is
-    # not an ImportError and so reaches here.
-    if isinstance(exc, OSError) and 'libpcap' in str(exc):
-        return f'{exc} -- no system libpcap for the `pcap` module to load'
+    # No OSError is matched here by its wording: a ``pcap`` module that will not load
+    # is answered before the attempt by :func:`_loader_failure` (#2129), and any other
+    # OSError -- reading the capture, say -- is a real failure.
     return None
 
 
@@ -406,10 +444,11 @@ def preflight(engine: 'str', capture: 'str') -> 'tuple[Optional[str], Optional[s
 
     These sources are consulted, in this order, and the order is the point: what the
     image recorded about not installing the engine at all (:func:`_not_attempted`),
-    then the engine's own ``unsupported_reason()``, then an actual attempt. The first
-    wins outright where it exists, since an engine the interpreter rules out cannot
-    describe its own ceiling -- it can only report what it finds in the environment
-    that ceiling produced.
+    then the engine's own ``unsupported_reason()``, then whether a ``pcap``-module
+    engine's module loads at all (:func:`_loader_failure`), then an actual attempt.
+    The first wins outright where it exists, since an engine the interpreter rules
+    out cannot describe its own ceiling -- it can only report what it finds in the
+    environment that ceiling produced.
 
     Args:
         engine: Engine name to try.
@@ -448,6 +487,8 @@ def _reason(text: 'str') -> 'str':
         return f'{text} -- {build_failure}' if build_failure else text
 
     reason = _declared_reason(engine)
+    if reason is None:
+        reason = _loader_failure(engine)
     if reason is not None:
         return _reason(reason), None
 

examples/benchmark/test_harness.py

@@ -1522,6 +1522,137 @@ def test_prose_is_rejoined_as_prose_and_log_output_is_not(self, monkeypatch, tmp
             'one sentence wrapped | across two lines.'
 
 
+#: The loader errors ``pcap-ct`` raised on the runners of #2129, abridged.
+LOADER_ERRORS = {
+    'macos': "dlopen(libc.so.6, 0x0006): tried: 'libc.so.6' (no such file), "
+             "'/usr/lib/libc.so.6' (no such file, not in dyld cache)",
+    'windows': 'Cannot find npcap/wpcap.dll or wpcap.dll library',
+}
+
+
+def fake_pcap(root, body):
+    """Install a stand-in ``pcap`` package under *root* whose ``__init__`` is *body*.
+
+    Args:
+        root: Directory to put on :data:`sys.path`.
+        body: Source of ``pcap/__init__.py``.
+
+    Returns:
+        *root*, as a string.
+
+    """
+    package = root / 'pcap'
+    package.mkdir(parents=True)
+    (package / '__init__.py').write_text(body, encoding='utf-8')
+    # pcap-ct ships ``pcap`` as a package that binds a ``_pcap`` submodule, which is
+    # how pcapkit tells it from upstream pypcap; a *body* that imports cleanly has to
+    # bind it too.
+    (package / '_pcap.py').write_text('', encoding='utf-8')
+    return str(root)
+
+
+def tiny_pcap(path):
+    """Write a little-endian PCAP holding one zero-padded Ethernet frame.
+
+    Args:
+        path: Where to write it.
+
+    Returns:
+        *path*, as a string.
+
+    """
+    import struct  # pylint: disable=import-outside-toplevel
+
+    frame = b'\xff' * 6 + b'\x00\x11\x22\x33\x44\x55' + b'\x08\x06' + b'\x00' * 46
+    header = struct.pack('<IHHiIII', 0xa1b2c3d4, 2, 4, 0, 0, 65535, 1)
+    record = struct.pack('<IIII', 0, 0, len(frame), len(frame))
+    path.write_bytes(header + record + frame)
+    return str(path)
+
+
+class TestLoaderFailure:
+    """A ``pcap`` module that will not load is a reason, not a fatal error (#2129).
+
+    On macOS and Windows, ``pcap-ct``'s libpcap loader raises :exc:`OSError` with a
+    message that does not mention libpcap. ``pypcap``'s preflight let it escape, and
+    the whole ``pcap_ct`` environment exited before measuring anything.
+
+    """
+
+    @pytest.fixture
+    def harness(self, monkeypatch, tmp_path):
+        """:mod:`benchmark`, with no image notes and no ``pcap`` module cached."""
+        benchmark = pytest.importorskip('benchmark')
+        monkeypatch.setattr(benchmark, 'INSTALL_FAILURES', str(tmp_path / 'absent'))
+        monkeypatch.setattr(benchmark, 'NOT_ATTEMPTED', str(tmp_path / 'absent'))
+        for name in [name for name in sys.modules
+                     if name == 'pcap' or name.startswith('pcap.')]:
+            monkeypatch.delitem(sys.modules, name)
+        yield benchmark
+        for name in [name for name in sys.modules
+                     if name == 'pcap' or name.startswith('pcap.')]:
+            del sys.modules[name]
+
+    @pytest.mark.parametrize('runner', sorted(LOADER_ERRORS))
+    def test_the_loader_error_is_the_reason_and_the_run_carries_on(
+            self, harness, monkeypatch, tmp_path, runner):
+        """Both ``pcap`` engines are unmeasured with the loader's message; ``default`` runs."""
+        message = LOADER_ERRORS[runner]
+        monkeypatch.syspath_prepend(fake_pcap(tmp_path / 'site', f'raise OSError({message!r})\n'))
+
+        document = harness.run(tiny_pcap(tmp_path / 'one.pcap'), ('default', 'pypcap', 'pcap_ct'),
+                               rounds=2, repeats=1, label='3.10-pcap_ct')
+
+        results = {entry['engine']: entry for entry in document['results']}
+        assert results['default']['status'] == 'measured'
+        for engine in ('pypcap', 'pcap_ct'):
+            assert results[engine]['status'] == 'unmeasured'
+            assert message in results[engine]['reason']
+            assert 'not installed' not in results[engine]['reason']
+
+    @pytest.mark.parametrize('runner', sorted(LOADER_ERRORS))
+    def test_preflight_names_the_loader_error_for_pypcap(self, harness, monkeypatch, tmp_path,
+                                                         runner):
+        """``pypcap`` declares nothing itself, so the harness has to ask the import."""
+        message = LOADER_ERRORS[runner]
+        monkeypatch.syspath_prepend(fake_pcap(tmp_path / 'site', f'raise OSError({message!r})\n'))
+
+        reason, driver = harness.preflight('pypcap', str(tmp_path / 'unused.pcap'))
+        assert reason == f'the `pcap` module is installed but unusable -- OSError: {message}'
+        assert driver is None
+
+    def test_an_oserror_reading_the_capture_is_still_fatal(self, harness, monkeypatch,
+                                                           tmp_path):
+        """Only the import is in scope: a ``pcap`` module that loads, then fails, re-raises.
+
+        The message mentions libpcap on purpose. Matching on that word is what the
+        harness did before, and it would have swallowed this real failure.
+
+        """
+        monkeypatch.syspath_prepend(fake_pcap(tmp_path / 'site',
+                                              "from . import _pcap\n__version__ = '1.3.0b3'\n"))
+
+        def _unreadable(engine, capture):
+            """Fail the way libpcap does on a damaged savefile."""
+            raise OSError('libpcap: truncated dump file; tried to read 16 header bytes')
+
+        monkeypatch.setattr(harness, '_extract', _unreadable)
+        with pytest.raises(OSError, match='truncated dump file'):
+            harness.preflight('pcap_ct', str(tmp_path / 'unused.pcap'))
+
+    def test_an_engine_without_a_pcap_module_is_not_probed(self, harness, monkeypatch):
+        """``dpkt`` and the rest never import ``pcap``, so their import is not asked."""
+        from pcapkit.foundation.engines import _pcap_backend  # pylint: disable=import-outside-toplevel
+
+        def _no_probe():
+            """Fail the test if the probe is reached."""
+            raise AssertionError('probed the pcap module for an engine that never imports it')
+
+        monkeypatch.setattr(_pcap_backend, 'probe', _no_probe)
+        for engine in ('default', 'dpkt', 'scapy', 'pyshark', 'pypcapfile'):
+            assert harness._loader_failure(engine) is None  # pylint: disable=protected-access
+
+
 class TestRoundTrip:
     """The JSON contract between the two halves of the harness."""