diff --git a/docs/source/pcapkit/foundation/engines/3rdparty.rst b/docs/source/pcapkit/foundation/engines/3rdparty.rst index 982ba14753..2efcc71e6a 100644 --- a/docs/source/pcapkit/foundation/engines/3rdparty.rst +++ b/docs/source/pcapkit/foundation/engines/3rdparty.rst @@ -69,8 +69,9 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. On PCAP-NG, `DPKT`_ yields **Enhanced Packet Blocks and (obsolete) Packet Blocks only**. ``dpkt.pcapng.Reader.__iter__`` skips every other block type without a warning (``# just ignore other blocks``, as of ``dpkt`` 1.9.8), and - that includes the **Simple Packet Block**, which carries a packet just as an - Enhanced Packet Block does. Such packets are silently missing from this + :class:`~pcapkit.foundation.engines.dpkt.PCAPNGReader`, which this engine + reads PCAP-NG with, keeps that block set. It includes the **Simple Packet + Block**, which carries a packet just as an Enhanced Packet Block does. Such packets are silently missing from this engine's output, and the frames after them are numbered as if they had never been captured. @@ -83,6 +84,15 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. `DPKT`_, not of :mod:`pcapkit`, so read a capture that may contain Simple Packet Blocks with the default engine instead. +.. note:: + + ``dpkt.pcapng.Reader`` reads only the **first** Interface Description Block + of a file, and applies its link type and timestamp resolution to every + packet. This engine therefore reads PCAP-NG with + :class:`~pcapkit.foundation.engines.dpkt.PCAPNGReader` instead, which + resolves each packet's link type, ``if_tsresol`` and ``if_tsoffset`` from + the interface it names, in its own section (:issue:`1379`). + .. autoclass:: pcapkit.foundation.engines.dpkt.DPKT :no-members: :show-inheritance: @@ -96,6 +106,10 @@ support, as is used by :class:`pcapkit.foundation.extraction.Extractor`. .. autoattribute:: _expkg .. autoattribute:: _extmp +.. autoclass:: pcapkit.foundation.engines.dpkt.PCAPNGReader + :members: datalink + :show-inheritance: + PyShark Support =============== diff --git a/pcapkit/foundation/engines/dpkt.py b/pcapkit/foundation/engines/dpkt.py index 39a545b9b3..d9d79e8005 100644 --- a/pcapkit/foundation/engines/dpkt.py +++ b/pcapkit/foundation/engines/dpkt.py @@ -10,36 +10,175 @@ .. _DPKT: https://dpkt.readthedocs.io """ +import decimal from typing import TYPE_CHECKING, cast +from pcapkit.const.pcapng.block_type import BlockType as Enum_BlockType +from pcapkit.const.pcapng.option_type import OptionType as Enum_OptionType from pcapkit.const.reg.linktype import LinkType as Enum_LinkType from pcapkit.foundation.engines.engine import EngineBase +from pcapkit.utilities.compat import localcontext from pcapkit.utilities.exceptions import FormatError, stacklevel from pcapkit.utilities.logging import get_logger from pcapkit.utilities.warnings import AttributeWarning, DPKTWarning, warn -__all__ = ['DPKT'] +__all__ = ['DPKT', 'PCAPNGReader'] if TYPE_CHECKING: - from typing import Optional, Type, Union + from typing import IO, Iterator, Literal, Optional, Type, Union from dpkt.dpkt import Packet as DPKTPacket from dpkt.pcap import Reader as PCAPReader - from dpkt.pcapng import Reader as PCAPNGReader + from pcapkit.foundation.engines.pcapng import Context from pcapkit.foundation.extraction import Extractor + from pcapkit.protocols.data.misc.pcapng import \ + InterfaceDescriptionBlock as Data_InterfaceDescriptionBlock + from pcapkit.protocols.data.misc.pcapng import SectionHeaderBlock as Data_SectionHeaderBlock - Reader = Union[PCAPReader, PCAPNGReader] + Reader = Union[PCAPReader, 'PCAPNGReader'] #: logging.Logger: Module-level logger, a child of the package-wide #: :data:`pcapkit.utilities.logging.logger`. logger = get_logger(__name__) +class PCAPNGReader: + """PCAP-NG reader for the DPKT engine, aware of every interface. + + :class:`dpkt.pcapng.Reader` reads only the first Interface Description + Block (IDB) of a file, and applies its link type and timestamp resolution + to every packet (:issue:`1379`). This reader walks the blocks itself + instead: it parses each Section Header Block and IDB with :mod:`pcapkit`'s + own :class:`~pcapkit.protocols.misc.pcapng.PCAPNG`, keeps the interfaces + of the current section, and resolves every packet's link type, + ``if_tsresol`` and ``if_tsoffset`` from the interface it names. Packet + blocks are still decoded by `DPKT`_. + + As :class:`dpkt.pcapng.Reader` does, it yields Enhanced Packet Blocks and + (obsolete) Packet Blocks only, and skips every other block. + + .. _DPKT: https://dpkt.readthedocs.io + + Args: + file: Source PCAP-NG stream, positioned at its first Section Header Block. + + """ + + def __init__(self, file: 'IO[bytes]') -> 'None': + #: Source PCAP-NG stream. + self._file = file + #: Context of the current section. + self._ctx = cast('Context', None) + #: Section index number. + self._sect = 0 + #: Link type of the packet last yielded. + self._linktype = cast('Enum_LinkType', Enum_LinkType.NULL) + #: Underlying block iterator. + self._iter = self._read() + + def datalink(self) -> 'Enum_LinkType': + """Link type of the interface the packet last yielded was captured on.""" + return self._linktype + + def __iter__(self) -> 'PCAPNGReader': + return self + + def __next__(self) -> 'tuple[float, bytes]': + return next(self._iter) + + def _read(self) -> 'Iterator[tuple[float, bytes]]': + """Read blocks until the next packet block, and yield its timestamp and data.""" + from pcapkit.foundation.engines.pcapng import Context + from pcapkit.protocols.misc.pcapng import PCAPNG as P_PCAPNG + + while True: + head = self._file.read(8) + if len(head) < 8: + return + + # the SHB block type is a palindrome, so it reads the same in + # either byte order; any other block uses the section's order + byteorder: 'Literal["big", "little"]' + if head[:4] == b'\x0a\x0d\x0d\x0a': + head += self._file.read(4) + byteorder = 'little' if head[8:] == b'\x4d\x3c\x2b\x1a' else 'big' + else: + byteorder = self._ctx.section.byteorder + block_type = int.from_bytes(head[:4], byteorder) + length = int.from_bytes(head[4:8], byteorder) + if length < 12: + raise FormatError(f'PCAP-NG: [Block {block_type}] invalid block length: {length}') + + buf = head + self._file.read(length - len(head)) + if len(buf) < length: + return + + if block_type == Enum_BlockType.Section_Header_Block: + self._sect += 1 + shb = P_PCAPNG(buf, num=0, sct=self._sect, ctx=None) + self._ctx = Context(cast('Data_SectionHeaderBlock', shb.info)) + elif block_type == Enum_BlockType.Interface_Description_Block: + idb = P_PCAPNG(buf, num=0, sct=self._sect, ctx=self._ctx) + self._ctx.interfaces.append(cast('Data_InterfaceDescriptionBlock', idb.info)) + elif block_type in (Enum_BlockType.Enhanced_Packet_Block, Enum_BlockType.Packet_Block): + yield self._read_packet(buf, block_type, byteorder == 'little') + + def _read_packet(self, buf: 'bytes', block_type: 'int', + little: 'bool') -> 'tuple[float, bytes]': + """Decode a packet block with `DPKT`_, against the interface it names. + + Args: + buf: Whole block, as read from the file. + block_type: Block type, an EPB or an (obsolete) Packet Block. + little: Whether the enclosing section is little-endian. + + Returns: + Timestamp in seconds since the UNIX epoch, and the packet data. + + Raises: + FormatError: If the section describes no interface of the ID the + block names. + + """ + import dpkt # isort:skip + + if block_type == Enum_BlockType.Enhanced_Packet_Block: + tag = 'EPB' + pkt = (dpkt.pcapng.EnhancedPacketBlockLE(buf) if little + else dpkt.pcapng.EnhancedPacketBlock(buf)) + else: + tag = 'Packet' + pkt = dpkt.pcapng.PacketBlockLE(buf) if little else dpkt.pcapng.PacketBlock(buf) + + iface_id = pkt.iface_id # pylint: disable=no-member + if iface_id >= len(self._ctx.interfaces): + raise FormatError(f'PCAP-NG: [{tag}] invalid interface ID: {iface_id}') + interface = self._ctx.interfaces[iface_id] + + tsresol = interface.options.get(Enum_OptionType.if_tsresol) + tsoffset = interface.options.get(Enum_OptionType.if_tsoffset) + resolution = 1_000_000 if tsresol is None else tsresol.resolution + offset = 0 if tsoffset is None else tsoffset.offset + + # same arithmetic as :meth:`PCAPNG._read_timestamp + # ` + ticks = (pkt.ts_high << 32) | pkt.ts_low + with localcontext(prec=64): + timestamp = decimal.Decimal(ticks) / resolution + offset + + self._linktype = interface.linktype + return float(timestamp), pkt.pkt_data + + class DPKT(EngineBase['DPKTPacket']): """DPKT engine support. - On PCAP-NG, `DPKT`_'s reader yields Enhanced Packet Blocks and (obsolete) + A PCAP-NG file is read by :class:`PCAPNGReader` rather than by + :class:`dpkt.pcapng.Reader`, so each packet takes the link type and the + timestamp resolution and offset of the interface it was captured on. + + Like `DPKT`_'s reader, it yields Enhanced Packet Blocks and (obsolete) Packet Blocks only, and skips every other block without a warning -- the Simple Packet Block included. A packet carried in a Simple Packet Block is therefore missing from this engine's output; the default engine reads it. @@ -89,7 +228,8 @@ def run(self) -> 'None': This method assigns :attr:`self._expkg ` as :mod:`dpkt` and :attr:`self._extmp ` - as an iterator from :class:`dpkt.pcap.Reader`. + as an iterator from :class:`dpkt.pcap.Reader`, or from + :class:`PCAPNGReader` for a PCAP-NG file. The global header is parsed and dumped first, by :meth:`self.extractor.record_header `. @@ -139,7 +279,7 @@ def run(self) -> 'None': reader = dpkt.pcap.Reader(ext._ifile) elif ext.magic_number in PCAPNG.MAGIC_NUMBER: logger.debug('dpkt: reading %s as PCAP-NG', ext._ifnm) - reader = dpkt.pcapng.Reader(ext._ifile) + reader = PCAPNGReader(ext._ifile) else: raise FormatError(f'unsupported file format: {ext.magic_number!r}') @@ -163,10 +303,11 @@ def read_frame(self) -> 'DPKTPacket': ext = self._extractor reader = self._extmp - linktype = Enum_LinkType.get(reader.datalink()) - # fetch DPKT packet + # fetch DPKT packet; a PCAP-NG reader reports the link type of the + # interface this packet was captured on, so ask only after reading it timestamp, pkt = cast('tuple[float, bytes]', next(reader)) + linktype = Enum_LinkType.get(reader.datalink()) protocol = self._get_protocol(linktype) try: packet = protocol(pkt) # type: DPKTPacket diff --git a/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py b/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py new file mode 100644 index 0000000000..1fabd22040 --- /dev/null +++ b/tests/foundation/engines/test_dpkt_pcapng_interfaces_unit.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +"""The DPKT engine resolves every PCAP-NG packet against its own interface. + +GitHub issue #1379: :class:`dpkt.pcapng.Reader` reads only the first Interface +Description Block (IDB) of a file, so the DPKT engine gave every packet the +first interface's link type and ``if_tsresol`` -- a packet on a nanosecond +interface came out 1000 times too late, a raw-IP interface was dissected as +Ethernet, and a later section's interfaces were never read at all. + +Each case writes a capture built here to a temporary directory: two sections of +opposite byte order, whose interfaces differ in link type, ``if_tsresol`` and +``if_tsoffset``. Everything from :mod:`pcapkit` is imported inside each test, +after :func:`~tests._support.reimport_once_per_class`, not at module load. + +""" + +import importlib.util +import os +import struct +import tempfile +import unittest +import warnings + +from tests._support import reimport_once_per_class + +HAS_DPKT = importlib.util.find_spec('dpkt') is not None + +#: Minimal IPv4 header (no payload), protocol 59 -- usable under any IP link type. +IPV4 = bytes.fromhex('45000014000100004000') + bytes(2) + bytes([10, 0, 0, 1, 10, 0, 0, 2]) +#: Ethernet frame carrying :data:`IPV4`. +ETHERNET = bytes(6) + bytes([2, 0, 0, 0, 0, 1]) + b'\x08\x00' + IPV4 + +#: Raw timestamp written into every packet block. +TICKS = 1_500_000_000_000_123 + + +def _pad(data: bytes) -> bytes: + return data + bytes(-len(data) % 4) + + +def _block(order: str, kind: int, body: bytes) -> bytes: + body = _pad(body) + length = len(body) + 12 + return struct.pack(f'{order}II', kind, length) + body + struct.pack(f'{order}I', length) + + +def _shb(order: str) -> bytes: + return _block(order, 0x0A0D0D0A, struct.pack(f'{order}IHHq', 0x1A2B3C4D, 1, 0, -1)) + + +def _idb(order: str, linktype: int, *, tsresol: 'int | None' = None, + tsoffset: 'int | None' = None) -> bytes: + options = b'' + if tsresol is not None: + options += _pad(struct.pack(f'{order}HHB', 9, 1, tsresol)) + if tsoffset is not None: + options += struct.pack(f'{order}HHq', 14, 8, tsoffset) + if options: + options += struct.pack(f'{order}HH', 0, 0) + return _block(order, 1, struct.pack(f'{order}HHI', linktype, 0, 65535) + options) + + +def _epb(order: str, iface: int, data: bytes) -> bytes: + return _block(order, 6, struct.pack(f'{order}IIIII', iface, TICKS >> 32, TICKS & 0xFFFF_FFFF, + len(data), len(data)) + data) + + +#: Packets of :func:`_capture`: (link type, parsed-as, timestamp), in file order. +EXPECTED = [ + (1, 'Ethernet', TICKS / 10**6), # section 1, interface 0 + (101, 'RawPacket', TICKS / 10**9), # section 1, interface 1, if_tsresol=9 + (1, 'Ethernet', TICKS / 10**6), # section 1, interface 0 again + (228, 'IP', TICKS / 2**10 + 100), # section 2, interface 0, 2^-10 + offset +] + + +def _capture() -> bytes: + """Two sections, little- then big-endian, whose interfaces all differ.""" + return b''.join([ + _shb('<'), + _idb('<', 1), # Ethernet, microseconds + _idb('<', 101, tsresol=9), # raw IP, nanoseconds + _epb('<', 0, ETHERNET), + _epb('<', 1, IPV4), + _epb('<', 0, ETHERNET), + _shb('>'), + _idb('>', 228, tsresol=0x8A, tsoffset=100), # IPv4, 2^-10 s, +100 s + _epb('>', 0, IPV4), + ]) + + +@unittest.skipUnless(HAS_DPKT, 'dpkt is not installed') +class TestDPKTPCAPNGInterfaces(unittest.TestCase): + """Pin the per-interface link type and timestamp of the DPKT engine.""" + + def setUp(self) -> None: + reimport_once_per_class(self) + self._tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + self.addCleanup(self._tmp.cleanup) + self.path = os.path.join(self._tmp.name, 'interfaces.pcapng') + with open(self.path, 'wb') as file: + file.write(_capture()) + + def _extract(self, engine: str): # type: ignore[no-untyped-def] + import pcapkit + + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + return pcapkit.extract(fin=self.path, engine=engine, nofile=True, store=True) + + def test_reader_resolves_each_packets_interface(self) -> None: + from pcapkit.foundation.engines.dpkt import PCAPNGReader + + with open(self.path, 'rb') as file: + reader = PCAPNGReader(file) + seen = [(int(reader.datalink()), timestamp, data) + for timestamp, data in reader] + + self.assertEqual([(lt, ts) for lt, ts, _ in seen], + [(lt, ts) for lt, _, ts in EXPECTED]) + self.assertEqual([data for _, _, data in seen], [ETHERNET, IPV4, ETHERNET, IPV4]) + + def _assert_engines_agree(self, count: int) -> None: + from pcapkit.toolkit.dpkt import TIMESTAMP_ATTR + + default = self._extract('default') + dpkt = self._extract('dpkt') + + self.assertEqual(dpkt.length, count) + self.assertEqual([getattr(frame, TIMESTAMP_ATTR) for frame in dpkt.frame], + [float(frame.info.timestamp_epoch) for frame in default.frame]) + self.assertEqual([type(frame).__name__ for frame in dpkt.frame], + [name for _, name, _ in EXPECTED[:count]]) + + def test_dpkt_engine_matches_default_engine_across_interfaces(self) -> None: + # section 1 only: one byte order, two interfaces + with open(self.path, 'r+b') as file: + file.truncate(len(_capture()) - len(_shb('>') + _idb('>', 228, tsresol=0x8A, tsoffset=100) + + _epb('>', 0, IPV4))) + self._assert_engines_agree(3) + + def test_dpkt_engine_matches_default_engine_across_sections(self) -> None: + self._assert_engines_agree(len(EXPECTED)) + + def test_unknown_interface_id_is_a_format_error(self) -> None: + from pcapkit.foundation.engines.dpkt import PCAPNGReader + from pcapkit.utilities.exceptions import FormatError + + with open(self.path, 'wb') as file: + file.write(_shb('<') + _idb('<', 1) + _epb('<', 1, ETHERNET)) + with open(self.path, 'rb') as file: + with self.assertRaisesRegex(FormatError, 'invalid interface ID: 1'): + next(PCAPNGReader(file)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/foundation/engines/test_runtime_engines.py b/tests/foundation/engines/test_runtime_engines.py index ee81dbfdee..b30c667d73 100644 --- a/tests/foundation/engines/test_runtime_engines.py +++ b/tests/foundation/engines/test_runtime_engines.py @@ -236,7 +236,7 @@ def __next__(self): pcapng_reader = FakeReader() extractor.magic_number = PCAPNG.MAGIC_NUMBER[0] - with mock.patch.object(engine._expkg.pcapng, 'Reader', return_value=pcapng_reader): + with mock.patch('pcapkit.foundation.engines.dpkt.PCAPNGReader', return_value=pcapng_reader): engine.run() self.assertIs(engine._extmp, pcapng_reader)